From 31f0360f1fd4ebc9dfcaed42d1c50d2448b4632e Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Sat, 24 Aug 2013 15:31:34 +0000 Subject: wic: Add mic w/pykickstart This is the starting point for the implemention described in [YOCTO 3847] which came to the conclusion that it would make sense to use kickstart syntax to implement image creation in OpenEmbedded. I subsequently realized that there was an existing tool that already implemented image creation using kickstart syntax, the Tizen/Meego mic tool. As such, it made sense to use that as a starting point - this commit essentially just copies the relevant Python code from the MIC tool to the scripts/lib dir, where it can be accessed by the previously created wic tool. Most of this will be removed or renamed by later commits, since we're initially focusing on partitioning only. Care should be taken so that we can easily add back any additional functionality should we decide later to expand the tool, though (we may also want to contribute our local changes to the mic tool to the Tizen project if it makes sense, and therefore should avoid gratuitous changes to the original code if possible). Added the /mic subdir from Tizen mic repo as a starting point: git clone git://review.tizen.org/tools/mic.git For reference, the top commit: commit 20164175ddc234a17b8a12c33d04b012347b1530 Author: Gui Chen Date: Sun Jun 30 22:32:16 2013 -0400 bump up to 0.19.2 Also added the /plugins subdir, moved to under the /mic subdir (to match the default plugin_dir location in mic.conf.in, which was renamed to yocto-image.conf (moved and renamed by later patches) and put into /scripts. Signed-off-by: Tom Zanussi Signed-off-by: Saul Wold Signed-off-by: Richard Purdie --- scripts/lib/mic/imager/raw.py | 501 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 501 insertions(+) create mode 100644 scripts/lib/mic/imager/raw.py (limited to 'scripts/lib/mic/imager/raw.py') diff --git a/scripts/lib/mic/imager/raw.py b/scripts/lib/mic/imager/raw.py new file mode 100644 index 0000000000..838191a6f1 --- /dev/null +++ b/scripts/lib/mic/imager/raw.py @@ -0,0 +1,501 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import stat +import shutil + +from mic import kickstart, msger +from mic.utils import fs_related, runner, misc +from mic.utils.partitionedfs import PartitionedMount +from mic.utils.errors import CreatorError, MountError +from mic.imager.baseimager import BaseImageCreator + + +class RawImageCreator(BaseImageCreator): + """Installs a system into a file containing a partitioned disk image. + + ApplianceImageCreator is an advanced ImageCreator subclass; a sparse file + is formatted with a partition table, each partition loopback mounted + and the system installed into an virtual disk. The disk image can + subsequently be booted in a virtual machine or accessed with kpartx + """ + + def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None, generate_bmap=None, fstab_entry="uuid"): + """Initialize a ApplianceImageCreator instance. + + This method takes the same arguments as ImageCreator.__init__() + """ + BaseImageCreator.__init__(self, creatoropts, pkgmgr) + + self.__instloop = None + self.__imgdir = None + self.__disks = {} + self.__disk_format = "raw" + self._disk_names = [] + self._ptable_format = self.ks.handler.bootloader.ptable + self.vmem = 512 + self.vcpu = 1 + self.checksum = False + self.use_uuid = fstab_entry == "uuid" + self.appliance_version = None + self.appliance_release = None + self.compress_image = compress_image + self.bmap_needed = generate_bmap + self._need_extlinux = not kickstart.use_installerfw(self.ks, "extlinux") + #self.getsource = False + #self.listpkg = False + + self._dep_checks.extend(["sync", "kpartx", "parted"]) + if self._need_extlinux: + self._dep_checks.extend(["extlinux"]) + + def configure(self, repodata = None): + import subprocess + def chroot(): + os.chroot(self._instroot) + os.chdir("/") + + if os.path.exists(self._instroot + "/usr/bin/Xorg"): + subprocess.call(["/bin/chmod", "u+s", "/usr/bin/Xorg"], + preexec_fn = chroot) + + BaseImageCreator.configure(self, repodata) + + def _get_fstab(self): + if kickstart.use_installerfw(self.ks, "fstab"): + # The fstab file will be generated by installer framework scripts + # instead. + return None + + s = "" + for mp in self.__instloop.mountOrder: + p = None + for p1 in self.__instloop.partitions: + if p1['mountpoint'] == mp: + p = p1 + break + + if self.use_uuid and p['uuid']: + device = "UUID=%s" % p['uuid'] + else: + device = "/dev/%s%-d" % (p['disk_name'], p['num']) + + s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % { + 'device': device, + 'mountpoint': p['mountpoint'], + 'fstype': p['fstype'], + 'fsopts': "defaults,noatime" if not p['fsopts'] else p['fsopts']} + + if p['mountpoint'] == "/": + for subvol in self.__instloop.subvolumes: + if subvol['mountpoint'] == "/": + continue + s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % { + 'device': "/dev/%s%-d" % (p['disk_name'], p['num']), + 'mountpoint': subvol['mountpoint'], + 'fstype': p['fstype'], + 'fsopts': "defaults,noatime" if not subvol['fsopts'] else subvol['fsopts']} + + s += "devpts /dev/pts devpts gid=5,mode=620 0 0\n" + s += "tmpfs /dev/shm tmpfs defaults 0 0\n" + s += "proc /proc proc defaults 0 0\n" + s += "sysfs /sys sysfs defaults 0 0\n" + return s + + def _create_mkinitrd_config(self): + """write to tell which modules to be included in initrd""" + + mkinitrd = "" + mkinitrd += "PROBE=\"no\"\n" + mkinitrd += "MODULES+=\"ext3 ata_piix sd_mod libata scsi_mod\"\n" + mkinitrd += "rootfs=\"ext3\"\n" + mkinitrd += "rootopts=\"defaults\"\n" + + msger.debug("Writing mkinitrd config %s/etc/sysconfig/mkinitrd" \ + % self._instroot) + os.makedirs(self._instroot + "/etc/sysconfig/",mode=644) + cfg = open(self._instroot + "/etc/sysconfig/mkinitrd", "w") + cfg.write(mkinitrd) + cfg.close() + + def _get_parts(self): + if not self.ks: + raise CreatorError("Failed to get partition info, " + "please check your kickstart setting.") + + # Set a default partition if no partition is given out + if not self.ks.handler.partition.partitions: + partstr = "part / --size 1900 --ondisk sda --fstype=ext3" + args = partstr.split() + pd = self.ks.handler.partition.parse(args[1:]) + if pd not in self.ks.handler.partition.partitions: + self.ks.handler.partition.partitions.append(pd) + + # partitions list from kickstart file + return kickstart.get_partitions(self.ks) + + def get_disk_names(self): + """ Returns a list of physical target disk names (e.g., 'sdb') which + will be created. """ + + if self._disk_names: + return self._disk_names + + #get partition info from ks handler + parts = self._get_parts() + + for i in range(len(parts)): + if parts[i].disk: + disk_name = parts[i].disk + else: + raise CreatorError("Failed to create disks, no --ondisk " + "specified in partition line of ks file") + + if parts[i].mountpoint and not parts[i].fstype: + raise CreatorError("Failed to create disks, no --fstype " + "specified for partition with mountpoint " + "'%s' in the ks file") + + self._disk_names.append(disk_name) + + return self._disk_names + + def _full_name(self, name, extention): + """ Construct full file name for a file we generate. """ + return "%s-%s.%s" % (self.name, name, extention) + + def _full_path(self, path, name, extention): + """ Construct full file path to a file we generate. """ + return os.path.join(path, self._full_name(name, extention)) + + # + # Actual implemention + # + def _mount_instroot(self, base_on = None): + parts = self._get_parts() + self.__instloop = PartitionedMount(self._instroot) + + for p in parts: + self.__instloop.add_partition(int(p.size), + p.disk, + p.mountpoint, + p.fstype, + p.label, + fsopts = p.fsopts, + boot = p.active, + align = p.align, + part_type = p.part_type) + + self.__instloop.layout_partitions(self._ptable_format) + + # Create the disks + self.__imgdir = self._mkdtemp() + for disk_name, disk in self.__instloop.disks.items(): + full_path = self._full_path(self.__imgdir, disk_name, "raw") + msger.debug("Adding disk %s as %s with size %s bytes" \ + % (disk_name, full_path, disk['min_size'])) + + disk_obj = fs_related.SparseLoopbackDisk(full_path, + disk['min_size']) + self.__disks[disk_name] = disk_obj + self.__instloop.add_disk(disk_name, disk_obj) + + self.__instloop.mount() + self._create_mkinitrd_config() + + def _get_required_packages(self): + required_packages = BaseImageCreator._get_required_packages(self) + if self._need_extlinux: + if not self.target_arch or not self.target_arch.startswith("arm"): + required_packages += ["syslinux", "syslinux-extlinux"] + return required_packages + + def _get_excluded_packages(self): + return BaseImageCreator._get_excluded_packages(self) + + def _get_syslinux_boot_config(self): + rootdev = None + root_part_uuid = None + for p in self.__instloop.partitions: + if p['mountpoint'] == "/": + rootdev = "/dev/%s%-d" % (p['disk_name'], p['num']) + root_part_uuid = p['partuuid'] + + return (rootdev, root_part_uuid) + + def _create_syslinux_config(self): + + splash = os.path.join(self._instroot, "boot/extlinux") + if os.path.exists(splash): + splashline = "menu background splash.jpg" + else: + splashline = "" + + (rootdev, root_part_uuid) = self._get_syslinux_boot_config() + options = self.ks.handler.bootloader.appendLine + + #XXX don't hardcode default kernel - see livecd code + syslinux_conf = "" + syslinux_conf += "prompt 0\n" + syslinux_conf += "timeout 1\n" + syslinux_conf += "\n" + syslinux_conf += "default vesamenu.c32\n" + syslinux_conf += "menu autoboot Starting %s...\n" % self.distro_name + syslinux_conf += "menu hidden\n" + syslinux_conf += "\n" + syslinux_conf += "%s\n" % splashline + syslinux_conf += "menu title Welcome to %s!\n" % self.distro_name + syslinux_conf += "menu color border 0 #ffffffff #00000000\n" + syslinux_conf += "menu color sel 7 #ffffffff #ff000000\n" + syslinux_conf += "menu color title 0 #ffffffff #00000000\n" + syslinux_conf += "menu color tabmsg 0 #ffffffff #00000000\n" + syslinux_conf += "menu color unsel 0 #ffffffff #00000000\n" + syslinux_conf += "menu color hotsel 0 #ff000000 #ffffffff\n" + syslinux_conf += "menu color hotkey 7 #ffffffff #ff000000\n" + syslinux_conf += "menu color timeout_msg 0 #ffffffff #00000000\n" + syslinux_conf += "menu color timeout 0 #ffffffff #00000000\n" + syslinux_conf += "menu color cmdline 0 #ffffffff #00000000\n" + + versions = [] + kernels = self._get_kernel_versions() + symkern = "%s/boot/vmlinuz" % self._instroot + + if os.path.lexists(symkern): + v = os.path.realpath(symkern).replace('%s-' % symkern, "") + syslinux_conf += "label %s\n" % self.distro_name.lower() + syslinux_conf += "\tmenu label %s (%s)\n" % (self.distro_name, v) + syslinux_conf += "\tlinux ../vmlinuz\n" + if self._ptable_format == 'msdos': + rootstr = rootdev + else: + if not root_part_uuid: + raise MountError("Cannot find the root GPT partition UUID") + rootstr = "PARTUUID=%s" % root_part_uuid + syslinux_conf += "\tappend ro root=%s %s\n" % (rootstr, options) + syslinux_conf += "\tmenu default\n" + else: + for kernel in kernels: + for version in kernels[kernel]: + versions.append(version) + + footlabel = 0 + for v in versions: + syslinux_conf += "label %s%d\n" \ + % (self.distro_name.lower(), footlabel) + syslinux_conf += "\tmenu label %s (%s)\n" % (self.distro_name, v) + syslinux_conf += "\tlinux ../vmlinuz-%s\n" % v + syslinux_conf += "\tappend ro root=%s %s\n" \ + % (rootdev, options) + if footlabel == 0: + syslinux_conf += "\tmenu default\n" + footlabel += 1; + + msger.debug("Writing syslinux config %s/boot/extlinux/extlinux.conf" \ + % self._instroot) + cfg = open(self._instroot + "/boot/extlinux/extlinux.conf", "w") + cfg.write(syslinux_conf) + cfg.close() + + def _install_syslinux(self): + for name in self.__disks.keys(): + loopdev = self.__disks[name].device + + # Set MBR + mbrfile = "%s/usr/share/syslinux/" % self._instroot + if self._ptable_format == 'gpt': + mbrfile += "gptmbr.bin" + else: + mbrfile += "mbr.bin" + + msger.debug("Installing syslinux bootloader '%s' to %s" % \ + (mbrfile, loopdev)) + + mbrsize = os.stat(mbrfile)[stat.ST_SIZE] + rc = runner.show(['dd', 'if=%s' % mbrfile, 'of=' + loopdev]) + if rc != 0: + raise MountError("Unable to set MBR to %s" % loopdev) + + + # Ensure all data is flushed to disk before doing syslinux install + runner.quiet('sync') + + fullpathsyslinux = fs_related.find_binary_path("extlinux") + rc = runner.show([fullpathsyslinux, + "-i", + "%s/boot/extlinux" % self._instroot]) + if rc != 0: + raise MountError("Unable to install syslinux bootloader to %s" \ + % loopdev) + + def _create_bootconfig(self): + #If syslinux is available do the required configurations. + if self._need_extlinux \ + and os.path.exists("%s/usr/share/syslinux/" % (self._instroot)) \ + and os.path.exists("%s/boot/extlinux/" % (self._instroot)): + self._create_syslinux_config() + self._install_syslinux() + + def _unmount_instroot(self): + if not self.__instloop is None: + try: + self.__instloop.cleanup() + except MountError, err: + msger.warning("%s" % err) + + def _resparse(self, size = None): + return self.__instloop.resparse(size) + + def _get_post_scripts_env(self, in_chroot): + env = BaseImageCreator._get_post_scripts_env(self, in_chroot) + + # Export the file-system UUIDs and partition UUIDs (AKA PARTUUIDs) + for p in self.__instloop.partitions: + env.update(self._set_part_env(p['ks_pnum'], "UUID", p['uuid'])) + env.update(self._set_part_env(p['ks_pnum'], "PARTUUID", p['partuuid'])) + + return env + + def _stage_final_image(self): + """Stage the final system image in _outdir. + write meta data + """ + self._resparse() + + if self.compress_image: + for imgfile in os.listdir(self.__imgdir): + if imgfile.endswith('.raw') or imgfile.endswith('bin'): + imgpath = os.path.join(self.__imgdir, imgfile) + misc.compressing(imgpath, self.compress_image) + + if self.pack_to: + dst = os.path.join(self._outdir, self.pack_to) + msger.info("Pack all raw images to %s" % dst) + misc.packing(dst, self.__imgdir) + else: + msger.debug("moving disks to stage location") + for imgfile in os.listdir(self.__imgdir): + src = os.path.join(self.__imgdir, imgfile) + dst = os.path.join(self._outdir, imgfile) + msger.debug("moving %s to %s" % (src,dst)) + shutil.move(src,dst) + self._write_image_xml() + + def _write_image_xml(self): + imgarch = "i686" + if self.target_arch and self.target_arch.startswith("arm"): + imgarch = "arm" + xml = "\n" + + name_attributes = "" + if self.appliance_version: + name_attributes += " version='%s'" % self.appliance_version + if self.appliance_release: + name_attributes += " release='%s'" % self.appliance_release + xml += " %s\n" % (name_attributes, self.name) + xml += " \n" + # XXX don't hardcode - determine based on the kernel we installed for + # grub baremetal vs xen + xml += " \n" + xml += " \n" + xml += " %s\n" % imgarch + xml += " \n" + xml += " \n" + xml += " \n" + xml += " \n" + + i = 0 + for name in self.__disks.keys(): + full_name = self._full_name(name, self.__disk_format) + xml += " \n" \ + % (full_name, chr(ord('a') + i)) + i = i + 1 + + xml += " \n" + xml += " \n" + xml += " %s\n" % self.vcpu + xml += " %d\n" %(self.vmem * 1024) + for network in self.ks.handler.network.network: + xml += " \n" + xml += " \n" + xml += " \n" + xml += " \n" + xml += " \n" + + if self.checksum is True: + for name in self.__disks.keys(): + diskpath = self._full_path(self._outdir, name, \ + self.__disk_format) + full_name = self._full_name(name, self.__disk_format) + + msger.debug("Generating disk signature for %s" % full_name) + + xml += " \n" \ + % (full_name, self.__disk_format) + + hashes = misc.calc_hashes(diskpath, ('sha1', 'sha256')) + + xml += " %s\n" \ + % hashes[0] + xml += " %s\n" \ + % hashes[1] + xml += " \n" + else: + for name in self.__disks.keys(): + full_name = self._full_name(name, self.__disk_format) + xml += " \n" \ + % (full_name, self.__disk_format) + + xml += " \n" + xml += "\n" + + msger.debug("writing image XML to %s/%s.xml" %(self._outdir, self.name)) + cfg = open("%s/%s.xml" % (self._outdir, self.name), "w") + cfg.write(xml) + cfg.close() + + def generate_bmap(self): + """ Generate block map file for the image. The idea is that while disk + images we generate may be large (e.g., 4GiB), they may actually contain + only little real data, e.g., 512MiB. This data are files, directories, + file-system meta-data, partition table, etc. In other words, when + flashing the image to the target device, you do not have to copy all the + 4GiB of data, you can copy only 512MiB of it, which is 4 times faster. + + This function generates the block map file for an arbitrary image that + mic has generated. The block map file is basically an XML file which + contains a list of blocks which have to be copied to the target device. + The other blocks are not used and there is no need to copy them. """ + + if self.bmap_needed is None: + return + + from mic.utils import BmapCreate + msger.info("Generating the map file(s)") + + for name in self.__disks.keys(): + image = self._full_path(self.__imgdir, name, self.__disk_format) + bmap_file = self._full_path(self._outdir, name, "bmap") + + msger.debug("Generating block map file '%s'" % bmap_file) + + try: + creator = BmapCreate.BmapCreate(image, bmap_file) + creator.generate() + del creator + except BmapCreate.Error as err: + raise CreatorError("Failed to create bmap file: %s" % str(err)) -- cgit v1.2.3