diff options
Diffstat (limited to 'meta/lib/oe')
| -rw-r--r-- | meta/lib/oe/image.py | 412 |
1 files changed, 0 insertions, 412 deletions
diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py deleted file mode 100644 index b2b002b190..0000000000 --- a/meta/lib/oe/image.py +++ /dev/null @@ -1,412 +0,0 @@ -from oe.utils import execute_pre_post_process -import os -import subprocess -import multiprocessing - - -def generate_image(arg): - (type, subimages, create_img_cmd, sprefix) = arg - - bb.note("Running image creation script for %s: %s ..." % - (type, create_img_cmd)) - - try: - output = subprocess.check_output(create_img_cmd, - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - return("Error: The image creation script '%s' returned %d:\n%s" % - (e.cmd, e.returncode, e.output)) - - bb.note("Script output:\n%s" % output) - - return None - - -""" -This class will help compute IMAGE_FSTYPE dependencies and group them in batches -that can be executed in parallel. - -The next example is for illustration purposes, highly unlikely to happen in real life. -It's just one of the test cases I used to test the algorithm: - -For: -IMAGE_FSTYPES = "i1 i2 i3 i4 i5" -IMAGE_TYPEDEP_i4 = "i2" -IMAGE_TYPEDEP_i5 = "i6 i4" -IMAGE_TYPEDEP_i6 = "i7" -IMAGE_TYPEDEP_i7 = "i2" - -We get the following list of batches that can be executed in parallel, having the -dependencies satisfied: - -[['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']] -""" -class ImageDepGraph(object): - def __init__(self, d): - self.d = d - self.graph = dict() - self.deps_array = dict() - - def _construct_dep_graph(self, image_fstypes): - graph = dict() - - def add_node(node): - base_type = self._image_base_type(node) - deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "") - base_deps = (self.d.getVar('IMAGE_TYPEDEP_' + base_type, True) or "") - - graph[node] = "" - for dep in deps.split() + base_deps.split(): - if not dep in graph[node]: - if graph[node] != "": - graph[node] += " " - graph[node] += dep - - if not dep in graph: - add_node(dep) - - for fstype in image_fstypes: - add_node(fstype) - - return graph - - def _clean_graph(self): - # Live and VMDK/VDI images will be processed via inheriting - # bbclass and does not get processed here. Remove them from the fstypes - # graph. Their dependencies are already added, so no worries here. - remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split() - - for item in remove_list: - self.graph.pop(item, None) - - def _image_base_type(self, type): - ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() - if type in ["vmdk", "vdi", "qcow2", "live", "iso", "hddimg"]: - type = "ext4" - basetype = type - for ctype in ctypes: - if type.endswith("." + ctype): - basetype = type[:-len("." + ctype)] - break - - return basetype - - def _compute_dependencies(self): - """ - returns dict object of nodes with [no_of_depends_on, no_of_depended_by] - for each node - """ - deps_array = dict() - for node in self.graph: - deps_array[node] = [0, 0] - - for node in self.graph: - deps = self.graph[node].split() - deps_array[node][0] += len(deps) - for dep in deps: - deps_array[dep][1] += 1 - - return deps_array - - def _sort_graph(self): - sorted_list = [] - group = [] - for node in self.graph: - if node not in self.deps_array: - continue - - depends_on = self.deps_array[node][0] - - if depends_on == 0: - group.append(node) - - if len(group) == 0 and len(self.deps_array) != 0: - bb.fatal("possible fstype circular dependency...") - - sorted_list.append(group) - - # remove added nodes from deps_array - for item in group: - for node in self.graph: - if item in self.graph[node].split(): - self.deps_array[node][0] -= 1 - - self.deps_array.pop(item, None) - - if len(self.deps_array): - # recursive call, to find the next group - sorted_list += self._sort_graph() - - return sorted_list - - def group_fstypes(self, image_fstypes): - self.graph = self._construct_dep_graph(image_fstypes) - - self._clean_graph() - - self.deps_array = self._compute_dependencies() - - alltypes = [node for node in self.graph] - - return (alltypes, self._sort_graph()) - - -class Image(ImageDepGraph): - def __init__(self, d): - self.d = d - - super(Image, self).__init__(d) - - def _get_rootfs_size(self): - """compute the rootfs size""" - rootfs_alignment = int(self.d.getVar('IMAGE_ROOTFS_ALIGNMENT', True)) - overhead_factor = float(self.d.getVar('IMAGE_OVERHEAD_FACTOR', True)) - rootfs_req_size = int(self.d.getVar('IMAGE_ROOTFS_SIZE', True)) - rootfs_extra_space = eval(self.d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True)) - rootfs_maxsize = self.d.getVar('IMAGE_ROOTFS_MAXSIZE', True) - - output = subprocess.check_output(['du', '-ks', - self.d.getVar('IMAGE_ROOTFS', True)]) - size_kb = int(output.split()[0]) - base_size = size_kb * overhead_factor - base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \ - rootfs_extra_space - - if base_size != int(base_size): - base_size = int(base_size + 1) - else: - base_size = int(base_size) - - base_size += rootfs_alignment - 1 - base_size -= base_size % rootfs_alignment - - # Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set) - if rootfs_maxsize: - rootfs_maxsize_int = int(rootfs_maxsize) - if base_size > rootfs_maxsize_int: - bb.fatal("The rootfs size %d(K) overrides the max size %d(K)" % \ - (base_size, rootfs_maxsize_int)) - - return base_size - - def _create_symlinks(self, subimages): - """create symlinks to the newly created image""" - deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) - img_name = self.d.getVar('IMAGE_NAME', True) - link_name = self.d.getVar('IMAGE_LINK_NAME', True) - manifest_name = self.d.getVar('IMAGE_MANIFEST', True) - - os.chdir(deploy_dir) - - if link_name: - for type in subimages: - if os.path.exists(img_name + ".rootfs." + type): - dst = link_name + "." + type - src = img_name + ".rootfs." + type - bb.note("Creating symlink: %s -> %s" % (dst, src)) - os.symlink(src, dst) - - if manifest_name is not None and \ - os.path.exists(manifest_name) and \ - not os.path.exists(link_name + ".manifest"): - os.symlink(os.path.basename(manifest_name), - link_name + ".manifest") - - def _remove_old_symlinks(self): - """remove the symlinks to old binaries""" - - if self.d.getVar('IMAGE_LINK_NAME', True): - deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) - for img in os.listdir(deploy_dir): - if img.find(self.d.getVar('IMAGE_LINK_NAME', True)) == 0: - img = os.path.join(deploy_dir, img) - if os.path.islink(img): - if self.d.getVar('RM_OLD_IMAGE', True) == "1" and \ - os.path.exists(os.path.realpath(img)): - os.remove(os.path.realpath(img)) - - os.remove(img) - - """ - This function will just filter out the compressed image types from the - fstype groups returning a (filtered_fstype_groups, cimages) tuple. - """ - def _filter_out_commpressed(self, fstype_groups): - ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() - cimages = {} - - filtered_groups = [] - for group in fstype_groups: - filtered_group = [] - for type in group: - basetype = None - for ctype in ctypes: - if type.endswith("." + ctype): - basetype = type[:-len("." + ctype)] - if basetype not in filtered_group: - filtered_group.append(basetype) - if basetype not in cimages: - cimages[basetype] = [] - if ctype not in cimages[basetype]: - cimages[basetype].append(ctype) - break - if not basetype and type not in filtered_group: - filtered_group.append(type) - - filtered_groups.append(filtered_group) - - return (filtered_groups, cimages) - - def _get_image_types(self): - """returns a (types, cimages) tuple""" - - alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split()) - - filtered_groups, cimages = self._filter_out_commpressed(fstype_groups) - - return (alltypes, filtered_groups, cimages) - - def _write_script(self, type, cmds, sprefix=""): - tempdir = self.d.getVar('T', True) - script_name = os.path.join(tempdir, sprefix + "create_image." + type) - rootfs_size = self._get_rootfs_size() - - self.d.setVar('img_creation_func', '\n'.join(cmds)) - self.d.setVarFlag('img_creation_func', 'func', '1') - self.d.setVarFlag('img_creation_func', 'fakeroot', '1') - self.d.setVar('ROOTFS_SIZE', str(rootfs_size)) - - with open(script_name, "w+") as script: - script.write("%s" % bb.build.shell_trap_code()) - script.write("export ROOTFS_SIZE=%d\n" % rootfs_size) - bb.data.emit_func('img_creation_func', script, self.d) - script.write("img_creation_func\n") - - os.chmod(script_name, 0775) - - return script_name - - def _get_imagecmds(self, sprefix=""): - old_overrides = self.d.getVar('OVERRIDES', 0) - - alltypes, fstype_groups, cimages = self._get_image_types() - - image_cmd_groups = [] - - bb.note("The image creation groups are: %s" % str(fstype_groups)) - for fstype_group in fstype_groups: - image_cmds = [] - for type in fstype_group: - cmds = [] - subimages = [] - - localdata = bb.data.createCopy(self.d) - localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides)) - bb.data.update_data(localdata) - localdata.setVar('type', type) - - image_cmd = localdata.getVar("IMAGE_CMD", True) - if image_cmd: - cmds.append("\t" + image_cmd) - else: - bb.fatal("No IMAGE_CMD defined for IMAGE_FSTYPES entry '%s' - possibly invalid type name or missing support class" % type) - cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) - - if type in cimages: - for ctype in cimages[type]: - cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True)) - subimages.append(type + "." + ctype) - - if type not in alltypes: - cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) - else: - subimages.append(type) - - script_name = self._write_script(type, cmds, sprefix) - - image_cmds.append((type, subimages, script_name, sprefix)) - - image_cmd_groups.append(image_cmds) - - return image_cmd_groups - - def _write_wic_env(self): - """ - Write environment variables used by wic - to tmp/sysroots/<machine>/imgdata/<image>.env - """ - wicvars = self.d.getVar('WICVARS', True) - if not wicvars: - return - - stdir = self.d.getVar('STAGING_DIR_TARGET', True) - outdir = os.path.join(stdir, 'imgdata') - bb.utils.mkdirhier(outdir) - basename = self.d.getVar('IMAGE_BASENAME', True) - with open(os.path.join(outdir, basename) + '.env', 'w') as envf: - for var in wicvars.split(): - value = self.d.getVar(var, True) - if value: - envf.write('%s="%s"\n' % (var, value.strip())) - - def create(self): - bb.note("###### Generate images #######") - - image_cmd_groups = self._get_imagecmds() - - # Process the debug filesystem... - debugfs_d = bb.data.createCopy(self.d) - if self.d.getVar('IMAGE_GEN_DEBUGFS', True) == "1": - bb.note("Processing debugfs image(s) ...") - orig_d = self.d - self.d = debugfs_d - - self.d.setVar('IMAGE_ROOTFS', orig_d.getVar('IMAGE_ROOTFS', True) + '-dbg') - self.d.setVar('IMAGE_NAME', orig_d.getVar('IMAGE_NAME', True) + '-dbg') - self.d.setVar('IMAGE_LINK_NAME', orig_d.getVar('IMAGE_LINK_NAME', True) + '-dbg') - - debugfs_image_fstypes = orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True) - if debugfs_image_fstypes: - self.d.setVar('IMAGE_FSTYPES', orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True)) - - self._remove_old_symlinks() - - image_cmd_groups += self._get_imagecmds("debugfs.") - - self.d = orig_d - - self._write_wic_env() - - for image_cmds in image_cmd_groups: - # create the images in parallel - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(generate_image, image_cmds)) - pool.close() - pool.join() - - for result in results: - if result is not None: - bb.fatal(result) - - for image_type, subimages, script, sprefix in image_cmds: - if sprefix == 'debugfs.': - bb.note("Creating symlinks for %s debugfs image ..." % image_type) - orig_d = self.d - self.d = debugfs_d - self._create_symlinks(subimages) - self.d = orig_d - else: - bb.note("Creating symlinks for %s image ..." % image_type) - self._create_symlinks(subimages) - -def create_image(d): - Image(d).create() - -if __name__ == "__main__": - """ - Image creation can be called independent from bitbake environment. - """ - """ - TBD - """ |