diff options
Diffstat (limited to 'meta/lib/oe/rootfs.py')
| -rw-r--r-- | meta/lib/oe/rootfs.py | 640 |
1 files changed, 431 insertions, 209 deletions
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py index 7e8d5d15a2..96591f370e 100644 --- a/meta/lib/oe/rootfs.py +++ b/meta/lib/oe/rootfs.py @@ -10,17 +10,18 @@ import subprocess import re -class Rootfs(object): +class Rootfs(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta - def __init__(self, d): + def __init__(self, d, progress_reporter=None, logcatcher=None): self.d = d self.pm = None - self.image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) - self.deploy_dir_image = self.d.getVar('DEPLOY_DIR_IMAGE', True) + self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') + self.deploydir = self.d.getVar('IMGDEPLOYDIR') + self.progress_reporter = progress_reporter + self.logcatcher = logcatcher self.install_order = Manifest.INSTALL_ORDER @@ -40,10 +41,56 @@ class Rootfs(object): def _log_check(self): pass + def _log_check_common(self, type, match): + # Ignore any lines containing log_check to avoid recursion, and ignore + # lines beginning with a + since sh -x may emit code which isn't + # actually executed, but may contain error messages + excludes = [ 'log_check', r'^\+' ] + if hasattr(self, 'log_check_expected_regexes'): + excludes.extend(self.log_check_expected_regexes) + excludes = [re.compile(x) for x in excludes] + r = re.compile(match) + log_path = self.d.expand("${T}/log.do_rootfs") + messages = [] + with open(log_path, 'r') as log: + for line in log: + if self.logcatcher and self.logcatcher.contains(line.rstrip()): + continue + for ee in excludes: + m = ee.search(line) + if m: + break + if m: + continue + + m = r.search(line) + if m: + messages.append('[log_check] %s' % line) + if messages: + if len(messages) == 1: + msg = '1 %s message' % type + else: + msg = '%d %s messages' % (len(messages), type) + msg = '[log_check] %s: found %s in the logfile:\n%s' % \ + (self.d.getVar('PN'), msg, ''.join(messages)) + if type == 'error': + bb.fatal(msg) + else: + bb.warn(msg) + + def _log_check_warn(self): + self._log_check_common('warning', '^(warn|Warn|WARNING:)') + + def _log_check_error(self): + self._log_check_common('error', self.log_check_regex) + def _insert_feed_uris(self): if bb.utils.contains("IMAGE_FEATURES", "package-management", True, False, self.d): - self.pm.insert_feeds_uris() + self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", + self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", + self.d.getVar('PACKAGE_FEED_ARCHS')) + @abstractmethod def _handle_intercept_failure(self, failed_script): @@ -58,8 +105,61 @@ class Rootfs(object): def _cleanup(self): pass + def _setup_dbg_rootfs(self, dirs): + gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' + if gen_debugfs != '1': + return + + bb.note(" Renaming the original rootfs...") + try: + shutil.rmtree(self.image_rootfs + '-orig') + except: + pass + os.rename(self.image_rootfs, self.image_rootfs + '-orig') + + bb.note(" Creating debug rootfs...") + bb.utils.mkdirhier(self.image_rootfs) + + bb.note(" Copying back package database...") + for dir in dirs: + if not os.path.isdir(self.image_rootfs + '-orig' + dir): + continue + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) + shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) + + cpath = oe.cachedpath.CachedPath() + # Copy files located in /usr/lib/debug or /usr/src/debug + for dir in ["/usr/lib/debug", "/usr/src/debug"]: + src = self.image_rootfs + '-orig' + dir + if cpath.exists(src): + dst = self.image_rootfs + dir + bb.utils.mkdirhier(os.path.dirname(dst)) + shutil.copytree(src, dst) + + # Copy files with suffix '.debug' or located in '.debug' dir. + for root, dirs, files in cpath.walk(self.image_rootfs + '-orig'): + relative_dir = root[len(self.image_rootfs + '-orig'):] + for f in files: + if f.endswith('.debug') or '/.debug' in relative_dir: + bb.utils.mkdirhier(self.image_rootfs + relative_dir) + shutil.copy(os.path.join(root, f), + self.image_rootfs + relative_dir) + + bb.note(" Install complementary '*-dbg' packages...") + self.pm.install_complementary('*-dbg') + + bb.note(" Rename debug rootfs...") + try: + shutil.rmtree(self.image_rootfs + '-dbg') + except: + pass + os.rename(self.image_rootfs, self.image_rootfs + '-dbg') + + bb.note(" Restoreing original rootfs...") + os.rename(self.image_rootfs + '-orig', self.image_rootfs) + def _exec_shell_cmd(self, cmd): - fakerootcmd = self.d.getVar('FAKEROOT', True) + fakerootcmd = self.d.getVar('FAKEROOT') if fakerootcmd is not None: exec_cmd = [fakerootcmd, cmd] else: @@ -74,39 +174,46 @@ class Rootfs(object): def create(self): bb.note("###### Generate rootfs #######") - pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND", True) - post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND", True) - - intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True), + pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") + post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") + rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') + + postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR") + if not postinst_intercepts_dir: + postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts") + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") bb.utils.remove(intercepts_dir, True) bb.utils.mkdirhier(self.image_rootfs) - bb.utils.mkdirhier(self.deploy_dir_image) - - shutil.copytree(self.d.expand("${COREBASE}/scripts/postinst-intercepts"), - intercepts_dir) + bb.utils.mkdirhier(self.deploydir) - shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"), - self.deploy_dir_image + - "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt") + shutil.copytree(postinst_intercepts_dir, intercepts_dir) execute_pre_post_process(self.d, pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # call the package manager dependent create method self._create() - sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir', True) + sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') bb.utils.mkdirhier(sysconfdir) with open(sysconfdir + "/version", "w+") as ver: - ver.write(self.d.getVar('BUILDNAME', True) + "\n") + ver.write(self.d.getVar('BUILDNAME') + "\n") + + execute_pre_post_process(self.d, rootfs_post_install_cmds) self._run_intercepts() execute_pre_post_process(self.d, post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d): delayed_postinsts = self._get_delayed_postinsts() @@ -115,19 +222,27 @@ class Rootfs(object): "offline and rootfs is read-only: %s" % delayed_postinsts) - if self.d.getVar('USE_DEVFS', True) != "1": + if self.d.getVar('USE_DEVFS') != "1": self._create_devfs() self._uninstall_unneeded() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._insert_feed_uris() self._run_ldconfig() - if self.d.getVar('USE_DEPMOD', True) != "0": + if self.d.getVar('USE_DEPMOD') != "0": self._generate_kernel_module_deps() self._cleanup() + self._log_check() + + if self.progress_reporter: + self.progress_reporter.next_stage() + def _uninstall_unneeded(self): # Remove unneeded init script symlinks @@ -135,49 +250,49 @@ class Rootfs(object): if delayed_postinsts is None: if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")): self._exec_shell_cmd(["update-rc.d", "-f", "-r", - self.d.getVar('IMAGE_ROOTFS', True), + self.d.getVar('IMAGE_ROOTFS'), "run-postinsts", "remove"]) - runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", - True, False, self.d) - if not runtime_pkgmanage: - # Remove components that we don't need if we're not going to install - # additional packages at runtime - if delayed_postinsts is None: - installed_pkgs_dir = self.d.expand('${WORKDIR}/installed_pkgs.txt') - pkgs_to_remove = list() - with open(installed_pkgs_dir, "r+") as installed_pkgs: - pkgs_installed = installed_pkgs.read().split('\n') - for pkg_installed in pkgs_installed[:]: - pkg = pkg_installed.split()[0] - if pkg in ["update-rc.d", - "base-passwd", - self.d.getVar("ROOTFS_BOOTSTRAP_INSTALL", True) - ]: - pkgs_to_remove.append(pkg) - pkgs_installed.remove(pkg_installed) - - if len(pkgs_to_remove) > 0: - self.pm.remove(pkgs_to_remove, False) - # Update installed_pkgs.txt - open(installed_pkgs_dir, "w+").write('\n'.join(pkgs_installed)) - - else: - self._save_postinsts() - - post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND", True) + image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", + True, False, self.d) + image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') + + if image_rorfs or image_rorfs_force == "1": + # Remove components that we don't need if it's a read-only rootfs + unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() + pkgs_installed = image_list_installed_packages(self.d) + # Make sure update-alternatives is last on the command line, so + # that it is removed last. This makes sure that its database is + # available while uninstalling packages, allowing alternative + # symlinks of packages to be uninstalled to be managed correctly. + provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") + pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) + + if len(pkgs_to_remove) > 0: + self.pm.remove(pkgs_to_remove, False) + + if delayed_postinsts: + self._save_postinsts() + if image_rorfs: + bb.warn("There are post install scripts " + "in a read-only rootfs") + + post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") execute_pre_post_process(self.d, post_uninstall_cmds) + runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", + True, False, self.d) if not runtime_pkgmanage: # Remove the package manager data files self.pm.remove_packaging_data() def _run_intercepts(self): - intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True), + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") bb.note("Running intercept scripts:") os.environ['D'] = self.image_rootfs + os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE') for script in os.listdir(intercepts_dir): script_full = os.path.join(intercepts_dir, script) @@ -189,8 +304,8 @@ class Rootfs(object): try: subprocess.check_output(script_full) except subprocess.CalledProcessError as e: - bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details!" % - (script, e.returncode)) + bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details! (Output: %s)" % + (script, e.returncode, e.output)) with open(script_full) as intercept: registered_pkgs = None @@ -209,21 +324,35 @@ class Rootfs(object): self._handle_intercept_failure(registered_pkgs) def _run_ldconfig(self): - if self.d.getVar('LDCONFIGDEPEND', True): + if self.d.getVar('LDCONFIGDEPEND'): bb.note("Executing: ldconfig -r" + self.image_rootfs + "-c new -v") self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 'new', '-v']) + def _check_for_kernel_modules(self, modules_dir): + for root, dirs, files in os.walk(modules_dir, topdown=True): + for name in files: + found_ko = name.endswith(".ko") + if found_ko: + return found_ko + return False + def _generate_kernel_module_deps(self): - kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR', True), "kernel-depmod", + modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') + # if we don't have any modules don't bother to do the depmod + if not self._check_for_kernel_modules(modules_dir): + bb.note("No Kernel Modules found, not running depmod") + return + + kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod", 'kernel-abiversion') if not os.path.exists(kernel_abi_ver_file): bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) kernel_ver = open(kernel_abi_ver_file).read().strip(' \n') - modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules', kernel_ver) + versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) - bb.utils.mkdirhier(modules_dir) + bb.utils.mkdirhier(versioned_modules_dir) self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver]) @@ -237,15 +366,15 @@ class Rootfs(object): """ def _create_devfs(self): devtable_list = [] - devtable = self.d.getVar('IMAGE_DEVICE_TABLE', True) + devtable = self.d.getVar('IMAGE_DEVICE_TABLE') if devtable is not None: devtable_list.append(devtable) else: - devtables = self.d.getVar('IMAGE_DEVICE_TABLES', True) + devtables = self.d.getVar('IMAGE_DEVICE_TABLES') if devtables is None: devtables = 'files/device_table-minimal.txt' for devtable in devtables.split(): - devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH', True), devtable)) + devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) for devtable in devtable_list: self._exec_shell_cmd(["makedevs", "-r", @@ -253,22 +382,24 @@ class Rootfs(object): class RpmRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(RpmRootfs, self).__init__(d) - + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(RpmRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ + '|exit 1|ERROR: |Error: |Error |ERROR '\ + '|Failed |Failed: |Failed$|Failed\(\d+\):)' self.manifest = RpmManifest(d, manifest_dir) self.pm = RpmPM(d, - d.getVar('IMAGE_ROOTFS', True), - self.d.getVar('TARGET_VENDOR', True) + d.getVar('IMAGE_ROOTFS'), + self.d.getVar('TARGET_VENDOR') ) - self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN', True) + self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN') if self.inc_rpm_image_gen != "1": bb.utils.remove(self.image_rootfs, True) else: self.pm.recovery_packaging_data() - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) self.pm.create_configs() @@ -300,17 +431,27 @@ class RpmRootfs(Rootfs): bb.note('incremental removed: %s' % ' '.join(pkg_to_remove)) self.pm.remove(pkg_to_remove) + self.pm.autoremove() + def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() + rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') + rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') # update PM index files self.pm.write_index() - self.pm.dump_all_available_pkgs() + execute_pre_post_process(self.d, rpm_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() if self.inc_rpm_image_gen == "1": self._create_incremental(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.update() pkgs = [] @@ -321,18 +462,34 @@ class RpmRootfs(Rootfs): else: pkgs += pkgs_to_install[pkg_type] + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs_attempt, True) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() - self._log_check() + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) + + execute_pre_post_process(self.d, rpm_post_process_cmds) if self.inc_rpm_image_gen == "1": self.pm.backup_packaging_data() - self.pm.rpm_setup_smart_target_config() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): @@ -354,43 +511,6 @@ class RpmRootfs(Rootfs): # already saved in /etc/rpm-postinsts pass - def _log_check_warn(self): - r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn)') - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - for line in log.read().split('\n'): - if 'log_check' in line or 'NOTE:' in line: - continue - - m = r.search(line) - if m: - bb.warn('log_check: There is a warn message in the logfile') - bb.warn('log_check: Matched keyword: [%s]' % m.group()) - bb.warn('log_check: %s\n' % line) - - def _log_check_error(self): - r = re.compile('(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)') - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - found_error = 0 - message = "\n" - for line in log.read().split('\n'): - if 'log_check' in line: - continue - - m = r.search(line) - if m: - found_error = 1 - bb.warn('log_check: There were error messages in the logfile') - bb.warn('log_check: Matched keyword: [%s]\n\n' % m.group()) - - if found_error >= 1 and found_error <= 5: - message += line + '\n' - found_error += 1 - - if found_error == 6: - bb.fatal(message) - def _log_check(self): self._log_check_warn() self._log_check_error() @@ -404,28 +524,124 @@ class RpmRootfs(Rootfs): self.pm.save_rpmpostinst(pkg) def _cleanup(self): - # during the execution of postprocess commands, rpm is called several - # times to get the files installed, dependencies, etc. This creates the - # __db.00* (Berkeley DB files that hold locks, rpm specific environment - # settings, etc.), that should not get into the final rootfs - self.pm.unlock_rpm_db() - bb.utils.remove(self.image_rootfs + "/install", True) + pass + +class DpkgOpkgRootfs(Rootfs): + def __init__(self, d, progress_reporter=None, logcatcher=None): + super(DpkgOpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + + def _get_pkgs_postinsts(self, status_file): + def _get_pkg_depends_list(pkg_depends): + pkg_depends_list = [] + # filter version requirements like libc (>= 1.1) + for dep in pkg_depends.split(', '): + m_dep = re.match("^(.*) \(.*\)$", dep) + if m_dep: + dep = m_dep.group(1) + pkg_depends_list.append(dep) + + return pkg_depends_list + + pkgs = {} + pkg_name = "" + pkg_status_match = False + pkg_depends = "" + + with open(status_file) as status: + data = status.read() + status.close() + for line in data.split('\n'): + m_pkg = re.match("^Package: (.*)", line) + m_status = re.match("^Status:.*unpacked", line) + m_depends = re.match("^Depends: (.*)", line) + + if m_pkg is not None: + if pkg_name and pkg_status_match: + pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) + + pkg_name = m_pkg.group(1) + pkg_status_match = False + pkg_depends = "" + elif m_status is not None: + pkg_status_match = True + elif m_depends is not None: + pkg_depends = m_depends.group(1) + # remove package dependencies not in postinsts + pkg_names = list(pkgs.keys()) + for pkg_name in pkg_names: + deps = pkgs[pkg_name][:] -class DpkgRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(DpkgRootfs, self).__init__(d) + for d in deps: + if d not in pkg_names: + pkgs[pkg_name].remove(d) + + return pkgs + + def _get_delayed_postinsts_common(self, status_file): + def _dep_resolve(graph, node, resolved, seen): + seen.append(node) + + for edge in graph[node]: + if edge not in resolved: + if edge in seen: + raise RuntimeError("Packages %s and %s have " \ + "a circular dependency in postinsts scripts." \ + % (node, edge)) + _dep_resolve(graph, edge, resolved, seen) + + resolved.append(node) + + pkg_list = [] + + pkgs = None + if not self.d.getVar('PACKAGE_INSTALL').strip(): + bb.note("Building empty image") + else: + pkgs = self._get_pkgs_postinsts(status_file) + if pkgs: + root = "__packagegroup_postinst__" + pkgs[root] = list(pkgs.keys()) + _dep_resolve(pkgs, root, pkg_list, []) + pkg_list.remove(root) + + if len(pkg_list) == 0: + return None + + return pkg_list + + def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir): + num = 0 + for p in self._get_delayed_postinsts(): + bb.utils.mkdirhier(dst_postinst_dir) + + if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): + shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), + os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) + + num += 1 + +class DpkgRootfs(DpkgOpkgRootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(DpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '^E:' + self.log_check_expected_regexes = \ + [ + "^E: Unmet dependencies." + ] bb.utils.remove(self.image_rootfs, True) - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) self.manifest = DpkgManifest(d, manifest_dir) - self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS', True), - d.getVar('PACKAGE_ARCHS', True), - d.getVar('DPKG_ARCH', True)) + self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS'), + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH')) def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() + deb_pre_process_cmds = self.d.getVar('DEB_PREPROCESS_COMMANDS') + deb_post_process_cmds = self.d.getVar('DEB_POSTPROCESS_COMMANDS') alt_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/alternatives") bb.utils.mkdirhier(alt_dir) @@ -433,74 +649,80 @@ class DpkgRootfs(Rootfs): # update PM index files self.pm.write_index() + execute_pre_post_process(self.d, deb_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + # Don't support incremental, so skip that + self.progress_reporter.next_stage() + self.pm.update() + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + # Don't support attemptonly, so skip that + self.progress_reporter.next_stage() + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/var/lib/dpkg']) + self.pm.fix_broken_dependencies() self.pm.mark_packages("installed") self.pm.run_pre_post_installs() + execute_pre_post_process(self.d, deb_post_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): - return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMAND'] + return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS'] def _get_delayed_postinsts(self): - pkg_list = [] - with open(self.image_rootfs + "/var/lib/dpkg/status") as status: - for line in status: - m_pkg = re.match("^Package: (.*)", line) - m_status = re.match("^Status:.*unpacked", line) - if m_pkg is not None: - pkg_name = m_pkg.group(1) - elif m_status is not None: - pkg_list.append(pkg_name) - - if len(pkg_list) == 0: - return None - - return pkg_list + status_file = self.image_rootfs + "/var/lib/dpkg/status" + return self._get_delayed_postinsts_common(status_file) def _save_postinsts(self): - num = 0 - for p in self._get_delayed_postinsts(): - dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/deb-postinsts") - src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/info") - - bb.utils.mkdirhier(dst_postinst_dir) - - if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): - shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) - - num += 1 + dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/deb-postinsts") + src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/info") + return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) def _handle_intercept_failure(self, registered_pkgs): self.pm.mark_packages("unpacked", registered_pkgs.split()) def _log_check(self): - pass + self._log_check_warn() + self._log_check_error() def _cleanup(self): pass -class OpkgRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(OpkgRootfs, self).__init__(d) +class OpkgRootfs(DpkgOpkgRootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(OpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '(exit 1|Collected errors)' self.manifest = OpkgManifest(d, manifest_dir) - self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True) - self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True) + self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") + self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS") - self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or "" + self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN') or "" if self._remove_old_rootfs(): bb.utils.remove(self.image_rootfs, True) self.pm = OpkgPM(d, @@ -514,7 +736,7 @@ class OpkgRootfs(Rootfs): self.pkg_archs) self.pm.recover_packaging_data() - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) def _prelink_file(self, root_dir, filename): bb.note('prelink %s in %s' % (filename, root_dir)) @@ -569,7 +791,7 @@ class OpkgRootfs(Rootfs): """ def _multilib_sanity_test(self, dirs): - allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP", True) + allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") if allow_replace is None: allow_replace = "" @@ -601,12 +823,12 @@ class OpkgRootfs(Rootfs): files[key] = item def _multilib_test_install(self, pkgs): - ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS", True) + ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") bb.utils.mkdirhier(ml_temp) dirs = [self.image_rootfs] - for variant in self.d.getVar("MULTILIB_VARIANTS", True).split(): + for variant in self.d.getVar("MULTILIB_VARIANTS").split(): ml_target_rootfs = os.path.join(ml_temp, variant) bb.utils.remove(ml_target_rootfs, True) @@ -666,9 +888,9 @@ class OpkgRootfs(Rootfs): old_vars_list = open(vars_list_file, 'r+').read() new_vars_list = '%s:%s:%s\n' % \ - ((self.d.getVar('BAD_RECOMMENDATIONS', True) or '').strip(), - (self.d.getVar('NO_RECOMMENDATIONS', True) or '').strip(), - (self.d.getVar('PACKAGE_EXCLUDE', True) or '').strip()) + ((self.d.getVar('BAD_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('NO_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('PACKAGE_EXCLUDE') or '').strip()) open(vars_list_file, 'w+').write(new_vars_list) if old_vars_list != new_vars_list: @@ -678,23 +900,33 @@ class OpkgRootfs(Rootfs): def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() - opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True) - opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS', True) - rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND', True) + opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') + opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') # update PM index files, unless users provide their own feeds - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": self.pm.write_index() execute_pre_post_process(self.d, opkg_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Steps are a bit different in order, skip next + self.progress_reporter.next_stage() + self.pm.update() self.pm.handle_bad_recommendations() + if self.progress_reporter: + self.progress_reporter.next_stage() + if self.inc_opkg_image_gen == "1": self._remove_extra_packages(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: # For multilib, we perform a sanity test before final install @@ -706,60 +938,50 @@ class OpkgRootfs(Rootfs): self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') + self._setup_dbg_rootfs([opkg_dir]) + execute_pre_post_process(self.d, opkg_post_process_cmds) - execute_pre_post_process(self.d, rootfs_post_install_cmds) if self.inc_opkg_image_gen == "1": self.pm.backup_packaging_data() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] def _get_delayed_postinsts(self): - pkg_list = [] status_file = os.path.join(self.image_rootfs, - self.d.getVar('OPKGLIBDIR', True).strip('/'), + self.d.getVar('OPKGLIBDIR').strip('/'), "opkg", "status") - - with open(status_file) as status: - for line in status: - m_pkg = re.match("^Package: (.*)", line) - m_status = re.match("^Status:.*unpacked", line) - if m_pkg is not None: - pkg_name = m_pkg.group(1) - elif m_status is not None: - pkg_list.append(pkg_name) - - if len(pkg_list) == 0: - return None - - return pkg_list + return self._get_delayed_postinsts_common(status_file) def _save_postinsts(self): - num = 0 - for p in self._get_delayed_postinsts(): - dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") - src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") - - bb.utils.mkdirhier(dst_postinst_dir) - - if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): - shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) - - num += 1 + dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") + src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") + return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) def _handle_intercept_failure(self, registered_pkgs): self.pm.mark_packages("unpacked", registered_pkgs.split()) def _log_check(self): - pass + self._log_check_warn() + self._log_check_error() def _cleanup(self): - pass + self.pm.remove_lists() def get_class_for_type(imgtype): return {"rpm": RpmRootfs, @@ -767,36 +989,36 @@ def get_class_for_type(imgtype): "deb": DpkgRootfs}[imgtype] def variable_depends(d, manifest_dir=None): - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') cls = get_class_for_type(img_type) return cls._depends_list() -def create_rootfs(d, manifest_dir=None): +def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): env_bkp = os.environ.copy() - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": - RpmRootfs(d, manifest_dir).create() + RpmRootfs(d, manifest_dir, progress_reporter, logcatcher).create() elif img_type == "ipk": - OpkgRootfs(d, manifest_dir).create() + OpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() elif img_type == "deb": - DpkgRootfs(d, manifest_dir).create() + DpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() os.environ.clear() os.environ.update(env_bkp) -def image_list_installed_packages(d, format=None, rootfs_dir=None): +def image_list_installed_packages(d, rootfs_dir=None): if not rootfs_dir: - rootfs_dir = d.getVar('IMAGE_ROOTFS', True) + rootfs_dir = d.getVar('IMAGE_ROOTFS') - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": - return RpmPkgsList(d, rootfs_dir).list(format) + return RpmPkgsList(d, rootfs_dir).list_pkgs() elif img_type == "ipk": - return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET", True)).list(format) + return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET")).list_pkgs() elif img_type == "deb": - return DpkgPkgsList(d, rootfs_dir).list(format) + return DpkgPkgsList(d, rootfs_dir).list_pkgs() if __name__ == "__main__": """ |