diff options
Diffstat (limited to 'meta/classes')
182 files changed, 10797 insertions, 6122 deletions
diff --git a/meta/classes/allarch.bbclass b/meta/classes/allarch.bbclass index 2fea7c04df..a7ce024649 100644 --- a/meta/classes/allarch.bbclass +++ b/meta/classes/allarch.bbclass @@ -1,22 +1,19 @@ # -# This class is used for architecture independent recipes/data files (usally scripts) +# This class is used for architecture independent recipes/data files (usually scripts) # -# Expand STAGING_DIR_HOST since for cross-canadian/native/nativesdk, this will -# point elsewhere after these changes. -STAGING_DIR_HOST := "${STAGING_DIR_HOST}" - PACKAGE_ARCH = "all" python () { # Allow this class to be included but overridden - only set # the values if we're still "all" package arch. - if d.getVar("PACKAGE_ARCH", True) == "all": + if d.getVar("PACKAGE_ARCH") == "all": # No need for virtual/libc or a cross compiler d.setVar("INHIBIT_DEFAULT_DEPS","1") # Set these to a common set of values, we shouldn't be using them other that for WORKDIR directory # naming anyway + d.setVar("baselib", "lib") d.setVar("TARGET_ARCH", "allarch") d.setVar("TARGET_OS", "linux") d.setVar("TARGET_CC_ARCH", "none") @@ -24,9 +21,16 @@ python () { d.setVar("TARGET_AS_ARCH", "none") d.setVar("TARGET_FPU", "") d.setVar("TARGET_PREFIX", "") - d.setVar("PACKAGE_EXTRA_ARCHS", "") + # Expand PACKAGE_EXTRA_ARCHS since the staging code needs this + # (this removes any dependencies from the hash perspective) + d.setVar("PACKAGE_EXTRA_ARCHS", d.getVar("PACKAGE_EXTRA_ARCHS")) d.setVar("SDK_ARCH", "none") d.setVar("SDK_CC_ARCH", "none") + d.setVar("TARGET_CPPFLAGS", "none") + d.setVar("TARGET_CFLAGS", "none") + d.setVar("TARGET_CXXFLAGS", "none") + d.setVar("TARGET_LDFLAGS", "none") + d.setVar("POPULATESYSROOTDEPS", "") # Avoid this being unnecessarily different due to nuances of # the target machine that aren't important for "all" arch @@ -37,7 +41,11 @@ python () { d.setVar("EXCLUDE_FROM_SHLIBS", "1") d.setVar("INHIBIT_PACKAGE_DEBUG_SPLIT", "1") d.setVar("INHIBIT_PACKAGE_STRIP", "1") + + # These multilib values shouldn't change allarch packages so exclude them + d.setVarFlag("emit_pkgdata", "vardepsexclude", "MULTILIB_VARIANTS") + d.setVarFlag("write_specfile", "vardepsexclude", "MULTILIBS") elif bb.data.inherits_class('packagegroup', d) and not bb.data.inherits_class('nativesdk', d): - bb.error("Please ensure recipe %s sets PACKAGE_ARCH before inherit packagegroup" % d.getVar("FILE", True)) + bb.error("Please ensure recipe %s sets PACKAGE_ARCH before inherit packagegroup" % d.getVar("FILE")) } diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass index 089d70748c..2c04557f79 100644 --- a/meta/classes/archiver.bbclass +++ b/meta/classes/archiver.bbclass @@ -52,7 +52,13 @@ do_deploy_all_archives[dirs] = "${WORKDIR}" python () { - pn = d.getVar('PN', True) + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break included, reason = copyleft_should_include(d) if not included: @@ -61,9 +67,15 @@ python () { else: bb.debug(1, 'archiver: %s is included: %s' % (pn, reason)) - ar_src = d.getVarFlag('ARCHIVER_MODE', 'src', True) - ar_dumpdata = d.getVarFlag('ARCHIVER_MODE', 'dumpdata', True) - ar_recipe = d.getVarFlag('ARCHIVER_MODE', 'recipe', True) + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc'] \ + and not pn.startswith('gcc-source'): + bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn) + return + + ar_src = d.getVarFlag('ARCHIVER_MODE', 'src') + ar_dumpdata = d.getVarFlag('ARCHIVER_MODE', 'dumpdata') + ar_recipe = d.getVarFlag('ARCHIVER_MODE', 'recipe') if ar_src == "original": d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_original' % pn) @@ -73,8 +85,15 @@ python () { # We can't use "addtask do_ar_configured after do_configure" since it # will cause the deptask of do_populate_sysroot to run not matter what # archives we need, so we add the depends here. - d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn) + + # There is a corner case with "gcc-source-${PV}" recipes, they don't have + # the "do_configure" task, so we need to use "do_preconfigure" + if pn.startswith("gcc-source-"): + d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_preconfigure' % pn) + else: + d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn) d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn) + elif ar_src: bb.fatal("Invalid ARCHIVER_MODE[src]: %s" % ar_src) @@ -85,9 +104,9 @@ python () { d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_recipe' % pn) # Output the srpm package - ar_srpm = d.getVarFlag('ARCHIVER_MODE', 'srpm', True) + ar_srpm = d.getVarFlag('ARCHIVER_MODE', 'srpm') if ar_srpm == "1": - if d.getVar('PACKAGES', True) != '' and d.getVar('IMAGE_PKGTYPE', True) == 'rpm': + if d.getVar('PACKAGES') != '' and d.getVar('IMAGE_PKGTYPE') == 'rpm': d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn) if ar_dumpdata == "1": d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn) @@ -99,27 +118,6 @@ python () { d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_patched' % pn) elif ar_src == "configured": d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn) - - # The gcc staff uses shared source - flag = d.getVarFlag("do_unpack", "stamp-base", True) - if flag: - if ar_src in [ 'original', 'patched' ]: - ar_outdir = os.path.join(d.getVar('ARCHIVER_TOPDIR', True), 'work-shared') - d.setVar('ARCHIVER_OUTDIR', ar_outdir) - d.setVarFlag('do_ar_original', 'stamp-base', flag) - d.setVarFlag('do_ar_patched', 'stamp-base', flag) - d.setVarFlag('do_unpack_and_patch', 'stamp-base', flag) - d.setVarFlag('do_ar_original', 'vardepsexclude', 'PN PF ARCHIVER_OUTDIR WORKDIR') - d.setVarFlag('do_unpack_and_patch', 'vardepsexclude', 'PN PF ARCHIVER_OUTDIR WORKDIR') - d.setVarFlag('do_ar_patched', 'vardepsexclude', 'PN PF ARCHIVER_OUTDIR WORKDIR') - d.setVarFlag('create_diff_gz', 'vardepsexclude', 'PF') - d.setVarFlag('create_tarball', 'vardepsexclude', 'PF') - - flag_clean = d.getVarFlag('do_unpack', 'stamp-base-clean', True) - if flag_clean: - d.setVarFlag('do_ar_original', 'stamp-base-clean', flag_clean) - d.setVarFlag('do_ar_patched', 'stamp-base-clean', flag_clean) - d.setVarFlag('do_unpack_and_patch', 'stamp-base-clean', flag_clean) } # Take all the sources for a recipe and puts them in WORKDIR/archiver-work/. @@ -127,34 +125,55 @@ python () { # (e.g. git repositories) is "unpacked" and then put into a tarball. python do_ar_original() { - import shutil, tarfile, tempfile + import shutil, tempfile - if d.getVarFlag('ARCHIVER_MODE', 'src', True) != "original": + if d.getVarFlag('ARCHIVER_MODE', 'src') != "original": return - ar_outdir = d.getVar('ARCHIVER_OUTDIR', True) + ar_outdir = d.getVar('ARCHIVER_OUTDIR') bb.note('Archiving the original source...') - fetch = bb.fetch2.Fetch([], d) + urls = d.getVar("SRC_URI").split() + # destsuffix (git fetcher) and subdir (everything else) are allowed to be + # absolute paths (for example, destsuffix=${S}/foobar). + # That messes with unpacking inside our tmpdir below, because the fetchers + # will then unpack in that directory and completely ignore the tmpdir. + # That breaks parallel tasks relying on ${S}, like do_compile. + # + # To solve this, we remove these parameters from all URLs. + # We do this even for relative paths because it makes the content of the + # archives more useful (no extra paths that are only used during + # compilation). + for i, url in enumerate(urls): + decoded = bb.fetch2.decodeurl(url) + for param in ('destsuffix', 'subdir'): + if param in decoded[5]: + del decoded[5][param] + encoded = bb.fetch2.encodeurl(decoded) + urls[i] = encoded + fetch = bb.fetch2.Fetch(urls, d) + tarball_suffix = {} for url in fetch.urls: local = fetch.localpath(url).rstrip("/"); if os.path.isfile(local): shutil.copy(local, ar_outdir) elif os.path.isdir(local): - basename = os.path.basename(local) - - tmpdir = tempfile.mkdtemp(dir=d.getVar('ARCHIVER_WORKDIR', True)) + tmpdir = tempfile.mkdtemp(dir=d.getVar('ARCHIVER_WORKDIR')) fetch.unpack(tmpdir, (url,)) - - os.chdir(tmpdir) - # We eliminate any AUTOINC+ in the revision. - try: - src_rev = bb.fetch2.get_srcrev(d).replace('AUTOINC+','') - except: - src_rev = 'NOREV' - tarname = os.path.join(ar_outdir, basename + '.' + src_rev + '.tar.gz') - tar = tarfile.open(tarname, 'w:gz') - tar.add('.') - tar.close() + # To handle recipes with more than one source, we add the "name" + # URL parameter as suffix. We treat it as an error when + # there's more than one URL without a name, or a name gets reused. + # This is an additional safety net, in practice the name has + # to be set when using the git fetcher, otherwise SRCREV cannot + # be set separately for each URL. + params = bb.fetch2.decodeurl(url)[5] + name = params.get('name', '') + if name in tarball_suffix: + if not name: + bb.fatal("Cannot determine archive names for original source because 'name' URL parameter is unset in more than one URL. Add it to at least one of these: %s %s" % (tarball_suffix[name], url)) + else: + bb.fatal("Cannot determine archive names for original source because 'name=' URL parameter '%s' is used twice. Make it unique in: %s %s" % (tarball_suffix[name], url)) + tarball_suffix[name] = url + create_tarball(d, tmpdir + '/.', name, ar_outdir) # Emit patch series files for 'original' bb.note('Writing patch series files...') @@ -172,76 +191,79 @@ python do_ar_original() { python do_ar_patched() { - if d.getVarFlag('ARCHIVER_MODE', 'src', True) != 'patched': + if d.getVarFlag('ARCHIVER_MODE', 'src') != 'patched': return # Get the ARCHIVER_OUTDIR before we reset the WORKDIR - ar_outdir = d.getVar('ARCHIVER_OUTDIR', True) + ar_outdir = d.getVar('ARCHIVER_OUTDIR') + ar_workdir = d.getVar('ARCHIVER_WORKDIR') bb.note('Archiving the patched source...') - d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR', True)) - # The gcc staff uses shared source - flag = d.getVarFlag('do_unpack', 'stamp-base', True) - if flag: - create_tarball(d, d.getVar('S', True), 'patched', ar_outdir, 'gcc') - else: - create_tarball(d, d.getVar('S', True), 'patched', ar_outdir) + d.setVar('WORKDIR', ar_workdir) + create_tarball(d, d.getVar('S'), 'patched', ar_outdir) } python do_ar_configured() { import shutil - ar_outdir = d.getVar('ARCHIVER_OUTDIR', True) - if d.getVarFlag('ARCHIVER_MODE', 'src', True) == 'configured': + ar_outdir = d.getVar('ARCHIVER_OUTDIR') + if d.getVarFlag('ARCHIVER_MODE', 'src') == 'configured': bb.note('Archiving the configured source...') + pn = d.getVar('PN') + # "gcc-source-${PV}" recipes don't have "do_configure" + # task, so we need to run "do_preconfigure" instead + if pn.startswith("gcc-source-"): + d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR')) + bb.build.exec_func('do_preconfigure', d) + # The libtool-native's do_configure will remove the # ${STAGING_DATADIR}/aclocal/libtool.m4, so we can't re-run the # do_configure, we archive the already configured ${S} to # instead of. - if d.getVar('PN', True) != 'libtool-native': + elif pn != 'libtool-native': # Change the WORKDIR to make do_configure run in another dir. - d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR', True)) + d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR')) if bb.data.inherits_class('kernel-yocto', d): bb.build.exec_func('do_kernel_configme', d) if bb.data.inherits_class('cmake', d): bb.build.exec_func('do_generate_toolchain_file', d) - prefuncs = d.getVarFlag('do_configure', 'prefuncs', True) + prefuncs = d.getVarFlag('do_configure', 'prefuncs') for func in (prefuncs or '').split(): if func != "sysroot_cleansstate": bb.build.exec_func(func, d) bb.build.exec_func('do_configure', d) - postfuncs = d.getVarFlag('do_configure', 'postfuncs', True) + postfuncs = d.getVarFlag('do_configure', 'postfuncs') for func in (postfuncs or '').split(): if func != "do_qa_configure": bb.build.exec_func(func, d) - srcdir = d.getVar('S', True) - builddir = d.getVar('B', True) + srcdir = d.getVar('S') + builddir = d.getVar('B') if srcdir != builddir: if os.path.exists(builddir): oe.path.copytree(builddir, os.path.join(srcdir, \ - 'build.%s.ar_configured' % d.getVar('PF', True))) + 'build.%s.ar_configured' % d.getVar('PF'))) create_tarball(d, srcdir, 'configured', ar_outdir) } -def create_tarball(d, srcdir, suffix, ar_outdir, pf=None): +def create_tarball(d, srcdir, suffix, ar_outdir): """ create the tarball from srcdir """ import tarfile + # Make sure we are only creating a single tarball for gcc sources + if (d.getVar('SRC_URI') == ""): + return + bb.utils.mkdirhier(ar_outdir) - if pf: - tarname = os.path.join(ar_outdir, '%s-%s.tar.gz' % (pf, suffix)) + if suffix: + filename = '%s-%s.tar.gz' % (d.getVar('PF'), suffix) else: - tarname = os.path.join(ar_outdir, '%s-%s.tar.gz' % \ - (d.getVar('PF', True), suffix)) + filename = '%s.tar.gz' % d.getVar('PF') + tarname = os.path.join(ar_outdir, filename) - srcdir = srcdir.rstrip('/') - dirname = os.path.dirname(srcdir) - basename = os.path.basename(srcdir) - os.chdir(dirname) bb.note('Creating %s' % tarname) tar = tarfile.open(tarname, 'w:gz') - tar.add(basename) + tar.add(srcdir, arcname=os.path.basename(srcdir)) tar.close() # creating .diff.gz between source.orig and source @@ -257,51 +279,52 @@ def create_diff_gz(d, src_orig, src, ar_outdir): # exclude. src_patched = src + '.patched' oe.path.copyhardlinktree(src, src_patched) - for i in d.getVarFlag('ARCHIVER_MODE', 'diff-exclude', True).split(): + for i in d.getVarFlag('ARCHIVER_MODE', 'diff-exclude').split(): bb.utils.remove(os.path.join(src_orig, i), recurse=True) bb.utils.remove(os.path.join(src_patched, i), recurse=True) dirname = os.path.dirname(src) basename = os.path.basename(src) os.chdir(dirname) - out_file = os.path.join(ar_outdir, '%s-diff.gz' % d.getVar('PF', True)) + out_file = os.path.join(ar_outdir, '%s-diff.gz' % d.getVar('PF')) diff_cmd = 'diff -Naur %s.orig %s.patched | gzip -c > %s' % (basename, basename, out_file) subprocess.call(diff_cmd, shell=True) bb.utils.remove(src_patched, recurse=True) # Run do_unpack and do_patch python do_unpack_and_patch() { - if d.getVarFlag('ARCHIVER_MODE', 'src', True) not in \ + if d.getVarFlag('ARCHIVER_MODE', 'src') not in \ [ 'patched', 'configured'] and \ - d.getVarFlag('ARCHIVER_MODE', 'diff', True) != '1': + d.getVarFlag('ARCHIVER_MODE', 'diff') != '1': return + ar_outdir = d.getVar('ARCHIVER_OUTDIR') + ar_workdir = d.getVar('ARCHIVER_WORKDIR') + pn = d.getVar('PN') - ar_outdir = d.getVar('ARCHIVER_OUTDIR', True) - - # Change the WORKDIR to make do_unpack do_patch run in another dir. - d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR', True)) + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not (bb.data.inherits_class('kernel-yocto', d) or pn.startswith('gcc-source')): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', ar_workdir) - # The changed 'WORKDIR' also casued 'B' changed, create dir 'B' for the - # possibly requiring of the following tasks (such as some recipes's - # do_patch required 'B' existed). - bb.utils.mkdirhier(d.getVar('B', True)) + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) - # The kernel source is ready after do_validate_branches - if bb.data.inherits_class('kernel-yocto', d): - bb.build.exec_func('do_unpack', d) - bb.build.exec_func('do_kernel_checkout', d) - bb.build.exec_func('do_validate_branches', d) - else: bb.build.exec_func('do_unpack', d) # Save the original source for creating the patches - if d.getVarFlag('ARCHIVER_MODE', 'diff', True) == '1': - src = d.getVar('S', True).rstrip('/') + if d.getVarFlag('ARCHIVER_MODE', 'diff') == '1': + src = d.getVar('S').rstrip('/') src_orig = '%s.orig' % src oe.path.copytree(src, src_orig) - bb.build.exec_func('do_patch', d) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or (bb.data.inherits_class('kernel-yocto', d) or pn.startswith('gcc-source'))): + bb.build.exec_func('do_patch', d) + # Create the patches - if d.getVarFlag('ARCHIVER_MODE', 'diff', True) == '1': + if d.getVarFlag('ARCHIVER_MODE', 'diff') == '1': bb.note('Creating diff gz...') create_diff_gz(d, src_orig, src, ar_outdir) bb.utils.remove(src_orig, recurse=True) @@ -316,14 +339,24 @@ python do_ar_recipe () { require_re = re.compile( r"require\s+(.+)" ) include_re = re.compile( r"include\s+(.+)" ) - bbfile = d.getVar('FILE', True) - outdir = os.path.join(d.getVar('WORKDIR', True), \ - '%s-recipe' % d.getVar('PF', True)) + bbfile = d.getVar('FILE') + outdir = os.path.join(d.getVar('WORKDIR'), \ + '%s-recipe' % d.getVar('PF')) bb.utils.mkdirhier(outdir) shutil.copy(bbfile, outdir) + pn = d.getVar('PN') + bbappend_files = d.getVar('BBINCLUDED').split() + # If recipe name is aa, we need to match files like aa.bbappend and aa_1.1.bbappend + # Files like aa1.bbappend or aa1_1.1.bbappend must be excluded. + bbappend_re = re.compile( r".*/%s_[^/]*\.bbappend$" %pn) + bbappend_re1 = re.compile( r".*/%s\.bbappend$" %pn) + for file in bbappend_files: + if bbappend_re.match(file) or bbappend_re1.match(file): + shutil.copy(file, outdir) + dirname = os.path.dirname(bbfile) - bbpath = '%s:%s' % (dirname, d.getVar('BBPATH', True)) + bbpath = '%s:%s' % (dirname, d.getVar('BBPATH')) f = open(bbfile, 'r') for line in f.readlines(): incfile = None @@ -332,12 +365,12 @@ python do_ar_recipe () { elif include_re.match(line): incfile = include_re.match(line).group(1) if incfile: - incfile = bb.data.expand(incfile, d) + incfile = d.expand(incfile) incfile = bb.utils.which(bbpath, incfile) if incfile: shutil.copy(incfile, outdir) - create_tarball(d, outdir, 'recipe', d.getVar('ARCHIVER_OUTDIR', True)) + create_tarball(d, outdir, 'recipe', d.getVar('ARCHIVER_OUTDIR')) bb.utils.remove(outdir, recurse=True) } @@ -346,28 +379,29 @@ python do_dumpdata () { dump environment data to ${PF}-showdata.dump """ - dumpfile = os.path.join(d.getVar('ARCHIVER_OUTDIR', True), \ - '%s-showdata.dump' % d.getVar('PF', True)) + dumpfile = os.path.join(d.getVar('ARCHIVER_OUTDIR'), \ + '%s-showdata.dump' % d.getVar('PF')) bb.note('Dumping metadata into %s' % dumpfile) - f = open(dumpfile, 'w') - # emit variables and shell functions - bb.data.emit_env(f, d, True) - # emit the metadata which isn't valid shell - for e in d.keys(): - if bb.data.getVarFlag(e, 'python', d): - f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, True))) - f.close() + with open(dumpfile, "w") as f: + # emit variables and shell functions + bb.data.emit_env(f, d, True) + # emit the metadata which isn't valid shell + for e in d.keys(): + if d.getVarFlag(e, "python", False): + f.write("\npython %s () {\n%s}\n" % (e, d.getVar(e, False))) } SSTATETASKS += "do_deploy_archives" do_deploy_archives () { - echo "Deploying source archive files ..." + echo "Deploying source archive files from ${ARCHIVER_TOPDIR} to ${DEPLOY_DIR_SRC}." } python do_deploy_archives_setscene () { sstate_setscene(d) } +do_deploy_archives[dirs] = "${ARCHIVER_TOPDIR}" do_deploy_archives[sstate-inputdirs] = "${ARCHIVER_TOPDIR}" do_deploy_archives[sstate-outputdirs] = "${DEPLOY_DIR_SRC}" +addtask do_deploy_archives_setscene addtask do_ar_original after do_unpack addtask do_unpack_and_patch after do_patch @@ -383,3 +417,9 @@ do_deploy_all_archives[recideptask] = "do_${BB_DEFAULT_TASK}" do_deploy_all_archives() { : } + +python () { + # Add tasks in the correct order, specifically for linux-yocto to avoid race condition + if bb.data.inherits_class('kernel-yocto', d): + bb.build.addtask('do_kernel_configme', 'do_configure', 'do_unpack_and_patch', d) +} diff --git a/meta/classes/autotools.bbclass b/meta/classes/autotools.bbclass index 9ccd7d2e2d..ac04a07cb5 100644 --- a/meta/classes/autotools.bbclass +++ b/meta/classes/autotools.bbclass @@ -1,8 +1,8 @@ def autotools_dep_prepend(d): - if d.getVar('INHIBIT_AUTOTOOLS_DEPS', True): + if d.getVar('INHIBIT_AUTOTOOLS_DEPS'): return '' - pn = d.getVar('PN', True) + pn = d.getVar('PN') deps = '' if pn in ['autoconf-native', 'automake-native', 'help2man-native']: @@ -14,22 +14,20 @@ def autotools_dep_prepend(d): if not bb.data.inherits_class('native', d) \ and not bb.data.inherits_class('nativesdk', d) \ and not bb.data.inherits_class('cross', d) \ - and not d.getVar('INHIBIT_DEFAULT_DEPS', True): + and not d.getVar('INHIBIT_DEFAULT_DEPS'): deps += 'libtool-cross ' return deps + 'gnu-config-native ' -EXTRA_OEMAKE = "" - DEPENDS_prepend = "${@autotools_dep_prepend(d)} " inherit siteinfo # Space separated list of shell scripts with variables defined to supply test # results for autoconf tests we cannot run at build time. -export CONFIG_SITE = "${@siteinfo_get_files(d, False)}" +export CONFIG_SITE = "${@siteinfo_get_files(d)}" -acpaths = "default" +acpaths ?= "default" EXTRA_AUTORECONF = "--exclude=autopoint" export lt_cv_sys_lib_dlsearch_path_spec = "${libdir} ${base_libdir}" @@ -77,26 +75,29 @@ CONFIGUREOPTS = " --build=${BUILD_SYS} \ ${@append_libtool_sysroot(d)}" CONFIGUREOPT_DEPTRACK ?= "--disable-dependency-tracking" +CACHED_CONFIGUREVARS ?= "" + +AUTOTOOLS_SCRIPT_PATH ?= "${S}" +CONFIGURE_SCRIPT ?= "${AUTOTOOLS_SCRIPT_PATH}/configure" + +AUTOTOOLS_AUXDIR ?= "${AUTOTOOLS_SCRIPT_PATH}" oe_runconf () { - cfgscript="${S}/configure" + # Use relative path to avoid buildpaths in files + cfgscript_name="`basename ${CONFIGURE_SCRIPT}`" + cfgscript=`python3 -c "import os; print(os.path.relpath(os.path.dirname('${CONFIGURE_SCRIPT}'), '.'))"`/$cfgscript_name if [ -x "$cfgscript" ] ; then bbnote "Running $cfgscript ${CONFIGUREOPTS} ${EXTRA_OECONF} $@" - set +e - ${CACHED_CONFIGUREVARS} $cfgscript ${CONFIGUREOPTS} ${EXTRA_OECONF} "$@" - if [ "$?" != "0" ]; then - echo "Configure failed. The contents of all config.log files follows to aid debugging" - find ${S} -ignore_readdir_race -name config.log -print -exec cat {} \; - die "oe_runconf failed" + if ! ${CACHED_CONFIGUREVARS} $cfgscript ${CONFIGUREOPTS} ${EXTRA_OECONF} "$@"; then + bbnote "The following config.log files may provide further information." + bbnote `find ${B} -ignore_readdir_race -type f -name config.log` + bbfatal_log "configure failed" fi - set -e else bbfatal "no configure script found at $cfgscript" fi } -AUTOTOOLS_AUXDIR ?= "${S}" - CONFIGURESTAMPFILE = "${WORKDIR}/configure.sstate" autotools_preconfigure() { @@ -105,14 +106,13 @@ autotools_preconfigure() { if [ "${S}" != "${B}" ]; then echo "Previously configured separate build directory detected, cleaning ${B}" rm -rf ${B} - mkdir ${B} + mkdir -p ${B} else # At least remove the .la files since automake won't automatically # regenerate them even if CFLAGS/LDFLAGS are different cd ${S} if [ "${CLEANBROKEN}" != "1" -a \( -e Makefile -o -e makefile -o -e GNUmakefile \) ]; then - echo "Running \"${MAKE} clean\" in ${S}" - ${MAKE} clean + oe_runmake clean fi find ${S} -ignore_readdir_race -name \*.la -delete fi @@ -122,99 +122,29 @@ autotools_preconfigure() { autotools_postconfigure(){ if [ -n "${CONFIGURESTAMPFILE}" ]; then + mkdir -p `dirname ${CONFIGURESTAMPFILE}` echo ${BB_TASKHASH} > ${CONFIGURESTAMPFILE} fi } EXTRACONFFUNCS ??= "" -do_configure[prefuncs] += "autotools_preconfigure autotools_copy_aclocals ${EXTRACONFFUNCS}" +EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" + +do_configure[prefuncs] += "autotools_preconfigure autotools_aclocals ${EXTRACONFFUNCS}" do_configure[postfuncs] += "autotools_postconfigure" -ACLOCALDIR = "${B}/aclocal-copy" - -python autotools_copy_aclocals () { - s = d.getVar("S", True) - if not os.path.exists(s + "/configure.in") and not os.path.exists(s + "/configure.ac"): - if not d.getVar("AUTOTOOLS_COPYACLOCAL", False): - return - - taskdepdata = d.getVar("BB_TASKDEPDATA", False) - #bb.warn(str(taskdepdata)) - pn = d.getVar("PN", True) - aclocaldir = d.getVar("ACLOCALDIR", True) - oe.path.remove(aclocaldir) - bb.utils.mkdirhier(aclocaldir) - start = None - configuredeps = [] - - for dep in taskdepdata: - data = taskdepdata[dep] - if data[1] == "do_configure" and data[0] == pn: - start = dep - break - if start is None: - bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") - - # We need to find configure tasks which are either from <target> -> <target> - # or <native> -> <native> but not <target> -> <native> unless they're direct - # dependencies. This mirrors what would get restored from sstate. - done = [dep] - next = [dep] - while next: - new = [] - for dep in next: - data = taskdepdata[dep] - for datadep in data[3]: - if datadep in done: - continue - done.append(datadep) - if (not data[0].endswith("-native")) and taskdepdata[datadep][0].endswith("-native") and dep != start: - continue - new.append(datadep) - if taskdepdata[datadep][1] == "do_configure": - configuredeps.append(taskdepdata[datadep][0]) - next = new - - #configuredeps2 = [] - #for dep in taskdepdata: - # data = taskdepdata[dep] - # if data[1] == "do_configure" and data[0] != pn: - # configuredeps2.append(data[0]) - #configuredeps.sort() - #configuredeps2.sort() - #bb.warn(str(configuredeps)) - #bb.warn(str(configuredeps2)) - - cp = [] - siteconf = [] - for c in configuredeps: - if c.endswith("-native"): - manifest = d.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}-%s.populate_sysroot" % c) - elif c.startswith("nativesdk-"): - manifest = d.expand("${SSTATE_MANIFESTS}/manifest-${SDK_ARCH}_${SDK_OS}-%s.populate_sysroot" % c) - elif "-cross-" in c or "-crosssdk" in c: - continue - else: - manifest = d.expand("${SSTATE_MANIFESTS}/manifest-${MACHINE}-%s.populate_sysroot" % c) - try: - f = open(manifest, "r") - for l in f: - if "/aclocal/" in l and l.strip().endswith(".m4"): - cp.append(l.strip()) - elif "config_site.d/" in l: - cp.append(l.strip()) - except: - bb.warn("%s not found" % manifest) - - for c in cp: - t = os.path.join(aclocaldir, os.path.basename(c)) - if not os.path.exists(t): - os.symlink(c, t) - - d.setVar("CONFIG_SITE", siteinfo_get_files(d, False)) +ACLOCALDIR = "${STAGING_DATADIR}/aclocal" +ACLOCALEXTRAPATH = "" +ACLOCALEXTRAPATH_class-target = " -I ${STAGING_DATADIR_NATIVE}/aclocal/" +ACLOCALEXTRAPATH_class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/" + +python autotools_aclocals () { + # Refresh variable with cache files + d.setVar("CONFIG_SITE", siteinfo_get_files(d, aclocalcache=True)) } -autotools_copy_aclocals[vardepsexclude] += "MACHINE SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA" + +CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am" autotools_do_configure() { # WARNING: gross hack follows: @@ -225,22 +155,27 @@ autotools_do_configure() { # for a package whose autotools are old, on an x86_64 machine, which the old # config.sub does not support. Work around this by installing them manually # regardless. - ( for ac in `find ${S} -ignore_readdir_race -name configure.in -o -name configure.ac`; do + + PRUNE_M4="" + + for ac in `find ${S} -ignore_readdir_race -name configure.in -o -name configure.ac`; do rm -f `dirname $ac`/configure - done ) - if [ -e ${S}/configure.in -o -e ${S}/configure.ac ]; then + done + if [ -e ${AUTOTOOLS_SCRIPT_PATH}/configure.in -o -e ${AUTOTOOLS_SCRIPT_PATH}/configure.ac ]; then olddir=`pwd` - cd ${S} + cd ${AUTOTOOLS_SCRIPT_PATH} + mkdir -p ${ACLOCALDIR} ACLOCAL="aclocal --system-acdir=${ACLOCALDIR}/" if [ x"${acpaths}" = xdefault ]; then acpaths= - for i in `find ${S} -ignore_readdir_race -maxdepth 2 -name \*.m4|grep -v 'aclocal.m4'| \ - grep -v 'acinclude.m4' | grep -v 'aclocal-copy' | sed -e 's,\(.*/\).*$,\1,'|sort -u`; do + for i in `find ${AUTOTOOLS_SCRIPT_PATH} -ignore_readdir_race -maxdepth 2 -name \*.m4|grep -v 'aclocal.m4'| \ + grep -v 'acinclude.m4' | sed -e 's,\(.*/\).*$,\1,'|sort -u`; do acpaths="$acpaths -I $i" done else acpaths="${acpaths}" fi + acpaths="$acpaths ${ACLOCALEXTRAPATH}" AUTOV=`automake --version | sed -e '1{s/.* //;s/\.[0-9]\+$//};q'` automake --version echo "AUTOV is $AUTOV" @@ -258,45 +193,53 @@ autotools_do_configure() { else CONFIGURE_AC=configure.ac fi - if grep "^[[:space:]]*AM_GLIB_GNU_GETTEXT" $CONFIGURE_AC >/dev/null; then - if grep "sed.*POTFILES" $CONFIGURE_AC >/dev/null; then + if grep -q "^[[:space:]]*AM_GLIB_GNU_GETTEXT" $CONFIGURE_AC; then + if grep -q "sed.*POTFILES" $CONFIGURE_AC; then : do nothing -- we still have an old unmodified configure.ac else bbnote Executing glib-gettextize --force --copy echo "no" | glib-gettextize --force --copy fi - else if grep "^[[:space:]]*AM_GNU_GETTEXT" $CONFIGURE_AC >/dev/null; then + elif grep -q "^[[:space:]]*AM_GNU_GETTEXT" $CONFIGURE_AC; then # We'd call gettextize here if it wasn't so broken... - cp ${STAGING_DATADIR_NATIVE}/gettext/config.rpath ${AUTOTOOLS_AUXDIR}/ - if [ -d ${S}/po/ ]; then - cp -f ${STAGING_DATADIR_NATIVE}/gettext/po/Makefile.in.in ${S}/po/ - if [ ! -e ${S}/po/remove-potcdate.sin ]; then - cp ${STAGING_DATADIR_NATIVE}/gettext/po/remove-potcdate.sin ${S}/po/ - fi + cp ${STAGING_DATADIR_NATIVE}/gettext/config.rpath ${AUTOTOOLS_AUXDIR}/ + if [ -d ${S}/po/ ]; then + cp -f ${STAGING_DATADIR_NATIVE}/gettext/po/Makefile.in.in ${S}/po/ + if [ ! -e ${S}/po/remove-potcdate.sin ]; then + cp ${STAGING_DATADIR_NATIVE}/gettext/po/remove-potcdate.sin ${S}/po/ fi - for i in gettext.m4 iconv.m4 lib-ld.m4 lib-link.m4 lib-prefix.m4 nls.m4 po.m4 progtest.m4; do - for j in `find ${S} -ignore_readdir_race -name $i | grep -v aclocal-copy`; do - rm $j - done - done fi + PRUNE_M4="$PRUNE_M4 gettext.m4 iconv.m4 lib-ld.m4 lib-link.m4 lib-prefix.m4 nls.m4 po.m4 progtest.m4" fi mkdir -p m4 - if grep "^[[:space:]]*[AI][CT]_PROG_INTLTOOL" $CONFIGURE_AC >/dev/null; then + if grep -q "^[[:space:]]*[AI][CT]_PROG_INTLTOOL" $CONFIGURE_AC; then + if ! echo "${DEPENDS}" | grep -q intltool-native; then + bbwarn "Missing DEPENDS on intltool-native" + fi + PRUNE_M4="$PRUNE_M4 intltool.m4" bbnote Executing intltoolize --copy --force --automake intltoolize --copy --force --automake fi + + for i in $PRUNE_M4; do + find ${S} -ignore_readdir_race -name $i -delete + done + bbnote Executing ACLOCAL=\"$ACLOCAL\" autoreconf --verbose --install --force ${EXTRA_AUTORECONF} $acpaths ACLOCAL="$ACLOCAL" autoreconf -Wcross --verbose --install --force ${EXTRA_AUTORECONF} $acpaths || die "autoreconf execution failed." cd $olddir fi - if [ -e ${S}/configure ]; then + if [ -e ${CONFIGURE_SCRIPT} ]; then oe_runconf else bbnote "nothing to configure" fi } +autotools_do_compile() { + oe_runmake +} + autotools_do_install() { oe_runmake 'DESTDIR=${D}' install # Info dir listing isn't interesting at this point so remove it if it exists. @@ -307,6 +250,6 @@ autotools_do_install() { inherit siteconfig -EXPORT_FUNCTIONS do_configure do_install +EXPORT_FUNCTIONS do_configure do_compile do_install B = "${WORKDIR}/build" diff --git a/meta/classes/autotools_stage.bbclass b/meta/classes/autotools_stage.bbclass deleted file mode 100644 index b3c41e4b4d..0000000000 --- a/meta/classes/autotools_stage.bbclass +++ /dev/null @@ -1,2 +0,0 @@ -inherit autotools - diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass index f0780011bd..d95afb7b9b 100644 --- a/meta/classes/base.bbclass +++ b/meta/classes/base.bbclass @@ -10,13 +10,13 @@ inherit utility-tasks inherit metadata_scm inherit logging -OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath" +OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license" OE_IMPORTS[type] = "list" def oe_import(d): import sys - bbpath = d.getVar("BBPATH", True).split(":") + bbpath = d.getVar("BBPATH").split(":") sys.path[0:0] = [os.path.join(dir, "lib") for dir in bbpath] def inject(name, value): @@ -37,7 +37,7 @@ def oe_import(d): OE_IMPORTED := "${@oe_import(d)}" def lsb_distro_identifier(d): - adjust = d.getVar('LSB_DISTRO_ADJUST', True) + adjust = d.getVar('LSB_DISTRO_ADJUST') adjust_func = None if adjust: try: @@ -72,7 +72,7 @@ def base_dep_prepend(d): # we need that built is the responsibility of the patch function / class, not # the application. if not d.getVar('INHIBIT_DEFAULT_DEPS', False): - if (d.getVar('HOST_SYS', True) != d.getVar('BUILD_SYS', True)): + if (d.getVar('HOST_SYS') != d.getVar('BUILD_SYS')): deps += " virtual/${TARGET_PREFIX}gcc virtual/${TARGET_PREFIX}compilerlibs virtual/libc " return deps @@ -83,11 +83,11 @@ DEPENDS_prepend="${BASEDEPENDS} " FILESPATH = "${@base_set_filespath(["${FILE_DIRNAME}/${BP}", "${FILE_DIRNAME}/${BPN}", "${FILE_DIRNAME}/files"], d)}" # THISDIR only works properly with imediate expansion as it has to run # in the context of the location its used (:=) -THISDIR = "${@os.path.dirname(d.getVar('FILE', True))}" +THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" def extra_path_elements(d): path = "" - elements = (d.getVar('EXTRANATIVEPATH', True) or "").split() + elements = (d.getVar('EXTRANATIVEPATH') or "").split() for e in elements: path = path + "${STAGING_BINDIR_NATIVE}/" + e + ":" return path @@ -96,8 +96,11 @@ PATH_prepend = "${@extra_path_elements(d)}" def get_lic_checksum_file_list(d): filelist = [] - lic_files = d.getVar("LIC_FILES_CHKSUM", True) or '' - tmpdir = d.getVar("TMPDIR", True) + lic_files = d.getVar("LIC_FILES_CHKSUM") or '' + tmpdir = d.getVar("TMPDIR") + s = d.getVar("S") + b = d.getVar("B") + workdir = d.getVar("WORKDIR") urls = lic_files.split() for url in urls: @@ -105,14 +108,36 @@ def get_lic_checksum_file_list(d): # any others should be covered by SRC_URI. try: path = bb.fetch.decodeurl(url)[2] + if not path: + raise bb.fetch.MalformedUrl(url) + if path[0] == '/': - if path.startswith(tmpdir): + if path.startswith((tmpdir, s, b, workdir)): continue filelist.append(path + ":" + str(os.path.exists(path))) except bb.fetch.MalformedUrl: - raise bb.build.FuncFailed(d.getVar('PN', True) + ": LIC_FILES_CHKSUM contains an invalid URL: " + url) + bb.fatal(d.getVar('PN') + ": LIC_FILES_CHKSUM contains an invalid URL: " + url) return " ".join(filelist) +def setup_hosttools_dir(dest, toolsvar, d, fatal=True): + tools = d.getVar(toolsvar).split() + origbbenv = d.getVar("BB_ORIGENV", False) + path = origbbenv.getVar("PATH") + bb.utils.mkdirhier(dest) + notfound = [] + for tool in tools: + desttool = os.path.join(dest, tool) + if not os.path.exists(desttool): + srctool = bb.utils.which(path, tool, executable=True) + if "ccache" in srctool: + srctool = bb.utils.which(path, tool, executable=True, direction=1) + if srctool: + os.symlink(srctool, desttool) + else: + notfound.append(tool) + if notfound and fatal: + bb.fatal("The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed:\n %s" % " ".join(notfound)) + addtask fetch do_fetch[dirs] = "${DL_DIR}" do_fetch[file-checksums] = "${@bb.fetch.get_checksum_file_list(d)}" @@ -120,7 +145,7 @@ do_fetch[file-checksums] += " ${@get_lic_checksum_file_list(d)}" do_fetch[vardeps] += "SRCREV" python base_do_fetch() { - src_uri = (d.getVar('SRC_URI', True) or "").split() + src_uri = (d.getVar('SRC_URI') or "").split() if len(src_uri) == 0: return @@ -128,40 +153,38 @@ python base_do_fetch() { fetcher = bb.fetch2.Fetch(src_uri, d) fetcher.download() except bb.fetch2.BBFetchException as e: - raise bb.build.FuncFailed(e) + bb.fatal(str(e)) } addtask unpack after do_fetch do_unpack[dirs] = "${WORKDIR}" + +python () { + if d.getVar('S') != d.getVar('WORKDIR'): + d.setVarFlag('do_unpack', 'cleandirs', '${S}') + else: + d.setVarFlag('do_unpack', 'cleandirs', os.path.join('${S}', 'patches')) +} python base_do_unpack() { - src_uri = (d.getVar('SRC_URI', True) or "").split() + src_uri = (d.getVar('SRC_URI') or "").split() if len(src_uri) == 0: return - rootdir = d.getVar('WORKDIR', True) - - # Ensure that we cleanup ${S}/patches - # TODO: Investigate if we can remove - # the entire ${S} in this case. - s_dir = d.getVar('S', True) - p_dir = os.path.join(s_dir, 'patches') - bb.utils.remove(p_dir, True) - try: fetcher = bb.fetch2.Fetch(src_uri, d) - fetcher.unpack(rootdir) + fetcher.unpack(d.getVar('WORKDIR')) except bb.fetch2.BBFetchException as e: - raise bb.build.FuncFailed(e) + bb.fatal(str(e)) } def pkgarch_mapping(d): # Compatibility mappings of TUNE_PKGARCH (opt in) - if d.getVar("PKGARCHCOMPAT_ARMV7A", True): - if d.getVar("TUNE_PKGARCH", True) == "armv7a-vfp-neon": + if d.getVar("PKGARCHCOMPAT_ARMV7A"): + if d.getVar("TUNE_PKGARCH") == "armv7a-vfp-neon": d.setVar("TUNE_PKGARCH", "armv7a") def get_layers_branch_rev(d): - layers = (d.getVar("BBLAYERS", True) or "").split() + layers = (d.getVar("BBLAYERS") or "").split() layers_branch_rev = ["%-17s = \"%s:%s\"" % (os.path.basename(i), \ base_get_metadata_git_branch(i, None).strip(), \ base_get_metadata_git_revision(i, None)) \ @@ -188,7 +211,7 @@ BUILDCFG_FUNCS[type] = "list" def buildcfg_vars(d): statusvars = oe.data.typed_value('BUILDCFG_VARS', d) for var in statusvars: - value = d.getVar(var, True) + value = d.getVar(var) if value is not None: yield '%-17s = "%s"' % (var, value) @@ -196,7 +219,7 @@ def buildcfg_neededvars(d): needed_vars = oe.data.typed_value("BUILDCFG_NEEDEDVARS", d) pesteruser = [] for v in needed_vars: - val = d.getVar(v, True) + val = d.getVar(v) if not val or val == 'INVALID': pesteruser.append(v) @@ -204,7 +227,7 @@ def buildcfg_neededvars(d): bb.fatal('The following variable(s) were not set: %s\nPlease set them directly, or choose a MACHINE or DISTRO that sets them.' % ', '.join(pesteruser)) addhandler base_eventhandler -base_eventhandler[eventmask] = "bb.event.ConfigParsed bb.event.BuildStarted bb.event.RecipePreFinalise bb.runqueue.sceneQueueComplete" +base_eventhandler[eventmask] = "bb.event.ConfigParsed bb.event.BuildStarted bb.event.RecipePreFinalise bb.runqueue.sceneQueueComplete bb.event.RecipeParsed" python base_eventhandler() { import bb.runqueue @@ -215,10 +238,12 @@ python base_eventhandler() { pkgarch_mapping(e.data) oe.utils.features_backfill("DISTRO_FEATURES", e.data) oe.utils.features_backfill("MACHINE_FEATURES", e.data) + # Works with the line in layer.conf which changes PATH to point here + setup_hosttools_dir(d.getVar('HOSTTOOLS_DIR'), 'HOSTTOOLS', d) + setup_hosttools_dir(d.getVar('HOSTTOOLS_DIR'), 'HOSTTOOLS_NONFATAL', d, fatal=False) if isinstance(e, bb.event.BuildStarted): localdata = bb.data.createCopy(e.data) - bb.data.update_data(localdata) statuslines = [] for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): g = globals() @@ -229,14 +254,15 @@ python base_eventhandler() { if flines: statuslines.extend(flines) - statusheader = e.data.getVar('BUILDCFG_HEADER', True) - bb.plain('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) + statusheader = e.data.getVar('BUILDCFG_HEADER') + if statusheader: + bb.plain('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) # This code is to silence warnings where the SDK variables overwrite the # target ones and we'd see dulpicate key names overwriting each other # for various PREFERRED_PROVIDERS if isinstance(e, bb.event.RecipePreFinalise): - if e.data.getVar("TARGET_PREFIX", True) == e.data.getVar("SDK_PREFIX", True): + if e.data.getVar("TARGET_PREFIX") == e.data.getVar("SDK_PREFIX"): e.data.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}binutils") e.data.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}gcc-initial") e.data.delVar("PREFERRED_PROVIDER_virtual/${TARGET_PREFIX}gcc") @@ -254,6 +280,24 @@ python base_eventhandler() { bb.debug(1, "Executing SceneQueue Completion commands: %s" % "\n".join(cmds)) bb.build.exec_func("completion_function", e.data) os.remove(completions) + + if isinstance(e, bb.event.RecipeParsed): + # + # If we have multiple providers of virtual/X and a PREFERRED_PROVIDER_virtual/X is set + # skip parsing for all the other providers which will mean they get uninstalled from the + # sysroot since they're now "unreachable". This makes switching virtual/kernel work in + # particular. + # + pn = d.getVar('PN') + source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', False) + if not source_mirror_fetch: + provs = (d.getVar("PROVIDES") or "").split() + multiwhitelist = (d.getVar("MULTI_PROVIDER_WHITELIST") or "").split() + for p in provs: + if p.startswith("virtual/") and p not in multiwhitelist: + profprov = d.getVar("PREFERRED_PROVIDER_" + p) + if profprov and pn != profprov: + raise bb.parse.SkipPackage("PREFERRED_PROVIDER_%s set to %s, not %s" % (p, profprov, pn)) } CONFIGURESTAMPFILE = "${WORKDIR}/configure.sstate" @@ -261,7 +305,7 @@ CLEANBROKEN = "0" addtask configure after do_patch do_configure[dirs] = "${B}" -do_configure[deptask] = "do_populate_sysroot" +do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot" base_do_configure() { if [ -n "${CONFIGURESTAMPFILE}" -a -e "${CONFIGURESTAMPFILE}" ]; then if [ "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" ]; then @@ -273,6 +317,7 @@ base_do_configure() { fi fi if [ -n "${CONFIGURESTAMPFILE}" ]; then + mkdir -p `dirname ${CONFIGURESTAMPFILE}` echo ${BB_TASKHASH} > ${CONFIGURESTAMPFILE} fi } @@ -288,7 +333,7 @@ base_do_compile() { } addtask install after do_compile -do_install[dirs] = "${D} ${B}" +do_install[dirs] = "${B}" # Remove and re-create ${D} so that is it guaranteed to be empty do_install[cleandirs] = "${D}" @@ -312,9 +357,9 @@ def set_packagetriplet(d): tos = [] tvs = [] - archs.append(d.getVar("PACKAGE_ARCHS", True).split()) - tos.append(d.getVar("TARGET_OS", True)) - tvs.append(d.getVar("TARGET_VENDOR", True)) + archs.append(d.getVar("PACKAGE_ARCHS").split()) + tos.append(d.getVar("TARGET_OS")) + tvs.append(d.getVar("TARGET_VENDOR")) def settriplet(d, varname, archs, tos, tvs): triplets = [] @@ -326,16 +371,15 @@ def set_packagetriplet(d): settriplet(d, "PKGTRIPLETS", archs, tos, tvs) - variants = d.getVar("MULTILIB_VARIANTS", True) or "" + variants = d.getVar("MULTILIB_VARIANTS") or "" for item in variants.split(): localdata = bb.data.createCopy(d) overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + item localdata.setVar("OVERRIDES", overrides) - bb.data.update_data(localdata) - archs.append(localdata.getVar("PACKAGE_ARCHS", True).split()) - tos.append(localdata.getVar("TARGET_OS", True)) - tvs.append(localdata.getVar("TARGET_VENDOR", True)) + archs.append(localdata.getVar("PACKAGE_ARCHS").split()) + tos.append(localdata.getVar("TARGET_OS")) + tvs.append(localdata.getVar("TARGET_VENDOR")) settriplet(d, "PKGMLTRIPLETS", archs, tos, tvs) @@ -350,10 +394,10 @@ python () { # PACKAGECONFIG[foo] = "--enable-foo,--disable-foo,foo_depends,foo_runtime_depends" pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {} if pkgconfigflags: - pkgconfig = (d.getVar('PACKAGECONFIG', True) or "").split() - pn = d.getVar("PN", True) + pkgconfig = (d.getVar('PACKAGECONFIG') or "").split() + pn = d.getVar("PN") - mlprefix = d.getVar("MLPREFIX", True) + mlprefix = d.getVar("MLPREFIX") def expandFilter(appends, extension, prefix): appends = bb.utils.explode_deps(d.expand(" ".join(appends))) @@ -363,7 +407,10 @@ python () { newappends.append(a) elif a.startswith("virtual/"): subs = a.split("/", 1)[1] - newappends.append("virtual/" + prefix + subs + extension) + if subs.startswith(prefix): + newappends.append(a + extension) + else: + newappends.append("virtual/" + prefix + subs + extension) else: if a.startswith(prefix): newappends.append(a + extension) @@ -375,11 +422,11 @@ python () { if not appends: return if varname.find("DEPENDS") != -1: - if pn.startswith("nativesdk-"): + if bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('cross-canadian', d) : appends = expandFilter(appends, "", "nativesdk-") - if pn.endswith("-native"): + elif bb.data.inherits_class('native', d): appends = expandFilter(appends, "-native", "") - if mlprefix: + elif mlprefix: appends = expandFilter(appends, "", mlprefix) varname = d.expand(varname) d.appendVar(varname, " " + " ".join(appends)) @@ -391,7 +438,8 @@ python () { items = flagval.split(",") num = len(items) if num > 4: - bb.error("Only enable,disable,depend,rdepend can be specified!") + bb.error("%s: PACKAGECONFIG[%s] Only enable,disable,depend,rdepend can be specified!" + % (d.getVar('PN'), flag)) if flag in pkgconfig: if num >= 3 and items[2]: @@ -404,13 +452,10 @@ python () { extraconf.append(items[1]) appendVar('DEPENDS', extradeps) appendVar('RDEPENDS_${PN}', extrardeps) - if bb.data.inherits_class('cmake', d): - appendVar('EXTRA_OECMAKE', extraconf) - else: - appendVar('EXTRA_OECONF', extraconf) + appendVar('PACKAGECONFIG_CONFARGS', extraconf) - pn = d.getVar('PN', True) - license = d.getVar('LICENSE', True) + pn = d.getVar('PN') + license = d.getVar('LICENSE') if license == "INVALID": bb.fatal('This recipe does not have the LICENSE field set (%s)' % pn) @@ -430,40 +475,40 @@ python () { d.setVarFlag('do_configure', 'umask', '022') d.setVarFlag('do_compile', 'umask', '022') d.appendVarFlag('do_install', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') - d.setVarFlag('do_install', 'fakeroot', 1) + d.setVarFlag('do_install', 'fakeroot', '1') d.setVarFlag('do_install', 'umask', '022') d.appendVarFlag('do_package', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') - d.setVarFlag('do_package', 'fakeroot', 1) + d.setVarFlag('do_package', 'fakeroot', '1') d.setVarFlag('do_package', 'umask', '022') - d.setVarFlag('do_package_setscene', 'fakeroot', 1) + d.setVarFlag('do_package_setscene', 'fakeroot', '1') d.appendVarFlag('do_package_setscene', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') - d.setVarFlag('do_devshell', 'fakeroot', 1) + d.setVarFlag('do_devshell', 'fakeroot', '1') d.appendVarFlag('do_devshell', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') - source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', 0) + + need_machine = d.getVar('COMPATIBLE_MACHINE') + if need_machine: + import re + compat_machines = (d.getVar('MACHINEOVERRIDES') or "").split(":") + for m in compat_machines: + if re.match(need_machine, m): + break + else: + raise bb.parse.SkipPackage("incompatible with machine %s (not in COMPATIBLE_MACHINE)" % d.getVar('MACHINE')) + + source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', False) if not source_mirror_fetch: - need_host = d.getVar('COMPATIBLE_HOST', True) + need_host = d.getVar('COMPATIBLE_HOST') if need_host: import re - this_host = d.getVar('HOST_SYS', True) + this_host = d.getVar('HOST_SYS') if not re.match(need_host, this_host): raise bb.parse.SkipPackage("incompatible with host %s (not in COMPATIBLE_HOST)" % this_host) - need_machine = d.getVar('COMPATIBLE_MACHINE', True) - if need_machine: - import re - compat_machines = (d.getVar('MACHINEOVERRIDES', True) or "").split(":") - for m in compat_machines: - if re.match(need_machine, m): - break - else: - raise bb.parse.SkipPackage("incompatible with machine %s (not in COMPATIBLE_MACHINE)" % d.getVar('MACHINE', True)) - - - bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE', True) or "").split() + bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() check_license = False if pn.startswith("nativesdk-") else True for t in ["-native", "-cross-${TARGET_ARCH}", "-cross-initial-${TARGET_ARCH}", - "-crosssdk-${SDK_ARCH}", "-crosssdk-initial-${SDK_ARCH}", + "-crosssdk-${SDK_SYS}", "-crosssdk-initial-${SDK_SYS}", "-cross-canadian-${TRANSLATED_TARGET_ARCH}"]: if pn.endswith(d.expand(t)): check_license = False @@ -475,32 +520,24 @@ python () { whitelist = [] incompatwl = [] - htincompatwl = [] for lic in bad_licenses: spdx_license = return_spdx(d, lic) - for w in ["HOSTTOOLS_WHITELIST_", "LGPLv2_WHITELIST_", "WHITELIST_"]: - whitelist.extend((d.getVar(w + lic, True) or "").split()) + for w in ["LGPLv2_WHITELIST_", "WHITELIST_"]: + whitelist.extend((d.getVar(w + lic) or "").split()) if spdx_license: - whitelist.extend((d.getVar(w + spdx_license, True) or "").split()) + whitelist.extend((d.getVar(w + spdx_license) or "").split()) ''' - We need to track what we are whitelisting and why. If pn is - incompatible and is not HOSTTOOLS_WHITELIST_ we need to be - able to note that the image that is created may infact - contain incompatible licenses despite INCOMPATIBLE_LICENSE - being set. + We need to track what we are whitelisting and why. If pn is + incompatible we need to be able to note that the image that + is created may infact contain incompatible licenses despite + INCOMPATIBLE_LICENSE being set. ''' - if "HOSTTOOLS" in w: - htincompatwl.extend((d.getVar(w + lic, True) or "").split()) - if spdx_license: - htincompatwl.extend((d.getVar(w + spdx_license, True) or "").split()) - else: - incompatwl.extend((d.getVar(w + lic, True) or "").split()) - if spdx_license: - incompatwl.extend((d.getVar(w + spdx_license, True) or "").split()) + incompatwl.extend((d.getVar(w + lic) or "").split()) + if spdx_license: + incompatwl.extend((d.getVar(w + spdx_license) or "").split()) if not pn in whitelist: - recipe_license = d.getVar('LICENSE', True) - pkgs = d.getVar('PACKAGES', True).split() + pkgs = d.getVar('PACKAGES').split() skipped_pkgs = [] unskipped_pkgs = [] for pkg in pkgs: @@ -511,59 +548,94 @@ python () { all_skipped = skipped_pkgs and not unskipped_pkgs if unskipped_pkgs: for pkg in skipped_pkgs: - bb.debug(1, "SKIPPING the package " + pkg + " at do_rootfs because it's " + recipe_license) - d.setVar('LICENSE_EXCLUSION-' + pkg, 1) + bb.debug(1, "SKIPPING the package " + pkg + " at do_rootfs because it's " + license) + mlprefix = d.getVar('MLPREFIX') + d.setVar('LICENSE_EXCLUSION-' + mlprefix + pkg, 1) for pkg in unskipped_pkgs: bb.debug(1, "INCLUDING the package " + pkg) elif all_skipped or incompatible_license(d, bad_licenses): - bb.debug(1, "SKIPPING recipe %s because it's %s" % (pn, recipe_license)) - raise bb.parse.SkipPackage("incompatible with license %s" % recipe_license) + bb.debug(1, "SKIPPING recipe %s because it's %s" % (pn, license)) + raise bb.parse.SkipPackage("it has an incompatible license: %s" % license) elif pn in whitelist: if pn in incompatwl: bb.note("INCLUDING " + pn + " as buildable despite INCOMPATIBLE_LICENSE because it has been whitelisted") - elif pn in htincompatwl: - bb.note("INCLUDING " + pn + " as buildable despite INCOMPATIBLE_LICENSE because it has been whitelisted for HOSTTOOLS") - - srcuri = d.getVar('SRC_URI', True) - # Svn packages should DEPEND on subversion-native - if "svn://" in srcuri: - d.appendVarFlag('do_fetch', 'depends', ' subversion-native:do_populate_sysroot') - - # Git packages should DEPEND on git-native - if "git://" in srcuri: - d.appendVarFlag('do_fetch', 'depends', ' git-native:do_populate_sysroot') - - # Mercurial packages should DEPEND on mercurial-native - elif "hg://" in srcuri: - d.appendVarFlag('do_fetch', 'depends', ' mercurial-native:do_populate_sysroot') - # OSC packages should DEPEND on osc-native - elif "osc://" in srcuri: - d.appendVarFlag('do_fetch', 'depends', ' osc-native:do_populate_sysroot') - - # *.lz4 should depends on lz4-native for unpacking - # Not endswith because of "*.patch.lz4;patch=1". Need bb.fetch.decodeurl in future - if '.lz4' in srcuri: - d.appendVarFlag('do_unpack', 'depends', ' lz4-native:do_populate_sysroot') - - # *.xz should depends on xz-native for unpacking - # Not endswith because of "*.patch.xz;patch=1". Need bb.fetch.decodeurl in future - if '.xz' in srcuri: - d.appendVarFlag('do_unpack', 'depends', ' xz-native:do_populate_sysroot') - - # unzip-native should already be staged before unpacking ZIP recipes - if ".zip" in srcuri: - d.appendVarFlag('do_unpack', 'depends', ' unzip-native:do_populate_sysroot') - - # file is needed by rpm2cpio.sh - if ".src.rpm" in srcuri: - d.appendVarFlag('do_unpack', 'depends', ' file-native:do_populate_sysroot') + # Try to verify per-package (LICENSE_<pkg>) values. LICENSE should be a + # superset of all per-package licenses. We do not do advanced (pattern) + # matching of license expressions - just check that all license strings + # in LICENSE_<pkg> are found in LICENSE. + license_set = oe.license.list_licenses(license) + for pkg in d.getVar('PACKAGES').split(): + pkg_license = d.getVar('LICENSE_' + pkg) + if pkg_license: + unlisted = oe.license.list_licenses(pkg_license) - license_set + if unlisted: + bb.warn("LICENSE_%s includes licenses (%s) that are not " + "listed in LICENSE" % (pkg, ' '.join(unlisted))) + + needsrcrev = False + srcuri = d.getVar('SRC_URI') + for uri in srcuri.split(): + (scheme, _ , path) = bb.fetch.decodeurl(uri)[:3] + + # HTTP/FTP use the wget fetcher + if scheme in ("http", "https", "ftp"): + d.appendVarFlag('do_fetch', 'depends', ' wget-native:do_populate_sysroot') + + # Svn packages should DEPEND on subversion-native + if scheme == "svn": + needsrcrev = True + d.appendVarFlag('do_fetch', 'depends', ' subversion-native:do_populate_sysroot') + + # Git packages should DEPEND on git-native + elif scheme in ("git", "gitsm"): + needsrcrev = True + d.appendVarFlag('do_fetch', 'depends', ' git-native:do_populate_sysroot') + + # Mercurial packages should DEPEND on mercurial-native + elif scheme == "hg": + needsrcrev = True + d.appendVarFlag('do_fetch', 'depends', ' mercurial-native:do_populate_sysroot') + + # Perforce packages support SRCREV = "${AUTOREV}" + elif scheme == "p4": + needsrcrev = True + + # OSC packages should DEPEND on osc-native + elif scheme == "osc": + d.appendVarFlag('do_fetch', 'depends', ' osc-native:do_populate_sysroot') + + elif scheme == "npm": + d.appendVarFlag('do_fetch', 'depends', ' nodejs-native:do_populate_sysroot') + + # *.lz4 should DEPEND on lz4-native for unpacking + if path.endswith('.lz4'): + d.appendVarFlag('do_unpack', 'depends', ' lz4-native:do_populate_sysroot') + + # *.lz should DEPEND on lzip-native for unpacking + elif path.endswith('.lz'): + d.appendVarFlag('do_unpack', 'depends', ' lzip-native:do_populate_sysroot') + + # *.xz should DEPEND on xz-native for unpacking + elif path.endswith('.xz'): + d.appendVarFlag('do_unpack', 'depends', ' xz-native:do_populate_sysroot') + + # .zip should DEPEND on unzip-native for unpacking + elif path.endswith('.zip'): + d.appendVarFlag('do_unpack', 'depends', ' unzip-native:do_populate_sysroot') + + # file is needed by rpm2cpio.sh + elif path.endswith('.src.rpm'): + d.appendVarFlag('do_unpack', 'depends', ' file-native:do_populate_sysroot') + + if needsrcrev: + d.setVar("SRCPV", "${@bb.fetch2.get_srcrev(d)}") set_packagetriplet(d) # 'multimachine' handling - mach_arch = d.getVar('MACHINE_ARCH', True) - pkg_arch = d.getVar('PACKAGE_ARCH', True) + mach_arch = d.getVar('MACHINE_ARCH') + pkg_arch = d.getVar('PACKAGE_ARCH') if (pkg_arch == mach_arch): # Already machine specific - nothing further to do @@ -573,11 +645,11 @@ python () { # We always try to scan SRC_URI for urls with machine overrides # unless the package sets SRC_URI_OVERRIDES_PACKAGE_ARCH=0 # - override = d.getVar('SRC_URI_OVERRIDES_PACKAGE_ARCH', True) + override = d.getVar('SRC_URI_OVERRIDES_PACKAGE_ARCH') if override != '0': paths = [] - fpaths = (d.getVar('FILESPATH', True) or '').split(':') - machine = d.getVar('MACHINE', True) + fpaths = (d.getVar('FILESPATH') or '').split(':') + machine = d.getVar('MACHINE') for p in fpaths: if os.path.basename(p) == machine and os.path.isdir(p): paths.append(p) @@ -594,34 +666,35 @@ python () { d.setVar('PACKAGE_ARCH', "${MACHINE_ARCH}") return - packages = d.getVar('PACKAGES', True).split() + packages = d.getVar('PACKAGES').split() for pkg in packages: - pkgarch = d.getVar("PACKAGE_ARCH_%s" % pkg, True) + pkgarch = d.getVar("PACKAGE_ARCH_%s" % pkg) # We could look for != PACKAGE_ARCH here but how to choose # if multiple differences are present? # Look through PACKAGE_ARCHS for the priority order? if pkgarch and pkgarch == mach_arch: d.setVar('PACKAGE_ARCH', "${MACHINE_ARCH}") - bb.warn("Recipe %s is marked as only being architecture specific but seems to have machine specific packages?! The recipe may as well mark itself as machine specific directly." % d.getVar("PN", True)) + bb.warn("Recipe %s is marked as only being architecture specific but seems to have machine specific packages?! The recipe may as well mark itself as machine specific directly." % d.getVar("PN")) } addtask cleansstate after do_clean python do_cleansstate() { sstate_clean_cachefiles(d) } - addtask cleanall after do_cleansstate +do_cleansstate[nostamp] = "1" + python do_cleanall() { - src_uri = (d.getVar('SRC_URI', True) or "").split() + src_uri = (d.getVar('SRC_URI') or "").split() if len(src_uri) == 0: return try: fetcher = bb.fetch2.Fetch(src_uri, d) fetcher.clean() - except bb.fetch2.BBFetchException, e: - raise bb.build.FuncFailed(e) + except bb.fetch2.BBFetchException as e: + bb.fatal(str(e)) } do_cleanall[nostamp] = "1" diff --git a/meta/classes/bash-completion.bbclass b/meta/classes/bash-completion.bbclass new file mode 100644 index 0000000000..80ee9b4874 --- /dev/null +++ b/meta/classes/bash-completion.bbclass @@ -0,0 +1,7 @@ +DEPENDS_append_class-target = " bash-completion" + +PACKAGES += "${PN}-bash-completion" + +FILES_${PN}-bash-completion = "${datadir}/bash-completion ${sysconfdir}/bash_completion.d" + +RDEPENDS_${PN}-bash-completion = "bash-completion" diff --git a/meta/classes/binconfig-disabled.bbclass b/meta/classes/binconfig-disabled.bbclass index 602a669aa1..096b670e12 100644 --- a/meta/classes/binconfig-disabled.bbclass +++ b/meta/classes/binconfig-disabled.bbclass @@ -15,6 +15,7 @@ do_install_append () { echo "echo 'ERROR: $x should not be used, use an alternative such as pkg-config' >&2" >> ${D}$x echo "echo '--should-not-have-used-$x'" >> ${D}$x echo "exit 1" >> ${D}$x + chmod +x ${D}$x done } diff --git a/meta/classes/binconfig.bbclass b/meta/classes/binconfig.bbclass index cbc4173601..39c3e2b17b 100644 --- a/meta/classes/binconfig.bbclass +++ b/meta/classes/binconfig.bbclass @@ -13,16 +13,16 @@ def get_binconfig_mangle(d): s += " -e 's:=%s${exec_prefix}/:=\\1OEEXECPREFIX/:'" % optional_quote s += " -e 's:-L${libdir}:-LOELIBDIR:;'" s += " -e 's:-I${includedir}:-IOEINCDIR:;'" + s += " -e 's:-L${WORKDIR}:-LOELIBDIR:'" + s += " -e 's:-I${WORKDIR}:-IOEINCDIR:'" s += " -e 's:OEBASELIBDIR:${STAGING_BASELIBDIR}:;'" s += " -e 's:OELIBDIR:${STAGING_LIBDIR}:;'" s += " -e 's:OEINCDIR:${STAGING_INCDIR}:;'" s += " -e 's:OEDATADIR:${STAGING_DATADIR}:'" s += " -e 's:OEPREFIX:${STAGING_DIR_HOST}${prefix}:'" s += " -e 's:OEEXECPREFIX:${STAGING_DIR_HOST}${exec_prefix}:'" - s += " -e 's:-I${WORKDIR}:-I${STAGING_INCDIR}:'" - s += " -e 's:-L${WORKDIR}:-L${STAGING_LIBDIR}:'" - if bb.data.getVar("OE_BINCONFIG_EXTRA_MANGLE", d): - s += bb.data.getVar("OE_BINCONFIG_EXTRA_MANGLE", d) + if d.getVar("OE_BINCONFIG_EXTRA_MANGLE", False): + s += d.getVar("OE_BINCONFIG_EXTRA_MANGLE") return s diff --git a/meta/classes/blacklist.bbclass b/meta/classes/blacklist.bbclass index a0141a82c0..e58564c34e 100644 --- a/meta/classes/blacklist.bbclass +++ b/meta/classes/blacklist.bbclass @@ -12,33 +12,8 @@ # PNBLACKLIST[pn] = "message" # -# Cope with PNBLACKLIST flags for multilib case -addhandler blacklist_multilib_eventhandler -blacklist_multilib_eventhandler[eventmask] = "bb.event.ConfigParsed" -python blacklist_multilib_eventhandler() { - multilibs = e.data.getVar('MULTILIBS', True) - if not multilibs: - return - - # this block has been copied from base.bbclass so keep it in sync - prefixes = [] - for ext in multilibs.split(): - eext = ext.split(':') - if len(eext) > 1 and eext[0] == 'multilib': - prefixes.append(eext[1]) - - blacklists = e.data.getVarFlags('PNBLACKLIST') or {} - for pkg, reason in blacklists.items(): - if pkg.endswith(("-native", "-crosssdk")) or pkg.startswith(("nativesdk-", "virtual/nativesdk-")) or 'cross-canadian' in pkg: - continue - for p in prefixes: - newpkg = p + "-" + pkg - if not e.data.getVarFlag('PNBLACKLIST', newpkg, True): - e.data.setVarFlag('PNBLACKLIST', newpkg, reason) -} - python () { - blacklist = d.getVarFlag('PNBLACKLIST', d.getVar('PN', True), True) + blacklist = d.getVarFlag('PNBLACKLIST', d.getVar('PN')) if blacklist: raise bb.parse.SkipPackage("Recipe is blacklisted: %s" % (blacklist)) diff --git a/meta/classes/boot-directdisk.bbclass b/meta/classes/boot-directdisk.bbclass deleted file mode 100644 index 600e21abcf..0000000000 --- a/meta/classes/boot-directdisk.bbclass +++ /dev/null @@ -1,194 +0,0 @@ -# boot-directdisk.bbclass -# (loosly based off bootimg.bbclass Copyright (C) 2004, Advanced Micro Devices, Inc.) -# -# Create an image which can be placed directly onto a harddisk using dd and then -# booted. -# -# This uses syslinux. extlinux would have been nice but required the ext2/3 -# partition to be mounted. grub requires to run itself as part of the install -# process. -# -# The end result is a 512 boot sector populated with an MBR and partition table -# followed by an msdos fat16 partition containing syslinux and a linux kernel -# completed by the ext2/3 rootfs. -# -# We have to push the msdos parition table size > 16MB so fat 16 is used as parted -# won't touch fat12 partitions. - -# External variables needed - -# ${ROOTFS} - the rootfs image to incorporate - -do_bootdirectdisk[depends] += "dosfstools-native:do_populate_sysroot \ - virtual/kernel:do_deploy \ - syslinux:do_populate_sysroot \ - syslinux-native:do_populate_sysroot \ - parted-native:do_populate_sysroot \ - mtools-native:do_populate_sysroot " - -PACKAGES = " " -EXCLUDE_FROM_WORLD = "1" - -BOOTDD_VOLUME_ID ?= "boot" -BOOTDD_EXTRA_SPACE ?= "16384" - -EFI = "${@bb.utils.contains("MACHINE_FEATURES", "efi", "1", "0", d)}" -EFI_PROVIDER ?= "grub-efi" -EFI_CLASS = "${@bb.utils.contains("MACHINE_FEATURES", "efi", "${EFI_PROVIDER}", "", d)}" - -# Include legacy boot if MACHINE_FEATURES includes "pcbios" or if it does not -# contain "efi". This way legacy is supported by default if neither is -# specified, maintaining the original behavior. -def pcbios(d): - pcbios = bb.utils.contains("MACHINE_FEATURES", "pcbios", "1", "0", d) - if pcbios == "0": - pcbios = bb.utils.contains("MACHINE_FEATURES", "efi", "0", "1", d) - return pcbios - -def pcbios_class(d): - if d.getVar("PCBIOS", True) == "1": - return "syslinux" - return "" - -PCBIOS = "${@pcbios(d)}" -PCBIOS_CLASS = "${@pcbios_class(d)}" - -inherit ${PCBIOS_CLASS} -inherit ${EFI_CLASS} - -# Get the build_syslinux_cfg() function from the syslinux class - -AUTO_SYSLINUXCFG = "1" -DISK_SIGNATURE ?= "${DISK_SIGNATURE_GENERATED}" -SYSLINUX_ROOT ?= "root=/dev/sda2" -SYSLINUX_TIMEOUT ?= "10" - -IS_VM = '${@bb.utils.contains_any("IMAGE_FSTYPES", ["vmdk", "vdi", "qcow2"], "true", "false", d)}' - -boot_direct_populate() { - dest=$1 - install -d $dest - - # Install bzImage, initrd, and rootfs.img in DEST for all loaders to use. - if [ -e ${DEPLOY_DIR_IMAGE}/bzImage ]; then - install -m 0644 ${DEPLOY_DIR_IMAGE}/bzImage $dest/vmlinuz - fi - - # initrd is made of concatenation of multiple filesystem images - if [ -n "${INITRD}" ]; then - rm -f $dest/initrd - for fs in ${INITRD} - do - if [ -s "${fs}" ]; then - cat ${fs} >> $dest/initrd - else - bbfatal "${fs} is invalid. initrd image creation failed." - fi - done - chmod 0644 $dest/initrd - fi -} - -build_boot_dd() { - HDDDIR="${S}/hdd/boot" - HDDIMG="${S}/hdd.image" - IMAGE=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hdddirect - - boot_direct_populate $HDDDIR - - if [ "${PCBIOS}" = "1" ]; then - syslinux_hddimg_populate $HDDDIR - fi - if [ "${EFI}" = "1" ]; then - efi_hddimg_populate $HDDDIR - fi - - if [ "${IS_VM}" = "true" ]; then - if [ "x${AUTO_SYSLINUXMENU}" = "x1" ] ; then - install -m 0644 ${STAGING_DIR}/${MACHINE}/usr/share/syslinux/vesamenu.c32 $HDDDIR/${SYSLINUXDIR}/ - if [ "x${SYSLINUX_SPLASH}" != "x" ] ; then - install -m 0644 ${SYSLINUX_SPLASH} $HDDDIR/${SYSLINUXDIR}/splash.lss - fi - fi - fi - - BLOCKS=`du -bks $HDDDIR | cut -f 1` - BLOCKS=`expr $BLOCKS + ${BOOTDD_EXTRA_SPACE}` - - # Ensure total sectors is an integral number of sectors per - # track or mcopy will complain. Sectors are 512 bytes, and we - # generate images with 32 sectors per track. This calculation is - # done in blocks, thus the mod by 16 instead of 32. - BLOCKS=$(expr $BLOCKS + $(expr 16 - $(expr $BLOCKS % 16))) - - mkdosfs -n ${BOOTDD_VOLUME_ID} -S 512 -C $HDDIMG $BLOCKS - mcopy -i $HDDIMG -s $HDDDIR/* ::/ - - if [ "${PCBIOS}" = "1" ]; then - syslinux_hdddirect_install $HDDIMG - fi - chmod 644 $HDDIMG - - ROOTFSBLOCKS=`du -Lbks ${ROOTFS} | cut -f 1` - TOTALSIZE=`expr $BLOCKS + $ROOTFSBLOCKS` - END1=`expr $BLOCKS \* 1024` - END2=`expr $END1 + 512` - END3=`expr \( $ROOTFSBLOCKS \* 1024 \) + $END1` - - echo $ROOTFSBLOCKS $TOTALSIZE $END1 $END2 $END3 - rm -rf $IMAGE - dd if=/dev/zero of=$IMAGE bs=1024 seek=$TOTALSIZE count=1 - - parted $IMAGE mklabel msdos - parted $IMAGE mkpart primary fat16 0 ${END1}B - parted $IMAGE unit B mkpart primary ext2 ${END2}B ${END3}B - parted $IMAGE set 1 boot on - - parted $IMAGE print - - awk "BEGIN { printf \"$(echo ${DISK_SIGNATURE} | fold -w 2 | tac | paste -sd '' | sed 's/\(..\)/\\x&/g')\" }" | \ - dd of=$IMAGE bs=1 seek=440 conv=notrunc - - OFFSET=`expr $END2 / 512` - if [ "${PCBIOS}" = "1" ]; then - dd if=${STAGING_DATADIR}/syslinux/mbr.bin of=$IMAGE conv=notrunc - fi - - dd if=$HDDIMG of=$IMAGE conv=notrunc seek=1 bs=512 - dd if=${ROOTFS} of=$IMAGE conv=notrunc seek=$OFFSET bs=512 - - cd ${DEPLOY_DIR_IMAGE} - rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.hdddirect - ln -s ${IMAGE_NAME}.hdddirect ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.hdddirect -} - -python do_bootdirectdisk() { - validate_disk_signature(d) - if d.getVar("PCBIOS", True) == "1": - bb.build.exec_func('build_syslinux_cfg', d) - if d.getVar("EFI", True) == "1": - bb.build.exec_func('build_efi_cfg', d) - bb.build.exec_func('build_boot_dd', d) -} - -def generate_disk_signature(): - import uuid - - signature = str(uuid.uuid4())[:8] - - if signature != '00000000': - return signature - else: - return 'ffffffff' - -def validate_disk_signature(d): - import re - - disk_signature = d.getVar("DISK_SIGNATURE", True) - - if not re.match(r'^[0-9a-fA-F]{8}$', disk_signature): - bb.fatal("DISK_SIGNATURE '%s' must be an 8 digit hex string" % disk_signature) - -DISK_SIGNATURE_GENERATED := "${@generate_disk_signature()}" - -addtask bootdirectdisk before do_build diff --git a/meta/classes/bootimg.bbclass b/meta/classes/bootimg.bbclass deleted file mode 100644 index ec9d0b7d52..0000000000 --- a/meta/classes/bootimg.bbclass +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright (C) 2004, Advanced Micro Devices, Inc. All Rights Reserved -# Released under the MIT license (see packages/COPYING) - -# Creates a bootable image using syslinux, your kernel and an optional -# initrd - -# -# End result is two things: -# -# 1. A .hddimg file which is an msdos filesystem containing syslinux, a kernel, -# an initrd and a rootfs image. These can be written to harddisks directly and -# also booted on USB flash disks (write them there with dd). -# -# 2. A CD .iso image - -# Boot process is that the initrd will boot and process which label was selected -# in syslinux. Actions based on the label are then performed (e.g. installing to -# an hdd) - -# External variables (also used by syslinux.bbclass) -# ${INITRD} - indicates a list of filesystem images to concatenate and use as an initrd (optional) -# ${COMPRESSISO} - Transparent compress ISO, reduce size ~40% if set to 1 -# ${NOISO} - skip building the ISO image if set to 1 -# ${NOHDD} - skip building the HDD image if set to 1 -# ${HDDIMG_ID} - FAT image volume-id -# ${ROOTFS} - indicates a filesystem image to include as the root filesystem (optional) - -do_bootimg[depends] += "dosfstools-native:do_populate_sysroot \ - mtools-native:do_populate_sysroot \ - cdrtools-native:do_populate_sysroot \ - virtual/kernel:do_deploy \ - ${@oe.utils.ifelse(d.getVar('COMPRESSISO', False),'zisofs-tools-native:do_populate_sysroot','')}" - -PACKAGES = " " -EXCLUDE_FROM_WORLD = "1" - -HDDDIR = "${S}/hddimg" -ISODIR = "${S}/iso" -EFIIMGDIR = "${S}/efi_img" -COMPACT_ISODIR = "${S}/iso.z" -COMPRESSISO ?= "0" - -BOOTIMG_VOLUME_ID ?= "boot" -BOOTIMG_EXTRA_SPACE ?= "512" - -EFI = "${@bb.utils.contains("MACHINE_FEATURES", "efi", "1", "0", d)}" -EFI_PROVIDER ?= "grub-efi" -EFI_CLASS = "${@bb.utils.contains("MACHINE_FEATURES", "efi", "${EFI_PROVIDER}", "", d)}" - -KERNEL_IMAGETYPE ??= "bzImage" - -# Include legacy boot if MACHINE_FEATURES includes "pcbios" or if it does not -# contain "efi". This way legacy is supported by default if neither is -# specified, maintaining the original behavior. -def pcbios(d): - pcbios = bb.utils.contains("MACHINE_FEATURES", "pcbios", "1", "0", d) - if pcbios == "0": - pcbios = bb.utils.contains("MACHINE_FEATURES", "efi", "0", "1", d) - return pcbios - -PCBIOS = "${@pcbios(d)}" - -# The syslinux is required for the isohybrid command and boot catalog -inherit syslinux -inherit ${EFI_CLASS} - -populate() { - DEST=$1 - install -d ${DEST} - - # Install kernel, initrd, and rootfs.img in DEST for all loaders to use. - install -m 0644 ${DEPLOY_DIR_IMAGE}/${KERNEL_IMAGETYPE} ${DEST}/vmlinuz - - # initrd is made of concatenation of multiple filesystem images - if [ -n "${INITRD}" ]; then - rm -f ${DEST}/initrd - for fs in ${INITRD} - do - if [ -s "${fs}" ]; then - cat ${fs} >> ${DEST}/initrd - else - bbfatal "${fs} is invalid. initrd image creation failed." - fi - done - chmod 0644 ${DEST}/initrd - fi - - if [ -n "${ROOTFS}" ] && [ -s "${ROOTFS}" ]; then - install -m 0644 ${ROOTFS} ${DEST}/rootfs.img - fi - -} - -build_iso() { - # Only create an ISO if we have an INITRD and NOISO was not set - if [ -z "${INITRD}" ] || [ "${NOISO}" = "1" ]; then - bbnote "ISO image will not be created." - return - fi - # ${INITRD} is a list of multiple filesystem images - for fs in ${INITRD} - do - if [ ! -s "${fs}" ]; then - bbnote "ISO image will not be created. ${fs} is invalid." - return - fi - done - - - populate ${ISODIR} - - if [ "${PCBIOS}" = "1" ]; then - syslinux_iso_populate ${ISODIR} - fi - if [ "${EFI}" = "1" ]; then - efi_iso_populate ${ISODIR} - build_fat_img ${EFIIMGDIR} ${ISODIR}/efi.img - fi - - # EFI only - if [ "${PCBIOS}" != "1" ] && [ "${EFI}" = "1" ] ; then - # Work around bug in isohybrid where it requires isolinux.bin - # In the boot catalog, even though it is not used - mkdir -p ${ISODIR}/${ISOLINUXDIR} - install -m 0644 ${STAGING_DATADIR}/syslinux/isolinux.bin ${ISODIR}${ISOLINUXDIR} - fi - - if [ "${COMPRESSISO}" = "1" ] ; then - # create compact directory, compress iso - mkdir -p ${COMPACT_ISODIR} - mkzftree -z 9 -p 4 -F ${ISODIR}/rootfs.img ${COMPACT_ISODIR}/rootfs.img - - # move compact iso to iso, then remove compact directory - mv ${COMPACT_ISODIR}/rootfs.img ${ISODIR}/rootfs.img - rm -Rf ${COMPACT_ISODIR} - mkisofs_compress_opts="-R -z -D -l" - else - mkisofs_compress_opts="-r" - fi - - # Check the size of ${ISODIR}/rootfs.img, use mkisofs -iso-level 3 - # when it exceeds 3.8GB, the specification is 4G - 1 bytes, we need - # leave a few space for other files. - mkisofs_iso_level="" - - if [ -n "${ROOTFS}" ] && [ -s "${ROOTFS}" ]; then - rootfs_img_size=`stat -c '%s' ${ISODIR}/rootfs.img` - # 4080218931 = 3.8 * 1024 * 1024 * 1024 - if [ $rootfs_img_size -gt 4080218931 ]; then - bbnote "${ISODIR}/rootfs.img execeeds 3.8GB, using '-iso-level 3' for mkisofs" - mkisofs_iso_level="-iso-level 3" - fi - fi - - if [ "${PCBIOS}" = "1" ] && [ "${EFI}" != "1" ] ; then - # PCBIOS only media - mkisofs -V ${BOOTIMG_VOLUME_ID} \ - -o ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.iso \ - -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ - $mkisofs_compress_opts \ - ${MKISOFS_OPTIONS} $mkisofs_iso_level ${ISODIR} - else - # EFI only OR EFI+PCBIOS - mkisofs -A ${BOOTIMG_VOLUME_ID} -V ${BOOTIMG_VOLUME_ID} \ - -o ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.iso \ - -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ - $mkisofs_compress_opts ${MKISOFS_OPTIONS} $mkisofs_iso_level \ - -eltorito-alt-boot -eltorito-platform efi \ - -b efi.img -no-emul-boot \ - ${ISODIR} - isohybrid_args="-u" - fi - - isohybrid $isohybrid_args ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.iso - - cd ${DEPLOY_DIR_IMAGE} - rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.iso - ln -s ${IMAGE_NAME}.iso ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.iso -} - -build_fat_img() { - FATSOURCEDIR=$1 - FATIMG=$2 - - # Calculate the size required for the final image including the - # data and filesystem overhead. - # Sectors: 512 bytes - # Blocks: 1024 bytes - - # Determine the sector count just for the data - SECTORS=$(expr $(du --apparent-size -ks ${FATSOURCEDIR} | cut -f 1) \* 2) - - # Account for the filesystem overhead. This includes directory - # entries in the clusters as well as the FAT itself. - # Assumptions: - # FAT32 (12 or 16 may be selected by mkdosfs, but the extra - # padding will be minimal on those smaller images and not - # worth the logic here to caclulate the smaller FAT sizes) - # < 16 entries per directory - # 8.3 filenames only - - # 32 bytes per dir entry - DIR_BYTES=$(expr $(find ${FATSOURCEDIR} | tail -n +2 | wc -l) \* 32) - # 32 bytes for every end-of-directory dir entry - DIR_BYTES=$(expr $DIR_BYTES + $(expr $(find ${FATSOURCEDIR} -type d | tail -n +2 | wc -l) \* 32)) - # 4 bytes per FAT entry per sector of data - FAT_BYTES=$(expr $SECTORS \* 4) - # 4 bytes per FAT entry per end-of-cluster list - FAT_BYTES=$(expr $FAT_BYTES + $(expr $(find ${FATSOURCEDIR} -type d | tail -n +2 | wc -l) \* 4)) - - # Use a ceiling function to determine FS overhead in sectors - DIR_SECTORS=$(expr $(expr $DIR_BYTES + 511) / 512) - # There are two FATs on the image - FAT_SECTORS=$(expr $(expr $(expr $FAT_BYTES + 511) / 512) \* 2) - SECTORS=$(expr $SECTORS + $(expr $DIR_SECTORS + $FAT_SECTORS)) - - # Determine the final size in blocks accounting for some padding - BLOCKS=$(expr $(expr $SECTORS / 2) + ${BOOTIMG_EXTRA_SPACE}) - - # Ensure total sectors is an integral number of sectors per - # track or mcopy will complain. Sectors are 512 bytes, and we - # generate images with 32 sectors per track. This calculation is - # done in blocks, thus the mod by 16 instead of 32. - BLOCKS=$(expr $BLOCKS + $(expr 16 - $(expr $BLOCKS % 16))) - - # mkdosfs will sometimes use FAT16 when it is not appropriate, - # resulting in a boot failure from SYSLINUX. Use FAT32 for - # images larger than 512MB, otherwise let mkdosfs decide. - if [ $(expr $BLOCKS / 1024) -gt 512 ]; then - FATSIZE="-F 32" - fi - - # mkdosfs will fail if ${FATIMG} exists. Since we are creating an - # new image, it is safe to delete any previous image. - if [ -e ${FATIMG} ]; then - rm ${FATIMG} - fi - - if [ -z "${HDDIMG_ID}" ]; then - mkdosfs ${FATSIZE} -n ${BOOTIMG_VOLUME_ID} -S 512 -C ${FATIMG} \ - ${BLOCKS} - else - mkdosfs ${FATSIZE} -n ${BOOTIMG_VOLUME_ID} -S 512 -C ${FATIMG} \ - ${BLOCKS} -i ${HDDIMG_ID} - fi - - # Copy FATSOURCEDIR recursively into the image file directly - mcopy -i ${FATIMG} -s ${FATSOURCEDIR}/* ::/ -} - -build_hddimg() { - # Create an HDD image - if [ "${NOHDD}" != "1" ] ; then - populate ${HDDDIR} - - if [ "${PCBIOS}" = "1" ]; then - syslinux_hddimg_populate ${HDDDIR} - fi - if [ "${EFI}" = "1" ]; then - efi_hddimg_populate ${HDDDIR} - fi - - # Check the size of ${HDDDIR}/rootfs.img, error out if it - # exceeds 4GB, it is the single file's max size of FAT fs. - if [ -f ${HDDDIR}/rootfs.img ]; then - rootfs_img_size=`stat -c '%s' ${HDDDIR}/rootfs.img` - max_size=`expr 4 \* 1024 \* 1024 \* 1024` - if [ $rootfs_img_size -gt $max_size ]; then - bberror "${HDDDIR}/rootfs.img execeeds 4GB," - bberror "this doesn't work on FAT filesystem, you can try either of:" - bberror "1) Reduce the size of rootfs.img" - bbfatal "2) Use iso, vmdk or vdi to instead of hddimg\n" - fi - fi - - build_fat_img ${HDDDIR} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hddimg - - if [ "${PCBIOS}" = "1" ]; then - syslinux_hddimg_install - fi - - chmod 644 ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hddimg - - cd ${DEPLOY_DIR_IMAGE} - rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.hddimg - ln -s ${IMAGE_NAME}.hddimg ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.hddimg - fi -} - -python do_bootimg() { - if d.getVar("PCBIOS", True) == "1": - bb.build.exec_func('build_syslinux_cfg', d) - if d.getVar("EFI", True) == "1": - bb.build.exec_func('build_efi_cfg', d) - bb.build.exec_func('build_hddimg', d) - bb.build.exec_func('build_iso', d) -} - -IMAGE_TYPEDEP_iso = "ext4" -IMAGE_TYPEDEP_hddimg = "ext4" -IMAGE_TYPES_MASKED += "iso hddimg" - -addtask bootimg before do_build diff --git a/meta/classes/bugzilla.bbclass b/meta/classes/bugzilla.bbclass index 3fc8956428..8909c27348 100644 --- a/meta/classes/bugzilla.bbclass +++ b/meta/classes/bugzilla.bbclass @@ -110,14 +110,14 @@ python bugzilla_eventhandler() { return if name == "TaskFailed": - xmlrpc = data.getVar("BUGZILLA_XMLRPC", True) - user = data.getVar("BUGZILLA_USER", True) - passw = data.getVar("BUGZILLA_PASS", True) - product = data.getVar("BUGZILLA_PRODUCT", True) - compon = data.getVar("BUGZILLA_COMPONENT", True) - version = data.getVar("BUGZILLA_VERSION", True) - - proxy = data.getVar('http_proxy', True ) + xmlrpc = data.getVar("BUGZILLA_XMLRPC") + user = data.getVar("BUGZILLA_USER") + passw = data.getVar("BUGZILLA_PASS") + product = data.getVar("BUGZILLA_PRODUCT") + compon = data.getVar("BUGZILLA_COMPONENT") + version = data.getVar("BUGZILLA_VERSION") + + proxy = data.getVar('http_proxy') if (proxy): import urllib2 s, u, p, hostport = urllib2._parse_proxy(proxy) @@ -133,14 +133,14 @@ python bugzilla_eventhandler() { 'component': compon} # evil hack to figure out what is going on - debug_file = open(os.path.join(data.getVar("TMPDIR", True),"..","bugzilla-log"),"a") + debug_file = open(os.path.join(data.getVar("TMPDIR"),"..","bugzilla-log"),"a") file = None - bugname = "%(package)s-%(pv)s-autobuild" % { "package" : data.getVar("PN", True), - "pv" : data.getVar("PV", True), + bugname = "%(package)s-%(pv)s-autobuild" % { "package" : data.getVar("PN"), + "pv" : data.getVar("PV"), } - log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T', True), event.task)) - text = "The %s step in %s failed at %s for machine %s" % (e.task, data.getVar("PN", True), data.getVar('DATETIME', True), data.getVar( 'MACHINE', True ) ) + log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T'), event.task)) + text = "The %s step in %s failed at %s for machine %s" % (e.task, data.getVar("PN"), data.getVar('DATETIME'), data.getVar('MACHINE') ) if len(log_file) != 0: print >> debug_file, "Adding log file %s" % log_file[0] file = open(log_file[0], 'r') @@ -168,7 +168,7 @@ python bugzilla_eventhandler() { if bug_number and log: print >> debug_file, "The bug is known as '%s'" % bug_number - desc = "Build log for machine %s" % (data.getVar('MACHINE', True)) + desc = "Build log for machine %s" % (data.getVar('MACHINE')) if not bugzilla_create_attachment(debug_file, server, args.copy(), bug_number, text, log_file[0], log, desc): print >> debug_file, "Failed to attach the build log for bug #%s" % bug_number else: diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass index 4db0441993..f543bb73d6 100644 --- a/meta/classes/buildhistory.bbclass +++ b/meta/classes/buildhistory.bbclass @@ -3,7 +3,7 @@ # # Based in part on testlab.bbclass and packagehistory.bbclass # -# Copyright (C) 2011-2014 Intel Corporation +# Copyright (C) 2011-2016 Intel Corporation # Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org> # @@ -11,33 +11,76 @@ BUILDHISTORY_FEATURES ?= "image package sdk" BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}" BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" -BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}/${IMAGE_BASENAME}" + +# Setting this to non-empty will remove the old content of the buildhistory as part of +# the current bitbake invocation and replace it with information about what was built +# during the build. +# +# This is meant to be used in continuous integration (CI) systems when invoking bitbake +# for full world builds. The effect in that case is that information about packages +# that no longer get build also gets removed from the buildhistory, which is not +# the case otherwise. +# +# The advantage over manually cleaning the buildhistory outside of bitbake is that +# the "version-going-backwards" check still works. When relying on that, be careful +# about failed world builds: they will lead to incomplete information in the +# buildhistory because information about packages that could not be built will +# also get removed. A CI system should handle that by discarding the buildhistory +# of failed builds. +# +# The expected usage is via auto.conf, but passing via the command line also works +# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 +BUILDHISTORY_RESET ?= "" + +BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}" +BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" +BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}" BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group" +BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf" BUILDHISTORY_COMMIT ?= "0" BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>" BUILDHISTORY_PUSH_REPO ?= "" SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory" -# We want to avoid influence the signatures of sstate tasks - first the function itself: +# We want to avoid influencing the signatures of sstate tasks - first the function itself: sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" # then the value added to SSTATEPOSTINSTFUNCS: SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" +# Similarly for our function that gets the output signatures +SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs" +sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs" +SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs" + +# All items excepts those listed here will be removed from a recipe's +# build history directory by buildhistory_emit_pkghistory(). This is +# necessary because some of these items (package directories, files that +# we no longer emit) might be obsolete. # -# Write out metadata about this package for comparision when writing future packages +# When extending build history, derive your class from buildhistory.bbclass +# and extend this list here with the additional files created by the derived +# class. +BUILDHISTORY_PRESERVE = "latest latest_srcrev" + +PATCH_GIT_USER_EMAIL ?= "buildhistory@oe" +PATCH_GIT_USER_NAME ?= "OpenEmbedded" + +# +# Write out metadata about this package for comparison when writing future packages # python buildhistory_emit_pkghistory() { - if not d.getVar('BB_CURRENTTASK', True) in ['packagedata', 'packagedata_setscene']: + if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: return 0 - if not "package" in (d.getVar('BUILDHISTORY_FEATURES', True) or "").split(): + if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): return 0 import re import json import errno - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') + oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') class RecipeInfo: def __init__(self, name): @@ -48,6 +91,7 @@ python buildhistory_emit_pkghistory() { self.depends = "" self.packages = "" self.srcrev = "" + self.layer = "" class PackageInfo: @@ -80,7 +124,7 @@ python buildhistory_emit_pkghistory() { pkginfo = PackageInfo(pkg) with open(histfile, "r") as f: for line in f: - lns = line.split('=') + lns = line.split('=', 1) name = lns[0].strip() value = lns[1].strip(" \t\r\n").strip('"') if name == "PE": @@ -110,7 +154,7 @@ python buildhistory_emit_pkghistory() { elif name == "RCONFLICTS": pkginfo.rconflicts = value elif name == "PKGSIZE": - pkginfo.size = long(value) + pkginfo.size = int(value) elif name == "FILES": pkginfo.files = value elif name == "FILELIST": @@ -128,13 +172,13 @@ python buildhistory_emit_pkghistory() { def getlastpkgversion(pkg): try: - histfile = os.path.join(pkghistdir, pkg, "latest") + histfile = os.path.join(oldpkghistdir, pkg, "latest") return readPackageInfo(pkg, histfile) except EnvironmentError: return None def sortpkglist(string): - pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+ [^ )]+\))?', string, 0) + pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0) pkglist = [p.group(0) for p in pkgiter] pkglist.sort() return ' '.join(pkglist) @@ -144,12 +188,13 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) - pn = d.getVar('PN', True) - pe = d.getVar('PE', True) or "0" - pv = d.getVar('PV', True) - pr = d.getVar('PR', True) + pn = d.getVar('PN') + pe = d.getVar('PE') or "0" + pv = d.getVar('PV') + pr = d.getVar('PR') + layer = bb.utils.get_file_layer(d.getVar('FILE', True), d) - pkgdata_dir = d.getVar('PKGDATA_DIR', True) + pkgdata_dir = d.getVar('PKGDATA_DIR') packages = "" try: with open(os.path.join(pkgdata_dir, pn)) as f: @@ -165,12 +210,13 @@ python buildhistory_emit_pkghistory() { raise packagelist = packages.split() + preserve = d.getVar('BUILDHISTORY_PRESERVE').split() if not os.path.exists(pkghistdir): bb.utils.mkdirhier(pkghistdir) else: # Remove files for packages that no longer exist for item in os.listdir(pkghistdir): - if item != "latest" and item != "latest_srcrev": + if item not in preserve: if item not in packagelist: itempath = os.path.join(pkghistdir, item) if os.path.isdir(itempath): @@ -184,11 +230,12 @@ python buildhistory_emit_pkghistory() { rcpinfo.pe = pe rcpinfo.pv = pv rcpinfo.pr = pr - rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS', True) or "")) + rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or "")) rcpinfo.packages = packages + rcpinfo.layer = layer write_recipehistory(rcpinfo, d) - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') for pkg in packagelist: pkgdata = {} with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: @@ -197,7 +244,7 @@ python buildhistory_emit_pkghistory() { key = item[0] if key.endswith('_' + pkg): key = key[:-len(pkg)-1] - pkgdata[key] = item[1].decode('utf-8').decode('string_escape') + pkgdata[key] = item[1] pkge = pkgdata.get('PKGE', '0') pkgv = pkgdata['PKGV'] @@ -238,7 +285,7 @@ python buildhistory_emit_pkghistory() { # Gather information about packaged files val = pkgdata.get('FILES_INFO', '') dictval = json.loads(val) - filelist = dictval.keys() + filelist = list(dictval.keys()) filelist.sort() pkginfo.filelist = " ".join(filelist) @@ -250,52 +297,70 @@ python buildhistory_emit_pkghistory() { bb.build.exec_func("buildhistory_list_pkg_files", d) } +python buildhistory_emit_outputsigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output') + bb.utils.mkdirhier(taskoutdir) + currenttask = d.getVar('BB_CURRENTTASK') + pn = d.getVar('PN') + taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask)) + + cwd = os.getcwd() + filesigs = {} + for root, _, files in os.walk(cwd): + for fname in files: + if fname == 'fixmepath': + continue + fullpath = os.path.join(root, fname) + filesigs[os.path.relpath(fullpath, cwd)] = bb.utils.sha256_file(fullpath) + with open(taskfile, 'w') as f: + for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]): + f.write('%s %s\n' % (fpath, fsig)) +} + def write_recipehistory(rcpinfo, d): - import codecs - bb.debug(2, "Writing recipe history") - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') infofile = os.path.join(pkghistdir, "latest") - with codecs.open(infofile, "w", encoding='utf8') as f: + with open(infofile, "w") as f: if rcpinfo.pe != "0": f.write(u"PE = %s\n" % rcpinfo.pe) f.write(u"PV = %s\n" % rcpinfo.pv) f.write(u"PR = %s\n" % rcpinfo.pr) f.write(u"DEPENDS = %s\n" % rcpinfo.depends) f.write(u"PACKAGES = %s\n" % rcpinfo.packages) + f.write(u"LAYER = %s\n" % rcpinfo.layer) def write_pkghistory(pkginfo, d): - import codecs - bb.debug(2, "Writing package history for package %s" % pkginfo.name) - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') pkgpath = os.path.join(pkghistdir, pkginfo.name) if not os.path.exists(pkgpath): bb.utils.mkdirhier(pkgpath) infofile = os.path.join(pkgpath, "latest") - with codecs.open(infofile, "w", encoding='utf8') as f: + with open(infofile, "w") as f: if pkginfo.pe != "0": f.write(u"PE = %s\n" % pkginfo.pe) f.write(u"PV = %s\n" % pkginfo.pv) f.write(u"PR = %s\n" % pkginfo.pr) - pkgvars = {} - pkgvars['PKG'] = pkginfo.pkg if pkginfo.pkg != pkginfo.name else '' - pkgvars['PKGE'] = pkginfo.pkge if pkginfo.pkge != pkginfo.pe else '' - pkgvars['PKGV'] = pkginfo.pkgv if pkginfo.pkgv != pkginfo.pv else '' - pkgvars['PKGR'] = pkginfo.pkgr if pkginfo.pkgr != pkginfo.pr else '' - for pkgvar in pkgvars: - val = pkgvars[pkgvar] - if val: - f.write(u"%s = %s\n" % (pkgvar, val)) - + if pkginfo.pkg != pkginfo.name: + f.write(u"PKG = %s\n" % pkginfo.pkg) + if pkginfo.pkge != pkginfo.pe: + f.write(u"PKGE = %s\n" % pkginfo.pkge) + if pkginfo.pkgv != pkginfo.pv: + f.write(u"PKGV = %s\n" % pkginfo.pkgv) + if pkginfo.pkgr != pkginfo.pr: + f.write(u"PKGR = %s\n" % pkginfo.pkgr) f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides) f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends) f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends) @@ -313,7 +378,7 @@ def write_pkghistory(pkginfo, d): filevarpath = os.path.join(pkgpath, "latest.%s" % filevar) val = pkginfo.filevars[filevar] if val: - with codecs.open(filevarpath, "w", encoding='utf8') as f: + with open(filevarpath, "w") as f: f.write(val) else: if os.path.exists(filevarpath): @@ -325,18 +390,21 @@ def write_pkghistory(pkginfo, d): def buildhistory_list_installed(d, rootfs_type="image"): from oe.rootfs import image_list_installed_packages from oe.sdk import sdk_list_installed_packages + from oe.utils import format_pkg_list process_list = [('file', 'bh_installed_pkgs.txt'),\ ('deps', 'bh_installed_pkgs_deps.txt')] + if rootfs_type == "image": + pkgs = image_list_installed_packages(d) + else: + pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target") + for output_type, output_file in process_list: - output_file_full = os.path.join(d.getVar('WORKDIR', True), output_file) + output_file_full = os.path.join(d.getVar('WORKDIR'), output_file) with open(output_file_full, 'w') as output: - if rootfs_type == "image": - output.write(image_list_installed_packages(d, output_type)) - else: - output.write(sdk_list_installed_packages(d, rootfs_type == "sdk_target", output_type)) + output.write(format_pkg_list(pkgs, output_type)) python buildhistory_list_installed_image() { buildhistory_list_installed(d) @@ -377,15 +445,8 @@ buildhistory_get_installed() { rm $1/depends.tmp # Produce installed package sizes list - printf "" > $1/installed-package-sizes.tmp - cat $pkgcache | while read pkg pkgfile pkgarch - do - size=`oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" ${pkg}_${pkgarch}` - if [ "$size" != "" ] ; then - echo "$size $pkg" >> $1/installed-package-sizes.tmp - fi - done - cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt + oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp + cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt rm $1/installed-package-sizes.tmp # We're now done with the cache, delete it @@ -393,7 +454,7 @@ buildhistory_get_installed() { if [ "$2" != "sdk" ] ; then # Produce some cut-down graphs (for readability) - grep -v kernel_image $1/depends.dot | grep -v kernel-2 | grep -v kernel-3 > $1/depends-nokernel.dot + grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot @@ -465,6 +526,7 @@ buildhistory_get_imageinfo() { return fi + mkdir -p ${BUILDHISTORY_DIR_IMAGE} buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt # Collect files requested in BUILDHISTORY_IMAGE_FILES @@ -499,6 +561,15 @@ buildhistory_get_sdkinfo() { buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt + # Collect files requested in BUILDHISTORY_SDK_FILES + rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files + for f in ${BUILDHISTORY_SDK_FILES}; do + if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then + mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f` + cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f + fi + done + # Record some machine-readable meta-information about the SDK printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END @@ -508,26 +579,70 @@ END echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt } +python buildhistory_get_extra_sdkinfo() { + import operator + import math + + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \ + "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + tasksizes = {} + filesizes = {} + for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')): + for fn in files: + if fn.endswith('.tgz'): + fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024)) + task = fn.rsplit(':', 1)[1].split('_', 1)[1].split('.')[0] + origtotal = tasksizes.get(task, 0) + tasksizes[task] = origtotal + fsize + filesizes[fn] = fsize + with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f: + filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1, 0), reverse=True) + for fn, size in filesizes_sorted: + f.write('%10d KiB %s\n' % (size, fn)) + with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f: + tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1, 0), reverse=True) + for task, size in tasksizes_sorted: + f.write('%10d KiB %s\n' % (size, task)) +} + # By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of # unneeded packages but before the removal of packaging files -ROOTFS_POSTUNINSTALL_COMMAND += " buildhistory_list_installed_image ;\ - buildhistory_get_image_installed ; " +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;" +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed" -IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; " +IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo" # We want these to be the last run so that we get called after complementary package installation -POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target ;\ - buildhistory_get_sdk_installed_target ; " -POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host ;\ - buildhistory_get_sdk_installed_host ; " - -SDK_POSTPROCESS_COMMAND += "buildhistory_get_sdkinfo ; " +POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;" +POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;" +POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;" + +POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;" +POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;" +POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;" + +SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " +SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " + +python buildhistory_write_sigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + # Create sigs file + if hasattr(bb.parse.siggen, 'dump_siglist'): + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task') + bb.utils.mkdirhier(taskoutdir) + bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt')) +} def buildhistory_get_build_id(d): - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" localdata = bb.data.createCopy(d) - bb.data.update_data(localdata) statuslines = [] for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): g = globals() @@ -538,12 +653,12 @@ def buildhistory_get_build_id(d): if flines: statuslines.extend(flines) - statusheader = d.getVar('BUILDCFG_HEADER', True) + statusheader = d.getVar('BUILDCFG_HEADER') return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) def buildhistory_get_metadata_revs(d): # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want - layers = (d.getVar("BBLAYERS", True) or "").split() + layers = (d.getVar("BBLAYERS") or "").split() medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \ base_get_metadata_git_branch(i, None).strip(), \ base_get_metadata_git_revision(i, None)) \ @@ -555,7 +670,7 @@ def outputvars(vars, listvars, d): listvars = listvars.split() ret = "" for var in vars: - value = d.getVar(var, True) or "" + value = d.getVar(var) or "" if var in listvars: # Squash out spaces value = oe.utils.squashspaces(value) @@ -563,17 +678,20 @@ def outputvars(vars, listvars, d): return ret.rstrip('\n') def buildhistory_get_imagevars(d): - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(imagevars, listvars, d) def buildhistory_get_sdkvars(d): - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" - listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext': + # Extensible SDK uses some additional variables + sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN" + listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST" return outputvars(sdkvars, listvars, d) @@ -634,22 +752,16 @@ END ( cd ${BUILDHISTORY_DIR}/ # Initialise the repo if necessary - if [ ! -d .git ] ; then + if [ ! -e .git ] ; then git init -q else git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true git tag -f build-minus-1 > /dev/null 2>&1 || true fi - # If the user hasn't set up their name/email, set some defaults - # just for this repo (otherwise the commit will fail with older - # versions of git) - if ! git config user.email > /dev/null ; then - git config --local user.email "buildhistory@${DISTRO}" - fi - if ! git config user.name > /dev/null ; then - git config --local user.name "buildhistory" - fi + + check_git_config + # Check if there are new/changed files to commit (other than metadata-revs) repostatus=`git status --porcelain | grep -v " metadata-revs$"` HOSTNAME=`hostname 2>/dev/null || echo unknown` @@ -671,18 +783,37 @@ END } python buildhistory_eventhandler() { - if e.data.getVar('BUILDHISTORY_FEATURES', True).strip(): - if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1": - bb.note("Writing buildhistory") - localdata = bb.data.createCopy(e.data) - localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures)) - interrupted = getattr(e, '_interrupted', 0) - localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted)) - bb.build.exec_func("buildhistory_commit", localdata) + if e.data.getVar('BUILDHISTORY_FEATURES').strip(): + reset = e.data.getVar("BUILDHISTORY_RESET") + olddir = e.data.getVar("BUILDHISTORY_OLD_DIR") + if isinstance(e, bb.event.BuildStarted): + if reset: + import shutil + # Clean up after potentially interrupted build. + if os.path.isdir(olddir): + shutil.rmtree(olddir) + rootdir = e.data.getVar("BUILDHISTORY_DIR") + entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ] + bb.utils.mkdirhier(olddir) + for entry in entries: + os.rename(os.path.join(rootdir, entry), + os.path.join(olddir, entry)) + elif isinstance(e, bb.event.BuildCompleted): + if reset: + import shutil + shutil.rmtree(olddir) + if e.data.getVar("BUILDHISTORY_COMMIT") == "1": + bb.note("Writing buildhistory") + bb.build.exec_func("buildhistory_write_sigs", d) + localdata = bb.data.createCopy(e.data) + localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures)) + interrupted = getattr(e, '_interrupted', 0) + localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted)) + bb.build.exec_func("buildhistory_commit", localdata) } addhandler buildhistory_eventhandler -buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted" +buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted" # FIXME this ought to be moved into the fetcher @@ -692,7 +823,7 @@ def _get_srcrev_values(d): """ scms = [] - fetcher = bb.fetch.Fetch(d.getVar('SRC_URI', True).split(), d) + fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d) urldata = fetcher.ud for u in urldata: if urldata[u].method.supports_srcrev(): @@ -724,7 +855,7 @@ def _get_srcrev_values(d): do_fetch[postfuncs] += "write_srcrev" do_fetch[vardepsexclude] += "write_srcrev" python write_srcrev() { - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') srcrevfile = os.path.join(pkghistdir, 'latest_srcrev') srcrevs, tag_srcrevs = _get_srcrev_values(d) @@ -751,12 +882,12 @@ python write_srcrev() { f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev)) f.write('SRCREV_%s = "%s"\n' % (name, srcrev)) else: - f.write('SRCREV = "%s"\n' % srcrevs.itervalues().next()) + f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values()))) if len(tag_srcrevs) > 0: for name, srcrev in tag_srcrevs.items(): f.write('# tag_%s = "%s"\n' % (name, srcrev)) if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev: - pkg = d.getVar('PN', True) + pkg = d.getVar('PN') bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev)) else: diff --git a/meta/classes/buildstats-summary.bbclass b/meta/classes/buildstats-summary.bbclass index 05ead9fdbf..f9b241b6c5 100644 --- a/meta/classes/buildstats-summary.bbclass +++ b/meta/classes/buildstats-summary.bbclass @@ -3,12 +3,11 @@ python buildstats_summary () { import collections import os.path - bn = get_bn(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) + bsdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}") if not os.path.exists(bsdir): return - sstatetasks = (e.data.getVar('SSTATETASKS', True) or '').split() + sstatetasks = (e.data.getVar('SSTATETASKS') or '').split() built = collections.defaultdict(lambda: [set(), set()]) for pf in os.listdir(bsdir): taskdir = os.path.join(bsdir, pf) @@ -31,7 +30,11 @@ python buildstats_summary () { header_printed = True bb.note("Build completion summary:") - bb.note(" {0}: {1}% sstate reuse ({2} setscene, {3} scratch)".format(t, 100*len(sstate)/(len(sstate)+len(no_sstate)), len(sstate), len(no_sstate))) + sstate_count = len(sstate) + no_sstate_count = len(no_sstate) + total_count = sstate_count + no_sstate_count + bb.note(" {0}: {1:.1f}% sstate reuse({2} setscene, {3} scratch)".format( + t, round(100 * sstate_count / total_count, 1), sstate_count, no_sstate_count)) } addhandler buildstats_summary buildstats_summary[eventmask] = "bb.event.BuildCompleted" diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass index 22ec571b88..960653c704 100644 --- a/meta/classes/buildstats.bbclass +++ b/meta/classes/buildstats.bbclass @@ -1,135 +1,77 @@ BUILDSTATS_BASE = "${TMPDIR}/buildstats/" -BUILDSTATS_BNFILE = "${BUILDSTATS_BASE}/.buildname" -BUILDSTATS_DEVFILE = "${BUILDSTATS_BASE}/.device" ################################################################################ # Build statistics gathering. # # The CPU and Time gathering/tracking functions and bbevent inspiration -# were written by Christopher Larson and can be seen here: -# http://kergoth.pastey.net/142813 +# were written by Christopher Larson. # ################################################################################ -def get_process_cputime(pid): +def get_buildprocess_cputime(pid): with open("/proc/%d/stat" % pid, "r") as f: fields = f.readline().rstrip().split() # 13: utime, 14: stime, 15: cutime, 16: cstime return sum(int(field) for field in fields[13:16]) +def get_process_cputime(pid): + import resource + with open("/proc/%d/stat" % pid, "r") as f: + fields = f.readline().rstrip().split() + stats = { + 'utime' : fields[13], + 'stime' : fields[14], + 'cutime' : fields[15], + 'cstime' : fields[16], + } + iostats = {} + if os.path.isfile("/proc/%d/io" % pid): + with open("/proc/%d/io" % pid, "r") as f: + while True: + i = f.readline().strip() + if not i: + break + if not ":" in i: + # one more extra line is appended (empty or containing "0") + # most probably due to race condition in kernel while + # updating IO stats + break + i = i.split(": ") + iostats[i[0]] = i[1] + resources = resource.getrusage(resource.RUSAGE_SELF) + childres = resource.getrusage(resource.RUSAGE_CHILDREN) + return stats, iostats, resources, childres + def get_cputime(): with open("/proc/stat", "r") as f: fields = f.readline().rstrip().split()[1:] return sum(int(field) for field in fields) -def set_bn(e): - bn = e.getPkgs()[0] + "-" + e.data.getVar('MACHINE', True) - try: - os.remove(e.data.getVar('BUILDSTATS_BNFILE', True)) - except: - pass - with open(e.data.getVar('BUILDSTATS_BNFILE', True), "w") as f: - f.write(os.path.join(bn, e.data.getVar('BUILDNAME', True))) - -def get_bn(e): - with open(e.data.getVar('BUILDSTATS_BNFILE', True)) as f: - bn = f.readline() - return bn - -def set_device(e): - tmpdir = e.data.getVar('TMPDIR', True) - devfile = e.data.getVar('BUILDSTATS_DEVFILE', True) - try: - os.remove(devfile) - except: - pass - ############################################################################ - # We look for the volume TMPDIR lives on. To do all disks would make little - # sense and not give us any particularly useful data. In theory we could do - # something like stick DL_DIR on a different partition and this would - # throw stats gathering off. The same goes with SSTATE_DIR. However, let's - # get the basics in here and work on the cornercases later. - # A note. /proc/diskstats does not contain info on encryptfs, tmpfs, etc. - # If we end up hitting one of these fs, we'll just skip diskstats collection. - ############################################################################ - device = os.stat(tmpdir) - majordev = os.major(long(device.st_dev)) - minordev = os.minor(long(device.st_dev)) - ############################################################################ - # Bug 1700: - # Because tmpfs/encryptfs/ramfs etc inserts no entry in /proc/diskstats - # we set rdev to NoLogicalDevice and search for it later. If we find NLD - # we do not collect diskstats as the method to collect meaningful statistics - # for these fs types requires a bit more research. - ############################################################################ - rdev = "NoLogicalDevice" - try: - with open("/proc/diskstats", "r") as f: - for line in f: - if majordev == int(line.split()[0]) and minordev == int(line.split()[1]): - rdev = line.split()[2] - except: - pass - with open(devfile, "w") as f: - f.write(rdev) - -def get_device(e): - with open(e.data.getVar('BUILDSTATS_DEVFILE', True)) as f: - device = f.readline() - return device - -def get_diskstats(dev): - import itertools - ############################################################################ - # For info on what these are, see kernel doc file iostats.txt - ############################################################################ - DSTAT_KEYS = ['ReadsComp', 'ReadsMerged', 'SectRead', 'TimeReads', 'WritesComp', 'SectWrite', 'TimeWrite', 'IOinProgress', 'TimeIO', 'WTimeIO'] - try: - with open("/proc/diskstats", "r") as f: - for x in f: - if dev in x: - diskstats_val = x.rstrip().split()[4:] - except IOError as e: - return - diskstats = dict(itertools.izip(DSTAT_KEYS, diskstats_val)) - return diskstats - -def set_diskdata(var, dev, data): - data.setVar(var, get_diskstats(dev)) +def set_timedata(var, d, server_time): + d.setVar(var, server_time) -def get_diskdata(var, dev, data): - olddiskdata = data.getVar(var, False) - diskdata = {} - if olddiskdata is None: +def get_timedata(var, d, end_time): + oldtime = d.getVar(var, False) + if oldtime is None: return - newdiskdata = get_diskstats(dev) - for key in olddiskdata.iterkeys(): - diskdata["Start"+key] = str(int(olddiskdata[key])) - diskdata["End"+key] = str(int(newdiskdata[key])) - return diskdata + return end_time - oldtime -def set_timedata(var, data, server_time=None): +def set_buildtimedata(var, d): import time - if server_time: - time = server_time - else: - time = time.time() + time = time.time() cputime = get_cputime() - proctime = get_process_cputime(os.getpid()) - data.setVar(var, (time, cputime, proctime)) + proctime = get_buildprocess_cputime(os.getpid()) + d.setVar(var, (time, cputime, proctime)) -def get_timedata(var, data, server_time=None): +def get_buildtimedata(var, d): import time - timedata = data.getVar(var, False) + timedata = d.getVar(var, False) if timedata is None: return oldtime, oldcpu, oldproc = timedata - procdiff = get_process_cputime(os.getpid()) - oldproc + procdiff = get_buildprocess_cputime(os.getpid()) - oldproc cpudiff = get_cputime() - oldcpu - if server_time: - end_time = server_time - else: - end_time = time.time() + end_time = time.time() timediff = end_time - oldtime if cpudiff > 0: cpuperc = float(procdiff) * 100 / cpudiff @@ -137,30 +79,27 @@ def get_timedata(var, data, server_time=None): cpuperc = None return timediff, cpuperc -def write_task_data(status, logfile, dev, e): - bn = get_bn(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) +def write_task_data(status, logfile, e, d): + bn = d.getVar('BUILDNAME') + bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) with open(os.path.join(logfile), "a") as f: - timedata = get_timedata("__timedata_task", e.data, e.time) - if timedata: - elapsedtime, cpu = timedata - f.write(bb.data.expand("${PF}: %s: Elapsed time: %0.2f seconds \n" % - (e.task, elapsedtime), e.data)) + elapsedtime = get_timedata("__timedata_task", d, e.time) + if elapsedtime: + f.write(d.expand("${PF}: %s\n" % e.task)) + f.write(d.expand("Elapsed time: %0.2f seconds\n" % elapsedtime)) + cpu, iostats, resources, childres = get_process_cputime(os.getpid()) if cpu: - f.write("CPU usage: %0.1f%% \n" % cpu) - ############################################################################ - # Here we gather up disk data. In an effort to avoid lying with stats - # I do a bare minimum of analysis of collected data. - # The simple fact is, doing disk io collection on a per process basis - # without effecting build time would be difficult. - # For the best information, running things with BB_TOTAL_THREADS = "1" - # would return accurate per task results. - ############################################################################ - if dev != "NoLogicalDevice": - diskdata = get_diskdata("__diskdata_task", dev, e.data) - if diskdata: - for key in sorted(diskdata.iterkeys()): - f.write(key + ": " + diskdata[key] + "\n") + f.write("utime: %s\n" % cpu['utime']) + f.write("stime: %s\n" % cpu['stime']) + f.write("cutime: %s\n" % cpu['cutime']) + f.write("cstime: %s\n" % cpu['cstime']) + for i in iostats: + f.write("IO %s: %s\n" % (i, iostats[i])) + rusages = ["ru_utime", "ru_stime", "ru_maxrss", "ru_minflt", "ru_majflt", "ru_inblock", "ru_oublock", "ru_nvcsw", "ru_nivcsw"] + for i in rusages: + f.write("rusage %s: %s\n" % (i, getattr(resources, i))) + for i in rusages: + f.write("Child rusage %s: %s\n" % (i, getattr(childres, i))) if status is "passed": f.write("Status: PASSED \n") else: @@ -170,25 +109,26 @@ def write_task_data(status, logfile, dev, e): python run_buildstats () { import bb.build import bb.event - import bb.data import time, subprocess, platform + bn = d.getVar('BUILDNAME') + bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) + taskdir = os.path.join(bsdir, d.getVar('PF')) + if isinstance(e, bb.event.BuildStarted): ######################################################################## - # at first pass make the buildstats heriarchy and then - # set the buildname + # If the kernel was not configured to provide I/O statistics, issue + # a one time warning. ######################################################################## - bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True)) - set_bn(e) - bn = get_bn(e) - set_device(e) - device = get_device(e) + if not os.path.isfile("/proc/%d/io" % os.getpid()): + bb.warn("The Linux kernel on your build host was not configured to provide process I/O statistics. (CONFIG_TASK_IO_ACCOUNTING is not set)") - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) + ######################################################################## + # at first pass make the buildstats hierarchy and then + # set the buildname + ######################################################################## bb.utils.mkdirhier(bsdir) - if device != "NoLogicalDevice": - set_diskdata("__diskdata_build", device, e.data) - set_timedata("__timedata_build", e.data) + set_buildtimedata("__timedata_build", d) build_time = os.path.join(bsdir, "build_stats") # write start of build into build_time with open(build_time, "a") as f: @@ -201,35 +141,21 @@ python run_buildstats () { f.write("Build Started: %0.2f \n" % time.time()) elif isinstance(e, bb.event.BuildCompleted): - bn = get_bn(e) - device = get_device(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) build_time = os.path.join(bsdir, "build_stats") with open(build_time, "a") as f: ######################################################################## # Write build statistics for the build ######################################################################## - timedata = get_timedata("__timedata_build", e.data) + timedata = get_buildtimedata("__timedata_build", d) if timedata: time, cpu = timedata # write end of build and cpu used into build_time f.write("Elapsed time: %0.2f seconds \n" % (time)) if cpu: f.write("CPU usage: %0.1f%% \n" % cpu) - if device != "NoLogicalDevice": - diskio = get_diskdata("__diskdata_build", device, e.data) - if diskio: - for key in sorted(diskio.iterkeys()): - f.write(key + ": " + diskio[key] + "\n") if isinstance(e, bb.build.TaskStarted): - bn = get_bn(e) - device = get_device(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) - taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) - if device != "NoLogicalDevice": - set_diskdata("__diskdata_task", device, e.data) - set_timedata("__timedata_task", e.data, e.time) + set_timedata("__timedata_task", d, e.time) bb.utils.mkdirhier(taskdir) # write into the task event file the name and start time with open(os.path.join(taskdir, e.task), "a") as f: @@ -237,24 +163,23 @@ python run_buildstats () { f.write("Started: %0.2f \n" % e.time) elif isinstance(e, bb.build.TaskSucceeded): - bn = get_bn(e) - device = get_device(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) - taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) - write_task_data("passed", os.path.join(taskdir, e.task), device, e) + write_task_data("passed", os.path.join(taskdir, e.task), e, d) if e.task == "do_rootfs": bs = os.path.join(bsdir, "build_stats") with open(bs, "a") as f: - rootfs = e.data.getVar('IMAGE_ROOTFS', True) - rootfs_size = subprocess.Popen(["du", "-sh", rootfs], stdout=subprocess.PIPE).stdout.read() - f.write("Uncompressed Rootfs size: %s" % rootfs_size) + rootfs = d.getVar('IMAGE_ROOTFS') + if os.path.isdir(rootfs): + try: + rootfs_size = subprocess.check_output(["du", "-sh", rootfs], + stderr=subprocess.STDOUT).decode('utf-8') + f.write("Uncompressed Rootfs size: %s" % rootfs_size) + except subprocess.CalledProcessError as err: + bb.warn("Failed to get rootfs size: %s" % err.output.decode('utf-8')) elif isinstance(e, bb.build.TaskFailed): - bn = get_bn(e) - device = get_device(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) - taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) - write_task_data("failed", os.path.join(taskdir, e.task), device, e) + # Can have a failure before TaskStarted so need to mkdir here too + bb.utils.mkdirhier(taskdir) + write_task_data("failed", os.path.join(taskdir, e.task), e, d) ######################################################################## # Lets make things easier and tell people where the build failed in # build_status. We do this here because BuildCompleted triggers no @@ -262,9 +187,33 @@ python run_buildstats () { ######################################################################## build_status = os.path.join(bsdir, "build_stats") with open(build_status, "a") as f: - f.write(e.data.expand("Failed at: ${PF} at task: %s \n" % e.task)) + f.write(d.expand("Failed at: ${PF} at task: %s \n" % e.task)) } addhandler run_buildstats run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed" +python runqueue_stats () { + import buildstats + from bb import event, runqueue + # We should not record any samples before the first task has started, + # because that's the first activity shown in the process chart. + # Besides, at that point we are sure that the build variables + # are available that we need to find the output directory. + # The persistent SystemStats is stored in the datastore and + # closed when the build is done. + system_stats = d.getVar('_buildstats_system_stats', False) + if not system_stats and isinstance(e, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted)): + system_stats = buildstats.SystemStats(d) + d.setVar('_buildstats_system_stats', system_stats) + if system_stats: + # Ensure that we sample at important events. + done = isinstance(e, bb.event.BuildCompleted) + system_stats.sample(e, force=done) + if done: + system_stats.close() + d.delVar('_buildstats_system_stats') +} + +addhandler runqueue_stats +runqueue_stats[eventmask] = "bb.runqueue.sceneQueueTaskStarted bb.runqueue.runQueueTaskStarted bb.event.HeartbeatEvent bb.event.BuildCompleted bb.event.MonitorDiskEvent" diff --git a/meta/classes/ccache.bbclass b/meta/classes/ccache.bbclass index 2cdce46932..d58c8f6e57 100644 --- a/meta/classes/ccache.bbclass +++ b/meta/classes/ccache.bbclass @@ -1,8 +1,15 @@ -CCACHE = "${@bb.utils.which(d.getVar('PATH', True), 'ccache') and 'ccache '}" -export CCACHE_DIR ?= "${TMPDIR}/ccache/${MULTIMACH_HOST_SYS}/${PN}" +CCACHE = "${@bb.utils.which(d.getVar('PATH'), 'ccache') and 'ccache '}" +export CCACHE_DIR ?= "${TMPDIR}/ccache/${MULTIMACH_TARGET_SYS}/${PN}" CCACHE_DISABLE[unexport] = "1" +# We need to stop ccache considering the current directory or the +# debug-prefix-map target directory to be significant when calculating +# its hash. Without this the cache would be invalidated every time +# ${PV} or ${PR} change. +export CCACHE_NOHASHDIR ?= "1" + +DEPENDS_append_class-target = " ccache-native" +DEPENDS[vardepvalueexclude] = " ccache-native" + do_configure[dirs] =+ "${CCACHE_DIR}" do_kernel_configme[dirs] =+ "${CCACHE_DIR}" - -do_clean[cleandirs] += "${CCACHE_DIR}" diff --git a/meta/classes/chrpath.bbclass b/meta/classes/chrpath.bbclass index e9160af6b4..ad3c3975a5 100644 --- a/meta/classes/chrpath.bbclass +++ b/meta/classes/chrpath.bbclass @@ -5,29 +5,36 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d): import subprocess as sub p = sub.Popen([cmd, '-l', fpath],stdout=sub.PIPE,stderr=sub.PIPE) - err, out = p.communicate() - # If returned succesfully, process stderr for results + out, err = p.communicate() + # If returned successfully, process stdout for results if p.returncode != 0: return + out = out.decode('utf-8') + # Handle RUNPATH as well as RPATH - err = err.replace("RUNPATH=","RPATH=") + out = out.replace("RUNPATH=","RPATH=") # Throw away everything other than the rpath list - curr_rpath = err.partition("RPATH=")[2] + curr_rpath = out.partition("RPATH=")[2] #bb.note("Current rpath for %s is %s" % (fpath, curr_rpath.strip())) - rpaths = curr_rpath.split(":") + rpaths = curr_rpath.strip().split(":") new_rpaths = [] modified = False for rpath in rpaths: # If rpath is already dynamic copy it to new_rpath and continue if rpath.find("$ORIGIN") != -1: - new_rpaths.append(rpath.strip()) + new_rpaths.append(rpath) continue rpath = os.path.normpath(rpath) if baseprefix not in rpath and tmpdir not in rpath: - new_rpaths.append(rpath.strip()) + # Skip standard search paths + if rpath in ['/lib', '/usr/lib', '/lib64/', '/usr/lib64']: + bb.warn("Skipping RPATH %s as is a standard search path for %s" % (rpath, fpath)) + modified = True + continue + new_rpaths.append(rpath) continue - new_rpaths.append("$ORIGIN/" + os.path.relpath(rpath.strip(), os.path.dirname(fpath.replace(rootdir, "/")))) + new_rpaths.append("$ORIGIN/" + os.path.relpath(rpath, os.path.dirname(fpath.replace(rootdir, "/")))) modified = True # if we have modified some rpaths call chrpath to update the binary @@ -37,18 +44,17 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d): p = sub.Popen([cmd, '-r', args, fpath],stdout=sub.PIPE,stderr=sub.PIPE) out, err = p.communicate() if p.returncode != 0: - bb.error("%s: chrpath command failed with exit code %d:\n%s%s" % (d.getVar('PN', True), p.returncode, out, err)) - raise bb.build.FuncFailed + bb.fatal("%s: chrpath command failed with exit code %d:\n%s%s" % (d.getVar('PN'), p.returncode, out, err)) def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d): import subprocess as sub p = sub.Popen([d.expand("${HOST_PREFIX}otool"), '-L', fpath],stdout=sub.PIPE,stderr=sub.PIPE) - err, out = p.communicate() - # If returned succesfully, process stderr for results + out, err = p.communicate() + # If returned successfully, process stdout for results if p.returncode != 0: return - for l in err.split("\n"): + for l in out.split("\n"): if "(compatibility" not in l: continue rpath = l.partition("(compatibility")[0].strip() @@ -57,7 +63,7 @@ def process_file_darwin(cmd, fpath, rootdir, baseprefix, tmpdir, d): newpath = "@loader_path/" + os.path.relpath(rpath, os.path.dirname(fpath.replace(rootdir, "/"))) p = sub.Popen([d.expand("${HOST_PREFIX}install_name_tool"), '-change', rpath, newpath, fpath],stdout=sub.PIPE,stderr=sub.PIPE) - err, out = p.communicate() + out, err = p.communicate() def process_dir (rootdir, directory, d): import stat @@ -66,7 +72,7 @@ def process_dir (rootdir, directory, d): cmd = d.expand('${CHRPATH_BIN}') tmpdir = os.path.normpath(d.getVar('TMPDIR', False)) baseprefix = os.path.normpath(d.expand('${base_prefix}')) - hostos = d.getVar("HOST_OS", True) + hostos = d.getVar("HOST_OS") #bb.debug("Checking %s for binaries to process" % directory) if not os.path.exists(directory): diff --git a/meta/classes/cmake.bbclass b/meta/classes/cmake.bbclass index ae3cc025e8..12df617ad8 100644 --- a/meta/classes/cmake.bbclass +++ b/meta/classes/cmake.bbclass @@ -1,5 +1,5 @@ # Path to the CMake file to process. -OECMAKE_SOURCEPATH ?= "${S}" +OECMAKE_SOURCEPATH ??= "${S}" DEPENDS_prepend = "cmake-native " B = "${WORKDIR}/build" @@ -7,9 +7,6 @@ B = "${WORKDIR}/build" # We need to unset CCACHE otherwise cmake gets too confused CCACHE = "" -# We want the staging and installing functions from autotools -inherit autotools - # C/C++ Compiler (without cpu arch/tune arguments) OECMAKE_C_COMPILER ?= "`echo ${CC} | sed 's/^\([^ ]*\).*/\1/'`" OECMAKE_CXX_COMPILER ?= "`echo ${CXX} | sed 's/^\([^ ]*\).*/\1/'`" @@ -18,10 +15,12 @@ OECMAKE_AR ?= "${AR}" # Compiler flags OECMAKE_C_FLAGS ?= "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS} ${CFLAGS}" OECMAKE_CXX_FLAGS ?= "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS} ${CXXFLAGS}" -OECMAKE_C_FLAGS_RELEASE ?= "${SELECTED_OPTIMIZATION} ${CFLAGS} -DNDEBUG" -OECMAKE_CXX_FLAGS_RELEASE ?= "${SELECTED_OPTIMIZATION} ${CXXFLAGS} -DNDEBUG" +OECMAKE_C_FLAGS_RELEASE ?= "-DNDEBUG" +OECMAKE_CXX_FLAGS_RELEASE ?= "-DNDEBUG" OECMAKE_C_LINK_FLAGS ?= "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS} ${CPPFLAGS} ${LDFLAGS}" OECMAKE_CXX_LINK_FLAGS ?= "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS} ${CXXFLAGS} ${LDFLAGS}" +CXXFLAGS += "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS}" +CFLAGS += "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS}" OECMAKE_RPATH ?= "" OECMAKE_PERLNATIVE_DIR ??= "" @@ -30,6 +29,8 @@ OECMAKE_EXTRA_ROOT_PATH ?= "" OECMAKE_FIND_ROOT_PATH_MODE_PROGRAM = "ONLY" OECMAKE_FIND_ROOT_PATH_MODE_PROGRAM_class-native = "BOTH" +EXTRA_OECMAKE_append = " ${PACKAGECONFIG_CONFARGS}" + # CMake expects target architectures in the format of uname(2), # which do not always match TARGET_ARCH, so all the necessary # conversions should happen here. @@ -41,11 +42,15 @@ def map_target_arch_to_uname_arch(target_arch): return target_arch cmake_do_generate_toolchain_file() { + if [ "${BUILD_SYS}" = "${HOST_SYS}" ]; then + cmake_crosscompiling="set( CMAKE_CROSSCOMPILING FALSE )" + fi cat > ${WORKDIR}/toolchain.cmake <<EOF # CMake system name must be something like "Linux". # This is important for cross-compiling. +$cmake_crosscompiling set( CMAKE_SYSTEM_NAME `echo ${TARGET_OS} | sed -e 's/^./\u&/' -e 's/^\(Linux\).*/\1/'` ) -set( CMAKE_SYSTEM_PROCESSOR ${@map_target_arch_to_uname_arch(d.getVar('TARGET_ARCH', True))} ) +set( CMAKE_SYSTEM_PROCESSOR ${@map_target_arch_to_uname_arch(d.getVar('TARGET_ARCH'))} ) set( CMAKE_C_COMPILER ${OECMAKE_C_COMPILER} ) set( CMAKE_CXX_COMPILER ${OECMAKE_CXX_COMPILER} ) set( CMAKE_ASM_COMPILER ${OECMAKE_C_COMPILER} ) @@ -53,9 +58,9 @@ set( CMAKE_AR ${OECMAKE_AR} CACHE FILEPATH "Archiver" ) set( CMAKE_C_FLAGS "${OECMAKE_C_FLAGS}" CACHE STRING "CFLAGS" ) set( CMAKE_CXX_FLAGS "${OECMAKE_CXX_FLAGS}" CACHE STRING "CXXFLAGS" ) set( CMAKE_ASM_FLAGS "${OECMAKE_C_FLAGS}" CACHE STRING "ASM FLAGS" ) -set( CMAKE_C_FLAGS_RELEASE "${OECMAKE_C_FLAGS_RELEASE}" CACHE STRING "CFLAGS for release" ) -set( CMAKE_CXX_FLAGS_RELEASE "${OECMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "CXXFLAGS for release" ) -set( CMAKE_ASM_FLAGS_RELEASE "${OECMAKE_C_FLAGS_RELEASE}" CACHE STRING "ASM FLAGS for release" ) +set( CMAKE_C_FLAGS_RELEASE "${OECMAKE_C_FLAGS_RELEASE}" CACHE STRING "Additional CFLAGS for release" ) +set( CMAKE_CXX_FLAGS_RELEASE "${OECMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Additional CXXFLAGS for release" ) +set( CMAKE_ASM_FLAGS_RELEASE "${OECMAKE_C_FLAGS_RELEASE}" CACHE STRING "Additional ASM FLAGS for release" ) set( CMAKE_C_LINK_FLAGS "${OECMAKE_C_LINK_FLAGS}" CACHE STRING "LDFLAGS" ) set( CMAKE_CXX_LINK_FLAGS "${OECMAKE_CXX_LINK_FLAGS}" CACHE STRING "LDFLAGS" ) @@ -85,6 +90,8 @@ EOF addtask generate_toolchain_file after do_patch before do_configure +CONFIGURE_FILES = "CMakeLists.txt" + cmake_do_configure() { if [ "${OECMAKE_BUILDPATH}" ]; then bbnote "cmake.bbclass no longer uses OECMAKE_BUILDPATH. The default behaviour is now out-of-tree builds with B=WORKDIR/build." @@ -100,39 +107,41 @@ cmake_do_configure() { # Just like autotools cmake can use a site file to cache result that need generated binaries to run if [ -e ${WORKDIR}/site-file.cmake ] ; then - OECMAKE_SITEFILE=" -C ${WORKDIR}/site-file.cmake" + oecmake_sitefile="-C ${WORKDIR}/site-file.cmake" else - OECMAKE_SITEFILE="" + oecmake_sitefile= fi cmake \ - ${OECMAKE_SITEFILE} \ + $oecmake_sitefile \ ${OECMAKE_SOURCEPATH} \ -DCMAKE_INSTALL_PREFIX:PATH=${prefix} \ - -DCMAKE_INSTALL_BINDIR:PATH=${bindir} \ - -DCMAKE_INSTALL_SBINDIR:PATH=${sbindir} \ - -DCMAKE_INSTALL_LIBEXECDIR:PATH=${libexecdir} \ + -DCMAKE_INSTALL_BINDIR:PATH=${@os.path.relpath(d.getVar('bindir'), d.getVar('prefix'))} \ + -DCMAKE_INSTALL_SBINDIR:PATH=${@os.path.relpath(d.getVar('sbindir'), d.getVar('prefix'))} \ + -DCMAKE_INSTALL_LIBEXECDIR:PATH=${@os.path.relpath(d.getVar('libexecdir'), d.getVar('prefix'))} \ -DCMAKE_INSTALL_SYSCONFDIR:PATH=${sysconfdir} \ - -DCMAKE_INSTALL_SHAREDSTATEDIR:PATH=${sharedstatedir} \ + -DCMAKE_INSTALL_SHAREDSTATEDIR:PATH=${@os.path.relpath(d.getVar('sharedstatedir'), d. getVar('prefix'))} \ -DCMAKE_INSTALL_LOCALSTATEDIR:PATH=${localstatedir} \ - -DCMAKE_INSTALL_LIBDIR:PATH=${libdir} \ - -DCMAKE_INSTALL_INCLUDEDIR:PATH=${includedir} \ - -DCMAKE_INSTALL_DATAROOTDIR:PATH=${datadir} \ + -DCMAKE_INSTALL_LIBDIR:PATH=${@os.path.relpath(d.getVar('libdir'), d.getVar('prefix'))} \ + -DCMAKE_INSTALL_INCLUDEDIR:PATH=${@os.path.relpath(d.getVar('includedir'), d.getVar('prefix'))} \ + -DCMAKE_INSTALL_DATAROOTDIR:PATH=${@os.path.relpath(d.getVar('datadir'), d.getVar('prefix'))} \ -DCMAKE_INSTALL_SO_NO_EXE=0 \ -DCMAKE_TOOLCHAIN_FILE=${WORKDIR}/toolchain.cmake \ -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DCMAKE_NO_SYSTEM_FROM_IMPORTED=1 \ ${EXTRA_OECMAKE} \ -Wno-dev } +do_compile[progress] = "percent" cmake_do_compile() { cd ${B} - base_do_compile + base_do_compile VERBOSE=1 } cmake_do_install() { cd ${B} - autotools_do_install + oe_runmake 'DESTDIR=${D}' install } EXPORT_FUNCTIONS do_configure do_compile do_install do_generate_toolchain_file diff --git a/meta/classes/cml1.bbclass b/meta/classes/cml1.bbclass index 95cf584ad8..38e6613c48 100644 --- a/meta/classes/cml1.bbclass +++ b/meta/classes/cml1.bbclass @@ -26,7 +26,8 @@ python do_menuconfig() { except OSError: mtime = 0 - oe_terminal("${SHELL} -c \"make ${KCONFIG_CONFIG_COMMAND}; if [ \$? -ne 0 ]; then echo 'Command failed.'; printf 'Press any key to continue... '; read r; fi\"", '${PN} Configuration', d) + oe_terminal("${SHELL} -c \"make %s; if [ \$? -ne 0 ]; then echo 'Command failed.'; printf 'Press any key to continue... '; read r; fi\"" % d.getVar('KCONFIG_CONFIG_COMMAND'), + d.getVar('PN') + ' Configuration', d) # FIXME this check can be removed when the minimum bitbake version has been bumped if hasattr(bb.build, 'write_taint'): @@ -41,13 +42,14 @@ python do_menuconfig() { } do_menuconfig[depends] += "ncurses-native:do_populate_sysroot" do_menuconfig[nostamp] = "1" +do_menuconfig[dirs] = "${B}" addtask menuconfig after do_configure python do_diffconfig() { import shutil import subprocess - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') fragment = workdir + '/fragment.cfg' configorig = '.config.orig' config = '.config' @@ -72,4 +74,5 @@ python do_diffconfig() { } do_diffconfig[nostamp] = "1" +do_diffconfig[dirs] = "${B}" addtask diffconfig diff --git a/meta/classes/compress_doc.bbclass b/meta/classes/compress_doc.bbclass index 9b58d82ce5..069d53492e 100644 --- a/meta/classes/compress_doc.bbclass +++ b/meta/classes/compress_doc.bbclass @@ -31,25 +31,25 @@ DOC_DECOMPRESS_CMD[xz] ?= "unxz -v" PACKAGE_PREPROCESS_FUNCS += "package_do_compress_doc compress_doc_updatealternatives" python package_do_compress_doc() { - compress_mode = d.getVar('DOC_COMPRESS', True) - compress_list = (d.getVar('DOC_COMPRESS_LIST', True) or '').split() + compress_mode = d.getVar('DOC_COMPRESS') + compress_list = (d.getVar('DOC_COMPRESS_LIST') or '').split() if compress_mode not in compress_list: bb.fatal('Compression policy %s not supported (not listed in %s)\n' % (compress_mode, compress_list)) - dvar = d.getVar('PKGD', True) + dvar = d.getVar('PKGD') compress_cmds = {} decompress_cmds = {} for mode in compress_list: compress_cmds[mode] = d.getVarFlag('DOC_COMPRESS_CMD', mode) decompress_cmds[mode] = d.getVarFlag('DOC_DECOMPRESS_CMD', mode) - mandir = os.path.abspath(dvar + os.sep + d.getVar("mandir", True)) + mandir = os.path.abspath(dvar + os.sep + d.getVar("mandir")) if os.path.exists(mandir): # Decompress doc files which format is not compress_mode decompress_doc(mandir, compress_mode, decompress_cmds) compress_doc(mandir, compress_mode, compress_cmds) - infodir = os.path.abspath(dvar + os.sep + d.getVar("infodir", True)) + infodir = os.path.abspath(dvar + os.sep + d.getVar("infodir")) if os.path.exists(infodir): # Decompress doc files which format is not compress_mode decompress_doc(infodir, compress_mode, decompress_cmds) @@ -218,18 +218,18 @@ python compress_doc_updatealternatives () { if not bb.data.inherits_class('update-alternatives', d): return - mandir = d.getVar("mandir", True) - infodir = d.getVar("infodir", True) - compress_mode = d.getVar('DOC_COMPRESS', True) - for pkg in (d.getVar('PACKAGES', True) or "").split(): - old_names = (d.getVar('ALTERNATIVE_%s' % pkg, True) or "").split() + mandir = d.getVar("mandir") + infodir = d.getVar("infodir") + compress_mode = d.getVar('DOC_COMPRESS') + for pkg in (d.getVar('PACKAGES') or "").split(): + old_names = (d.getVar('ALTERNATIVE_%s' % pkg) or "").split() new_names = [] for old_name in old_names: - old_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', old_name, True) - old_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, old_name, True) or \ - d.getVarFlag('ALTERNATIVE_TARGET', old_name, True) or \ - d.getVar('ALTERNATIVE_TARGET_%s' % pkg, True) or \ - d.getVar('ALTERNATIVE_TARGET', True) or \ + old_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', old_name) + old_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, old_name) or \ + d.getVarFlag('ALTERNATIVE_TARGET', old_name) or \ + d.getVar('ALTERNATIVE_TARGET_%s' % pkg) or \ + d.getVar('ALTERNATIVE_TARGET') or \ old_link # Sometimes old_target is specified as relative to the link name. old_target = os.path.join(os.path.dirname(old_link), old_target) @@ -241,15 +241,15 @@ python compress_doc_updatealternatives () { new_target = old_target + '.' + compress_mode d.delVarFlag('ALTERNATIVE_LINK_NAME', old_name) d.setVarFlag('ALTERNATIVE_LINK_NAME', new_name, new_link) - if d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, old_name, True): + if d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, old_name): d.delVarFlag('ALTERNATIVE_TARGET_%s' % pkg, old_name) d.setVarFlag('ALTERNATIVE_TARGET_%s' % pkg, new_name, new_target) - elif d.getVarFlag('ALTERNATIVE_TARGET', old_name, True): + elif d.getVarFlag('ALTERNATIVE_TARGET', old_name): d.delVarFlag('ALTERNATIVE_TARGET', old_name) d.setVarFlag('ALTERNATIVE_TARGET', new_name, new_target) - elif d.getVar('ALTERNATIVE_TARGET_%s' % pkg, True): + elif d.getVar('ALTERNATIVE_TARGET_%s' % pkg): d.setVar('ALTERNATIVE_TARGET_%s' % pkg, new_target) - elif d.getVar('ALTERNATIVE_TARGET', old_name, True): + elif d.getVar('ALTERNATIVE_TARGET'): d.setVar('ALTERNATIVE_TARGET', new_target) new_names.append(new_name) diff --git a/meta/classes/copyleft_compliance.bbclass b/meta/classes/copyleft_compliance.bbclass index 907c1836b3..eabf12ce7a 100644 --- a/meta/classes/copyleft_compliance.bbclass +++ b/meta/classes/copyleft_compliance.bbclass @@ -13,7 +13,7 @@ python do_prepare_copyleft_sources () { import os.path import shutil - p = d.getVar('P', True) + p = d.getVar('P') included, reason = copyleft_should_include(d) if not included: bb.debug(1, 'copyleft: %s is excluded: %s' % (p, reason)) @@ -21,13 +21,13 @@ python do_prepare_copyleft_sources () { else: bb.debug(1, 'copyleft: %s is included: %s' % (p, reason)) - sources_dir = d.getVar('COPYLEFT_SOURCES_DIR', True) - dl_dir = d.getVar('DL_DIR', True) - src_uri = d.getVar('SRC_URI', True).split() + sources_dir = d.getVar('COPYLEFT_SOURCES_DIR') + dl_dir = d.getVar('DL_DIR') + src_uri = d.getVar('SRC_URI').split() fetch = bb.fetch2.Fetch(src_uri, d) ud = fetch.ud - pf = d.getVar('PF', True) + pf = d.getVar('PF') dest = os.path.join(sources_dir, pf) shutil.rmtree(dest, ignore_errors=True) bb.utils.mkdirhier(dest) diff --git a/meta/classes/copyleft_filter.bbclass b/meta/classes/copyleft_filter.bbclass index 46be7f7d2f..5867bb9f7e 100644 --- a/meta/classes/copyleft_filter.bbclass +++ b/meta/classes/copyleft_filter.bbclass @@ -6,7 +6,7 @@ # # vi:sts=4:sw=4:et -COPYLEFT_LICENSE_INCLUDE ?= 'GPL* LGPL*' +COPYLEFT_LICENSE_INCLUDE ?= 'GPL* LGPL* AGPL*' COPYLEFT_LICENSE_INCLUDE[type] = 'list' COPYLEFT_LICENSE_INCLUDE[doc] = 'Space separated list of globs which include licenses' @@ -49,7 +49,7 @@ def copyleft_should_include(d): included, motive = False, 'recipe did not match anything' - recipe_type = d.getVar('COPYLEFT_RECIPE_TYPE', True) + recipe_type = d.getVar('COPYLEFT_RECIPE_TYPE') if recipe_type not in oe.data.typed_value('COPYLEFT_RECIPE_TYPES', d): include, motive = False, 'recipe type "%s" is excluded' % recipe_type @@ -57,9 +57,9 @@ def copyleft_should_include(d): exclude = oe.data.typed_value('COPYLEFT_LICENSE_EXCLUDE', d) try: - is_included, reason = oe.license.is_included(d.getVar('LICENSE', True), include, exclude) + is_included, reason = oe.license.is_included(d.getVar('LICENSE'), include, exclude) except oe.license.LicenseError as exc: - bb.fatal('%s: %s' % (d.getVar('PF', True), exc)) + bb.fatal('%s: %s' % (d.getVar('PF'), exc)) else: if is_included: if reason: @@ -69,10 +69,10 @@ def copyleft_should_include(d): else: included, motive = False, 'recipe has excluded licenses: %s' % ', '.join(reason) - if any(fnmatch(d.getVar('PN', True), name) \ + if any(fnmatch(d.getVar('PN'), name) \ for name in oe.data.typed_value('COPYLEFT_PN_INCLUDE', d)): included, motive = True, 'recipe included by name' - if any(fnmatch(d.getVar('PN', True), name) \ + if any(fnmatch(d.getVar('PN'), name) \ for name in oe.data.typed_value('COPYLEFT_PN_EXCLUDE', d)): included, motive = False, 'recipe excluded by name' diff --git a/meta/classes/core-image.bbclass b/meta/classes/core-image.bbclass index 501456dba7..a9a2cec68f 100644 --- a/meta/classes/core-image.bbclass +++ b/meta/classes/core-image.bbclass @@ -2,9 +2,6 @@ # # Copyright (C) 2007-2011 Linux Foundation -LIC_FILES_CHKSUM = "file://${COREBASE}/LICENSE;md5=4d92cd373abda3937c2bc47fbc49d690 \ - file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" - # IMAGE_FEATURES control content of the core reference images # # By default we install packagegroup-core-boot and packagegroup-base-extended packages; @@ -21,17 +18,21 @@ LIC_FILES_CHKSUM = "file://${COREBASE}/LICENSE;md5=4d92cd373abda3937c2bc47fbc49d # - tools-testapps - tools usable to make some device tests # - tools-sdk - SDK (C/C++ compiler, autotools, etc.) # - nfs-server - NFS server +# - nfs-client - NFS client # - ssh-server-dropbear - SSH server (dropbear) # - ssh-server-openssh - SSH server (openssh) -# - qt4-pkgs - Qt4/X11 and demo applications # - hwcodecs - Install hardware acceleration codecs # - package-management - installs package management tools and preserves the package manager database # - debug-tweaks - makes an image suitable for development, e.g. allowing passwordless root logins +# - empty-root-password +# - allow-empty-password +# - post-install-logging # - dev-pkgs - development packages (headers, etc.) for all installed packages in the rootfs # - dbg-pkgs - debug symbol packages for all installed packages in the rootfs # - doc-pkgs - documentation packages for all installed packages in the rootfs # - ptest-pkgs - ptest packages for all ptest-enabled recipes # - read-only-rootfs - tweaks an image to support read-only rootfs +# - splash - bootup splash screen # FEATURE_PACKAGES_x11 = "packagegroup-core-x11" FEATURE_PACKAGES_x11-base = "packagegroup-core-x11-base" @@ -42,9 +43,9 @@ FEATURE_PACKAGES_tools-profile = "packagegroup-core-tools-profile" FEATURE_PACKAGES_tools-testapps = "packagegroup-core-tools-testapps" FEATURE_PACKAGES_tools-sdk = "packagegroup-core-sdk packagegroup-core-standalone-sdk-target" FEATURE_PACKAGES_nfs-server = "packagegroup-core-nfs-server" +FEATURE_PACKAGES_nfs-client = "packagegroup-core-nfs-client" FEATURE_PACKAGES_ssh-server-dropbear = "packagegroup-core-ssh-dropbear" FEATURE_PACKAGES_ssh-server-openssh = "packagegroup-core-ssh-openssh" -FEATURE_PACKAGES_qt4-pkgs = "packagegroup-core-qt-demoapps" FEATURE_PACKAGES_hwcodecs = "${MACHINE_HWCODECS}" diff --git a/meta/classes/cpan-base.bbclass b/meta/classes/cpan-base.bbclass index d9817ba6b6..55ac052695 100644 --- a/meta/classes/cpan-base.bbclass +++ b/meta/classes/cpan-base.bbclass @@ -29,27 +29,12 @@ def get_perl_version(d): return m.group(1) return None -# Determine where the library directories are -def perl_get_libdirs(d): - libdir = d.getVar('libdir', True) - if is_target(d) == "no": - libdir += '/perl-native' - libdir += '/perl' - return libdir - def is_target(d): if not bb.data.inherits_class('native', d): return "yes" return "no" -PERLLIBDIRS := "${@perl_get_libdirs(d)}" +PERLLIBDIRS = "${libdir}/perl" +PERLLIBDIRS_class-native = "${libdir}/perl-native" PERLVERSION := "${@get_perl_version(d)}" PERLVERSION[vardepvalue] = "" - -FILES_${PN}-dbg += "${PERLLIBDIRS}/auto/*/.debug \ - ${PERLLIBDIRS}/auto/*/*/.debug \ - ${PERLLIBDIRS}/auto/*/*/*/.debug \ - ${PERLLIBDIRS}/vendor_perl/${PERLVERSION}/auto/*/.debug \ - ${PERLLIBDIRS}/vendor_perl/${PERLVERSION}/auto/*/*/.debug \ - ${PERLLIBDIRS}/vendor_perl/${PERLVERSION}/auto/*/*/*/.debug \ - " diff --git a/meta/classes/cpan.bbclass b/meta/classes/cpan.bbclass index e2bbd2f63a..8e079e0d55 100644 --- a/meta/classes/cpan.bbclass +++ b/meta/classes/cpan.bbclass @@ -17,7 +17,7 @@ export PERLHOSTLIB = "${STAGING_LIBDIR_NATIVE}/perl-native/perl/${@get_perl_vers cpan_do_configure () { export PERL5LIB="${PERL_ARCHLIB}" - yes '' | perl ${EXTRA_PERLFLAGS} Makefile.PL ${EXTRA_CPANFLAGS} + yes '' | perl ${EXTRA_PERLFLAGS} Makefile.PL INSTALLDIRS=vendor ${EXTRA_CPANFLAGS} # Makefile.PLs can exit with success without generating a # Makefile, e.g. in cases of missing configure time diff --git a/meta/classes/cpan_build.bbclass b/meta/classes/cpan_build.bbclass index 4f648a60cf..fac074d610 100644 --- a/meta/classes/cpan_build.bbclass +++ b/meta/classes/cpan_build.bbclass @@ -8,6 +8,7 @@ EXTRA_CPAN_BUILD_FLAGS ?= "" # Env var which tells perl if it should use host (no) or target (yes) settings export PERLCONFIGTARGET = "${@is_target(d)}" export PERL_ARCHLIB = "${STAGING_LIBDIR}${PERL_OWN_DIR}/perl/${@get_perl_version(d)}" +export PERLHOSTLIB = "${STAGING_LIBDIR_NATIVE}/perl-native/perl/${@get_perl_version(d)}/" export LD = "${CCLD}" cpan_build_do_configure () { @@ -16,22 +17,24 @@ cpan_build_do_configure () { . ${STAGING_LIBDIR}/perl/config.sh fi - perl Build.PL --installdirs vendor \ - --destdir ${D} \ - --install_path arch="${libdir}/perl" \ - --install_path script=${bindir} \ - --install_path bin=${bindir} \ - --install_path bindoc=${mandir}/man1 \ - --install_path libdoc=${mandir}/man3 \ - ${EXTRA_CPAN_BUILD_FLAGS} + perl Build.PL --installdirs vendor --destdir ${D} \ + ${EXTRA_CPAN_BUILD_FLAGS} + + # Build.PLs can exit with success without generating a + # Build, e.g. in cases of missing configure time + # dependencies. This is considered a best practice by + # cpantesters.org. See: + # * http://wiki.cpantesters.org/wiki/CPANAuthorNotes + # * http://www.nntp.perl.org/group/perl.qa/2008/08/msg11236.html + [ -e Build ] || bbfatal "No Build was generated by Build.PL" } cpan_build_do_compile () { - perl Build + perl Build verbose=1 } cpan_build_do_install () { - perl Build install + perl Build install --destdir ${D} } EXPORT_FUNCTIONS do_configure do_compile do_install diff --git a/meta/classes/cross-canadian.bbclass b/meta/classes/cross-canadian.bbclass index 65ba4572cd..49388d4cf2 100644 --- a/meta/classes/cross-canadian.bbclass +++ b/meta/classes/cross-canadian.bbclass @@ -1,5 +1,5 @@ # -# NOTE - When using this class the user is repsonsible for ensuring that +# NOTE - When using this class the user is responsible for ensuring that # TRANSLATED_TARGET_ARCH is added into PN. This ensures that if the TARGET_ARCH # is changed, another nativesdk xxx-canadian-cross can be installed # @@ -15,27 +15,33 @@ STAGING_BINDIR_TOOLCHAIN = "${STAGING_DIR_NATIVE}${bindir_native}/${SDK_ARCH}${S # Update BASE_PACKAGE_ARCH and PACKAGE_ARCHS # PACKAGE_ARCH = "${SDK_ARCH}-${SDKPKGSUFFIX}" -CANADIANEXTRAOS = "linux-uclibc linux-musl" +BASECANADIANEXTRAOS ?= "linux-uclibc linux-musl" +CANADIANEXTRAOS = "${BASECANADIANEXTRAOS}" CANADIANEXTRAVENDOR = "" MODIFYTOS ??= "1" python () { - archs = d.getVar('PACKAGE_ARCHS', True).split() + archs = d.getVar('PACKAGE_ARCHS').split() sdkarchs = [] for arch in archs: sdkarchs.append(arch + '-${SDKPKGSUFFIX}') d.setVar('PACKAGE_ARCHS', " ".join(sdkarchs)) # Allow the following code segment to be disabled, e.g. meta-environment - if d.getVar("MODIFYTOS", True) != "1": + if d.getVar("MODIFYTOS") != "1": return - if d.getVar("TCLIBC", True) == "baremetal": + if d.getVar("TCLIBC") == "baremetal": return - tos = d.getVar("TARGET_OS", True) + tos = d.getVar("TARGET_OS") whitelist = [] + extralibcs = [""] + if "uclibc" in d.getVar("BASECANADIANEXTRAOS"): + extralibcs.append("uclibc") + if "musl" in d.getVar("BASECANADIANEXTRAOS"): + extralibcs.append("musl") for variant in ["", "spe", "x32", "eabi", "n32"]: - for libc in ["", "uclibc", "musl"]: + for libc in extralibcs: entry = "linux" if variant and libc: entry = entry + "-" + libc + variant @@ -45,38 +51,45 @@ python () { entry = entry + "-" + libc whitelist.append(entry) if tos not in whitelist: - bb.fatal("Building cross-candian for an unknown TARGET_SYS (%s), please update cross-canadian.bbclass" % d.getVar("TARGET_SYS", True)) + bb.fatal("Building cross-candian for an unknown TARGET_SYS (%s), please update cross-canadian.bbclass" % d.getVar("TARGET_SYS")) for n in ["PROVIDES", "DEPENDS"]: - d.setVar(n, d.getVar(n, True)) - d.setVar("STAGING_BINDIR_TOOLCHAIN", d.getVar("STAGING_BINDIR_TOOLCHAIN", True)) + d.setVar(n, d.getVar(n)) + d.setVar("STAGING_BINDIR_TOOLCHAIN", d.getVar("STAGING_BINDIR_TOOLCHAIN")) for prefix in ["AR", "AS", "DLLTOOL", "CC", "CXX", "GCC", "LD", "LIPO", "NM", "OBJDUMP", "RANLIB", "STRIP", "WINDRES"]: n = prefix + "_FOR_TARGET" - d.setVar(n, d.getVar(n, True)) + d.setVar(n, d.getVar(n)) # This is a bit ugly. We need to zero LIBC/ABI extension which will change TARGET_OS # however we need the old value in some variables. We expand those here first. - tarch = d.getVar("TARGET_ARCH", True) + tarch = d.getVar("TARGET_ARCH") if tarch == "x86_64": d.setVar("LIBCEXTENSION", "") d.setVar("ABIEXTENSION", "") - d.appendVar("CANADIANEXTRAOS", " linux-gnux32 linux-uclibcx32 linux-muslx32") + d.appendVar("CANADIANEXTRAOS", " linux-gnux32") + for extraos in d.getVar("BASECANADIANEXTRAOS").split(): + d.appendVar("CANADIANEXTRAOS", " " + extraos + "x32") elif tarch == "powerpc": # PowerPC can build "linux" and "linux-gnuspe" d.setVar("LIBCEXTENSION", "") d.setVar("ABIEXTENSION", "") - d.appendVar("CANADIANEXTRAOS", " linux-gnuspe linux-uclibcspe linux-muslspe") + d.appendVar("CANADIANEXTRAOS", " linux-gnuspe") + for extraos in d.getVar("BASECANADIANEXTRAOS").split(): + d.appendVar("CANADIANEXTRAOS", " " + extraos + "spe") elif tarch == "mips64": - d.appendVar("CANADIANEXTRAOS", " linux-gnun32 linux-uclibcn32 linux-musln32") - if tarch == "arm": + d.appendVar("CANADIANEXTRAOS", " linux-gnun32") + for extraos in d.getVar("BASECANADIANEXTRAOS").split(): + d.appendVar("CANADIANEXTRAOS", " " + extraos + "n32") + if tarch == "arm" or tarch == "armeb": + d.appendVar("CANADIANEXTRAOS", " linux-gnueabi linux-musleabi linux-uclibceabi") d.setVar("TARGET_OS", "linux-gnueabi") else: d.setVar("TARGET_OS", "linux") # Also need to handle multilib target vendors - vendors = d.getVar("CANADIANEXTRAVENDOR", True) + vendors = d.getVar("CANADIANEXTRAVENDOR") if not vendors: vendors = all_multilib_tune_values(d, 'TARGET_VENDOR') - origvendor = d.getVar("TARGET_VENDOR_MULTILIB_ORIGINAL", True) + origvendor = d.getVar("TARGET_VENDOR_MULTILIB_ORIGINAL") if origvendor: d.setVar("TARGET_VENDOR", origvendor) if origvendor not in vendors.split(): @@ -87,9 +100,9 @@ MULTIMACH_TARGET_SYS = "${PACKAGE_ARCH}${HOST_VENDOR}-${HOST_OS}" INHIBIT_DEFAULT_DEPS = "1" -STAGING_DIR_HOST = "${STAGING_DIR}/${HOST_ARCH}-${SDKPKGSUFFIX}${HOST_VENDOR}-${HOST_OS}" +STAGING_DIR_HOST = "${RECIPE_SYSROOT}" -TOOLCHAIN_OPTIONS = " --sysroot=${STAGING_DIR}/${HOST_ARCH}-${SDKPKGSUFFIX}${HOST_VENDOR}-${HOST_OS}" +TOOLCHAIN_OPTIONS = " --sysroot=${RECIPE_SYSROOT}" PATH_append = ":${TMPDIR}/sysroots/${HOST_ARCH}/${bindir_cross}" PKGHIST_DIR = "${TMPDIR}/pkghistory/${HOST_ARCH}-${SDKPKGSUFFIX}${HOST_VENDOR}-${HOST_OS}/" @@ -102,8 +115,13 @@ HOST_CC_ARCH = "${SDK_CC_ARCH}" HOST_LD_ARCH = "${SDK_LD_ARCH}" HOST_AS_ARCH = "${SDK_AS_ARCH}" +TARGET_CPPFLAGS = "${BUILDSDK_CPPFLAGS}" +TARGET_CFLAGS = "${BUILDSDK_CFLAGS}" +TARGET_CXXFLAGS = "${BUILDSDK_CXXFLAGS}" +TARGET_LDFLAGS = "${BUILDSDK_LDFLAGS}" + #assign DPKG_ARCH -DPKG_ARCH = "${SDK_ARCH}" +DPKG_ARCH = "${@debian_arch_map(d.getVar('SDK_ARCH'), '')}" CPPFLAGS = "${BUILDSDK_CPPFLAGS}" CFLAGS = "${BUILDSDK_CFLAGS}" @@ -143,9 +161,6 @@ libdir = "${exec_prefix}/lib/${TARGET_ARCH}${TARGET_VENDOR}-${TARGET_OS}" libexecdir = "${exec_prefix}/libexec/${TARGET_ARCH}${TARGET_VENDOR}-${TARGET_OS}" FILES_${PN} = "${prefix}" -FILES_${PN}-dbg += "${prefix}/.debug \ - ${prefix}/bin/.debug \ - " export PKG_CONFIG_DIR = "${STAGING_DIR_HOST}${layout_libdir}/pkgconfig" export PKG_CONFIG_SYSROOT_DIR = "${STAGING_DIR_HOST}" @@ -159,6 +174,7 @@ USE_NLS = "${SDKUSE_NLS}" # and not any particular tune that is enabled. TARGET_ARCH[vardepsexclude] = "TUNE_ARCH" +PKGDATA_DIR = "${TMPDIR}/pkgdata/${SDK_SYS}" # If MLPREFIX is set by multilib code, shlibs # points to the wrong place so force it SHLIBSDIRS = "${PKGDATA_DIR}/nativesdk-shlibs2" diff --git a/meta/classes/cross.bbclass b/meta/classes/cross.bbclass index 3eab5b907b..8757303678 100644 --- a/meta/classes/cross.bbclass +++ b/meta/classes/cross.bbclass @@ -17,15 +17,22 @@ HOST_CC_ARCH = "${BUILD_CC_ARCH}" HOST_LD_ARCH = "${BUILD_LD_ARCH}" HOST_AS_ARCH = "${BUILD_AS_ARCH}" -STAGING_DIR_HOST = "${STAGING_DIR}/${HOST_ARCH}${HOST_VENDOR}-${HOST_OS}" +export lt_cv_sys_lib_dlsearch_path_spec = "${libdir} ${base_libdir} /lib /lib64 /usr/lib /usr/lib64" + +STAGING_DIR_HOST = "${RECIPE_SYSROOT_NATIVE}" PACKAGE_ARCH = "${BUILD_ARCH}" -MULTIMACH_TARGET_SYS = "${PACKAGE_ARCH}${BUILD_VENDOR}-${BUILD_OS}" +MULTIMACH_TARGET_SYS = "${BUILD_ARCH}${BUILD_VENDOR}-${BUILD_OS}" export PKG_CONFIG_DIR = "${exec_prefix}/lib/pkgconfig" export PKG_CONFIG_SYSROOT_DIR = "" +TARGET_CPPFLAGS = "${BUILD_CPPFLAGS}" +TARGET_CFLAGS = "${BUILD_CFLAGS}" +TARGET_CXXFLAGS = "${BUILD_CXXFLAGS}" +TARGET_LDFLAGS = "${BUILD_LDFLAGS}" + CPPFLAGS = "${BUILD_CPPFLAGS}" CFLAGS = "${BUILD_CFLAGS}" CXXFLAGS = "${BUILD_CFLAGS}" @@ -36,6 +43,10 @@ TOOLCHAIN_OPTIONS = "" DEPENDS_GETTEXT = "gettext-native" +# This class encodes staging paths into its scripts data so can only be +# reused if we manipulate the paths. +SSTATE_SCAN_CMD ?= "${SSTATE_SCAN_CMD_NATIVE}" + # Path mangling needed by the cross packaging # Note that we use := here to ensure that libdir and includedir are # target paths. @@ -68,10 +79,16 @@ do_install () { USE_NLS = "no" -deltask package -deltask packagedata -deltask package_qa -deltask package_write_ipk -deltask package_write_deb -deltask package_write_rpm -deltask package_write +export CC = "${BUILD_CC}" +export CXX = "${BUILD_CXX}" +export FC = "${BUILD_FC}" +export CPP = "${BUILD_CPP}" +export LD = "${BUILD_LD}" +export CCLD = "${BUILD_CCLD}" +export AR = "${BUILD_AR}" +export AS = "${BUILD_AS}" +export RANLIB = "${BUILD_RANLIB}" +export STRIP = "${BUILD_STRIP}" +export NM = "${BUILD_NM}" + +inherit nopackages diff --git a/meta/classes/crosssdk.bbclass b/meta/classes/crosssdk.bbclass index 87d5cf5d37..ddb98d22bc 100644 --- a/meta/classes/crosssdk.bbclass +++ b/meta/classes/crosssdk.bbclass @@ -5,12 +5,15 @@ MACHINEOVERRIDES = "" PACKAGE_ARCH = "${SDK_ARCH}" python () { # set TUNE_PKGARCH to SDK_ARCH - d.setVar('TUNE_PKGARCH', d.getVar('SDK_ARCH', True)) + d.setVar('TUNE_PKGARCH', d.getVar('SDK_ARCH')) } -STAGING_DIR_TARGET = "${STAGING_DIR}/${SDK_ARCH}-${SDKPKGSUFFIX}${SDK_VENDOR}-${SDK_OS}" STAGING_BINDIR_TOOLCHAIN = "${STAGING_DIR_NATIVE}${bindir_native}/${TARGET_ARCH}${TARGET_VENDOR}-${TARGET_OS}" +# This class encodes staging paths into its scripts data so can only be +# reused if we manipulate the paths. +SSTATE_SCAN_CMD ?= "${SSTATE_SCAN_CMD_NATIVE}" + TARGET_ARCH = "${SDK_ARCH}" TARGET_VENDOR = "${SDK_VENDOR}" TARGET_OS = "${SDK_OS}" @@ -18,8 +21,13 @@ TARGET_PREFIX = "${SDK_PREFIX}" TARGET_CC_ARCH = "${SDK_CC_ARCH}" TARGET_LD_ARCH = "${SDK_LD_ARCH}" TARGET_AS_ARCH = "${SDK_AS_ARCH}" +TARGET_CPPFLAGS = "${BUILD_CPPFLAGS}" +TARGET_CFLAGS = "${BUILD_CFLAGS}" +TARGET_CXXFLAGS = "${BUILD_CXXFLAGS}" +TARGET_LDFLAGS = "${BUILD_LDFLAGS}" TARGET_FPU = "" + target_libdir = "${SDKPATHNATIVE}${libdir_nativesdk}" target_includedir = "${SDKPATHNATIVE}${includedir_nativesdk}" target_base_libdir = "${SDKPATHNATIVE}${base_libdir_nativesdk}" @@ -30,7 +38,7 @@ baselib = "lib" do_populate_sysroot[stamp-extra-info] = "" do_packagedata[stamp-extra-info] = "" -# Need to force this to ensure consitency accross architectures +# Need to force this to ensure consitency across architectures EXTRA_OECONF_GCC_FLOAT = "" USE_NLS = "no" diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass new file mode 100644 index 0000000000..0e4294fdc4 --- /dev/null +++ b/meta/classes/cve-check.bbclass @@ -0,0 +1,269 @@ +# This class is used to check recipes against public CVEs. +# +# In order to use this class just inherit the class in the +# local.conf file and it will add the cve_check task for +# every recipe. The task can be used per recipe, per image, +# or using the special cases "world" and "universe". The +# cve_check task will print a warning for every unpatched +# CVE found and generate a file in the recipe WORKDIR/cve +# directory. If an image is build it will generate a report +# in DEPLOY_DIR_IMAGE for all the packages used. +# +# Example: +# bitbake -c cve_check openssl +# bitbake core-image-sato +# bitbake -k -c cve_check universe +# +# DISCLAIMER +# +# This class/tool is meant to be used as support and not +# the only method to check against CVEs. Running this tool +# doesn't guarantee your packages are free of CVEs. + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ?= "${BPN}" + +CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd.db" + +CVE_CHECK_LOCAL_DIR ?= "${WORKDIR}/cve" +CVE_CHECK_LOCAL_FILE ?= "${CVE_CHECK_LOCAL_DIR}/cve.log" +CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check" + +CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" +CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" +CVE_CHECK_COPY_FILES ??= "1" +CVE_CHECK_CREATE_MANIFEST ??= "1" + +# Whitelist for packages (PN) +CVE_CHECK_PN_WHITELIST = "\ + glibc-locale \ +" + +# Whitelist for CVE and version of package +CVE_CHECK_CVE_WHITELIST = "{\ + 'CVE-2014-2524': ('6.3','5.2',), \ +}" + +python do_cve_check () { + """ + Check recipe for patched and unpatched CVEs + """ + + if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")): + patched_cves = get_patches_cves(d) + patched, unpatched = check_cves(d, patched_cves) + if patched or unpatched: + cve_data = get_cve_info(d, patched + unpatched) + cve_write_data(d, patched, unpatched, cve_data) + else: + bb.note("Failed to update CVE database, skipping CVE check") +} + +addtask cve_check after do_unpack before do_build +do_cve_check[depends] = "cve-check-tool-native:do_populate_sysroot cve-check-tool-native:do_populate_cve_db" +do_cve_check[nostamp] = "1" + +python cve_check_cleanup () { + """ + Delete the file used to gather all the CVE information. + """ + + bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE")) +} + +addhandler cve_check_cleanup +cve_check_cleanup[eventmask] = "bb.cooker.CookerExit" + +python cve_check_write_rootfs_manifest () { + """ + Create CVE manifest when building an image + """ + + import shutil + + if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")): + bb.note("Writing rootfs CVE manifest") + deploy_dir = d.getVar("DEPLOY_DIR_IMAGE") + link_name = d.getVar("IMAGE_LINK_NAME") + manifest_name = d.getVar("CVE_CHECK_MANIFEST") + cve_tmp_file = d.getVar("CVE_CHECK_TMP_FILE") + + shutil.copyfile(cve_tmp_file, manifest_name) + + if manifest_name and os.path.exists(manifest_name): + manifest_link = os.path.join(deploy_dir, "%s.cve" % link_name) + # If we already have another manifest, update symlinks + if os.path.exists(os.path.realpath(manifest_link)): + os.remove(manifest_link) + os.symlink(os.path.basename(manifest_name), manifest_link) + bb.plain("Image CVE report stored in: %s" % manifest_name) +} + +ROOTFS_POSTPROCESS_COMMAND_prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" + +def get_patches_cves(d): + """ + Get patches that solve CVEs using the "CVE: " tag. + """ + + import re + + pn = d.getVar("PN") + cve_match = re.compile("CVE:( CVE\-\d{4}\-\d+)+") + patched_cves = set() + bb.debug(2, "Looking for patches that solves CVEs for %s" % pn) + for url in src_patches(d): + patch_file = bb.fetch.decodeurl(url)[2] + with open(patch_file, "r", encoding="utf-8") as f: + try: + patch_text = f.read() + except UnicodeDecodeError: + bb.debug(1, "Failed to read patch %s using UTF-8 encoding" + " trying with iso8859-1" % patch_file) + f.close() + with open(patch_file, "r", encoding="iso8859-1") as f: + patch_text = f.read() + + # Search for the "CVE: " line + match = cve_match.search(patch_text) + if match: + # Get only the CVEs without the "CVE: " tag + cves = patch_text[match.start()+5:match.end()] + for cve in cves.split(): + bb.debug(2, "Patch %s solves %s" % (patch_file, cve)) + patched_cves.add(cve) + else: + bb.debug(2, "Patch %s doesn't solve CVEs" % patch_file) + + return patched_cves + +def check_cves(d, patched_cves): + """ + Run cve-check-tool looking for patched and unpatched CVEs. + """ + + import ast, csv, tempfile, subprocess, io + + cves_patched = [] + cves_unpatched = [] + bpn = d.getVar("CVE_PRODUCT") + pv = d.getVar("PV").split("git+")[0] + cves = " ".join(patched_cves) + cve_db_dir = d.getVar("CVE_CHECK_DB_DIR") + cve_whitelist = ast.literal_eval(d.getVar("CVE_CHECK_CVE_WHITELIST")) + cve_cmd = "cve-check-tool" + cmd = [cve_cmd, "--no-html", "--csv", "--not-affected", "-t", "faux", "-d", cve_db_dir] + + # If the recipe has been whitlisted we return empty lists + if d.getVar("PN") in d.getVar("CVE_CHECK_PN_WHITELIST").split(): + bb.note("Recipe has been whitelisted, skipping check") + return ([], []) + + # It is needed to export the proxies to download the database using HTTP + bb.utils.export_proxies(d) + + try: + # Write the faux CSV file to be used with cve-check-tool + fd, faux = tempfile.mkstemp(prefix="cve-faux-") + with os.fdopen(fd, "w") as f: + f.write("%s,%s,%s," % (bpn, pv, cves)) + cmd.append(faux) + + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + bb.debug(2, "Output of command %s:\n%s" % ("\n".join(cmd), output)) + except subprocess.CalledProcessError as e: + bb.warn("Couldn't check for CVEs: %s (output %s)" % (e, e.output)) + finally: + os.remove(faux) + + for row in csv.reader(io.StringIO(output)): + # Third row has the unpatched CVEs + if row[2]: + for cve in row[2].split(): + # Skip if the CVE has been whitlisted for the current version + if pv in cve_whitelist.get(cve,[]): + bb.note("%s-%s has been whitelisted for %s" % (bpn, pv, cve)) + else: + cves_unpatched.append(cve) + bb.debug(2, "%s-%s is not patched for %s" % (bpn, pv, cve)) + # Fourth row has patched CVEs + if row[3]: + for cve in row[3].split(): + cves_patched.append(cve) + bb.debug(2, "%s-%s is patched for %s" % (bpn, pv, cve)) + + return (cves_patched, cves_unpatched) + +def get_cve_info(d, cves): + """ + Get CVE information from the database used by cve-check-tool. + + Unfortunately the only way to get CVE info is set the output to + html (hard to parse) or query directly the database. + """ + + try: + import sqlite3 + except ImportError: + from pysqlite2 import dbapi2 as sqlite3 + + cve_data = {} + db_file = d.getVar("CVE_CHECK_DB_FILE") + placeholder = ",".join("?" * len(cves)) + query = "SELECT * FROM NVD WHERE id IN (%s)" % placeholder + conn = sqlite3.connect(db_file) + cur = conn.cursor() + for row in cur.execute(query, tuple(cves)): + cve_data[row[0]] = {} + cve_data[row[0]]["summary"] = row[1] + cve_data[row[0]]["score"] = row[2] + cve_data[row[0]]["modified"] = row[3] + cve_data[row[0]]["vector"] = row[4] + conn.close() + + return cve_data + +def cve_write_data(d, patched, unpatched, cve_data): + """ + Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and + CVE manifest if enabled. + """ + + cve_file = d.getVar("CVE_CHECK_LOCAL_FILE") + nvd_link = "https://web.nvd.nist.gov/view/vuln/detail?vulnId=" + write_string = "" + first_alert = True + bb.utils.mkdirhier(d.getVar("CVE_CHECK_LOCAL_DIR")) + + for cve in sorted(cve_data): + write_string += "PACKAGE NAME: %s\n" % d.getVar("PN") + write_string += "PACKAGE VERSION: %s\n" % d.getVar("PV") + write_string += "CVE: %s\n" % cve + if cve in patched: + write_string += "CVE STATUS: Patched\n" + else: + write_string += "CVE STATUS: Unpatched\n" + if first_alert: + bb.warn("Found unpatched CVE, for more information check %s" % cve_file) + first_alert = False + write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["summary"] + write_string += "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["score"] + write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] + write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) + + with open(cve_file, "w") as f: + bb.note("Writing file %s with CVE information" % cve_file) + f.write(write_string) + + if d.getVar("CVE_CHECK_COPY_FILES") == "1": + cve_dir = d.getVar("CVE_CHECK_DIR") + bb.utils.mkdirhier(cve_dir) + deploy_file = os.path.join(cve_dir, d.getVar("PN")) + with open(deploy_file, "w") as f: + f.write(write_string) + + if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1": + with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f: + f.write("%s" % write_string) diff --git a/meta/classes/debian.bbclass b/meta/classes/debian.bbclass index 1b6979a285..8124558b81 100644 --- a/meta/classes/debian.bbclass +++ b/meta/classes/debian.bbclass @@ -20,17 +20,17 @@ do_package_write_tar[rdeptask] = "${DEBIANRDEP}" do_package_write_rpm[rdeptask] = "${DEBIANRDEP}" python () { - if not d.getVar("PACKAGES", True): + if not d.getVar("PACKAGES"): d.setVar("DEBIANRDEP", "") } python debian_package_name_hook () { import glob, copy, stat, errno, re - pkgdest = d.getVar('PKGDEST', True) - packages = d.getVar('PACKAGES', True) - bin_re = re.compile(".*/s?" + os.path.basename(d.getVar("bindir", True)) + "$") - lib_re = re.compile(".*/" + os.path.basename(d.getVar("libdir", True)) + "$") + pkgdest = d.getVar('PKGDEST') + packages = d.getVar('PACKAGES') + bin_re = re.compile(".*/s?" + os.path.basename(d.getVar("bindir")) + "$") + lib_re = re.compile(".*/" + os.path.basename(d.getVar("libdir")) + "$") so_re = re.compile("lib.*\.so") def socrunch(s): @@ -53,11 +53,11 @@ python debian_package_name_hook () { return (s[stat.ST_MODE] & stat.S_IEXEC) def add_rprovides(pkg, d): - newpkg = d.getVar('PKG_' + pkg, False) + newpkg = d.getVar('PKG_' + pkg) if newpkg and newpkg != pkg: - provs = (d.getVar('RPROVIDES_' + pkg, True) or "").split() + provs = (d.getVar('RPROVIDES_' + pkg) or "").split() if pkg not in provs: - d.appendVar('RPROVIDES_' + pkg, " " + pkg + " (=" + d.getVar("PKGV", True) + ")") + d.appendVar('RPROVIDES_' + pkg, " " + pkg + " (=" + d.getVar("PKGV") + ")") def auto_libname(packages, orig_pkg): sonames = [] @@ -70,7 +70,7 @@ python debian_package_name_hook () { if lib_re.match(root): has_libs = 1 if so_re.match(os.path.basename(file)): - cmd = (d.getVar('TARGET_PREFIX', True) or "") + "objdump -p " + file + " 2>/dev/null" + cmd = (d.getVar('TARGET_PREFIX') or "") + "objdump -p " + file + " 2>/dev/null" fd = os.popen(cmd) lines = fd.readlines() fd.close() @@ -84,7 +84,7 @@ python debian_package_name_hook () { if len(sonames) == 1: soname = sonames[0] elif len(sonames) > 1: - lead = d.getVar('LEAD_SONAME', True) + lead = d.getVar('LEAD_SONAME') if lead: r = re.compile(lead) filtered = [] @@ -115,7 +115,7 @@ python debian_package_name_hook () { newpkg = pkgname else: newpkg = pkg.replace(orig_pkg, devname, 1) - mlpre=d.getVar('MLPREFIX', True) + mlpre=d.getVar('MLPREFIX') if mlpre: if not newpkg.find(mlpre) == 0: newpkg = mlpre + newpkg @@ -131,7 +131,7 @@ python debian_package_name_hook () { # and later # DEBUG: LIBNAMES: pkgname libtic5 devname libtic pkg ncurses-libticw orig_pkg ncurses-libtic debian_pn None newpkg libticw # so we need to handle ncurses-libticw->libticw5 before ncurses-libtic->libtic5 - for pkg in sorted((d.getVar('AUTO_LIBNAME_PKGS', True) or "").split(), reverse=True): + for pkg in sorted((d.getVar('AUTO_LIBNAME_PKGS') or "").split(), reverse=True): auto_libname(packages, pkg) } diff --git a/meta/classes/deploy.bbclass b/meta/classes/deploy.bbclass index 78f5e4a7ba..8ad07da015 100644 --- a/meta/classes/deploy.bbclass +++ b/meta/classes/deploy.bbclass @@ -8,3 +8,4 @@ python do_deploy_setscene () { } addtask do_deploy_setscene do_deploy[dirs] = "${DEPLOYDIR} ${B}" +do_deploy[stamp-extra-info] = "${MACHINE}" diff --git a/meta/classes/devshell.bbclass b/meta/classes/devshell.bbclass index 4451436473..4de7ea6fce 100644 --- a/meta/classes/devshell.bbclass +++ b/meta/classes/devshell.bbclass @@ -5,14 +5,14 @@ DEVSHELL = "${SHELL}" python do_devshell () { if d.getVarFlag("do_devshell", "manualfakeroot"): d.prependVar("DEVSHELL", "pseudo ") - fakeenv = d.getVar("FAKEROOTENV", True).split() + fakeenv = d.getVar("FAKEROOTENV").split() for f in fakeenv: k = f.split("=") d.setVar(k[0], k[1]) d.appendVar("OE_TERMINAL_EXPORTS", " " + k[0]) d.delVarFlag("do_devshell", "fakeroot") - oe_terminal(d.getVar('DEVSHELL', True), 'OpenEmbedded Developer Shell', d) + oe_terminal(d.getVar('DEVSHELL'), 'OpenEmbedded Developer Shell', d) } addtask devshell after do_patch @@ -65,9 +65,6 @@ def devpyshell(d): os.dup2(m, sys.stdout.fileno()) os.dup2(m, sys.stderr.fileno()) - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) - sys.stdin = os.fdopen(sys.stdin.fileno(), 'r', 0) - bb.utils.nonblockingfd(sys.stdout) bb.utils.nonblockingfd(sys.stderr) bb.utils.nonblockingfd(sys.stdin) @@ -85,7 +82,7 @@ def devpyshell(d): more = False i = code.InteractiveInterpreter(locals=_context) - print("OE PyShell (PN = %s)\n" % d.getVar("PN", True)) + print("OE PyShell (PN = %s)\n" % d.getVar("PN")) def prompt(more): if more: @@ -93,6 +90,7 @@ def devpyshell(d): else: prompt = ps1 sys.stdout.write(prompt) + sys.stdout.flush() # Restore Ctrl+C since bitbake masks this def signal_handler(signal, frame): @@ -114,6 +112,7 @@ def devpyshell(d): continue except EOFError as e: sys.stdout.write("\n") + sys.stdout.flush() except (OSError, IOError) as e: if e.errno == 11: continue diff --git a/meta/classes/devupstream.bbclass b/meta/classes/devupstream.bbclass new file mode 100644 index 0000000000..7780c5482c --- /dev/null +++ b/meta/classes/devupstream.bbclass @@ -0,0 +1,48 @@ +# Class for use in BBCLASSEXTEND to make it easier to have a single recipe that +# can build both stable tarballs and snapshots from upstream source +# repositories. +# +# Usage: +# BBCLASSEXTEND = "devupstream:target" +# SRC_URI_class-devupstream = "git://git.example.com/example" +# SRCREV_class-devupstream = "abcdef" +# +# If the first entry in SRC_URI is a git: URL then S is rewritten to +# WORKDIR/git. +# +# There are a few caveats that remain to be solved: +# - You can't build native or nativesdk recipes using for example +# devupstream:native, you can only build target recipes. +# - If the fetcher requires native tools (such as subversion-native) then +# bitbake won't be able to add them automatically. + +CLASSOVERRIDE .= ":class-devupstream" + +python devupstream_virtclass_handler () { + # Do nothing if this is inherited, as it's for BBCLASSEXTEND + if "devupstream" not in (d.getVar('BBCLASSEXTEND') or ""): + bb.error("Don't inherit devupstream, use BBCLASSEXTEND") + return + + variant = d.getVar("BBEXTENDVARIANT") + if variant not in ("target"): + bb.error("Pass the variant when using devupstream, for example devupstream:target") + return + + # Develpment releases are never preferred by default + d.setVar("DEFAULT_PREFERENCE", "-1") + + uri = bb.fetch2.URI(d.getVar("SRC_URI").split()[0]) + + if uri.scheme == "git": + d.setVar("S", "${WORKDIR}/git") + + # Modify the PV if the recipe hasn't already overridden it + pv = d.getVar("PV") + proto_marker = "+" + uri.scheme + if proto_marker not in pv: + d.setVar("PV", pv + proto_marker + "${SRCPV}") +} + +addhandler devupstream_virtclass_handler +devupstream_virtclass_handler[eventmask] = "bb.event.RecipePreFinalise" diff --git a/meta/classes/distro_features_check.bbclass b/meta/classes/distro_features_check.bbclass index 7e91dbcf4a..e74d3c04ba 100644 --- a/meta/classes/distro_features_check.bbclass +++ b/meta/classes/distro_features_check.bbclass @@ -11,15 +11,15 @@ python () { # Assume at least one var is set. - distro_features = (d.getVar('DISTRO_FEATURES', True) or "").split() + distro_features = (d.getVar('DISTRO_FEATURES') or "").split() - any_of_distro_features = d.getVar('ANY_OF_DISTRO_FEATURES', True) + any_of_distro_features = d.getVar('ANY_OF_DISTRO_FEATURES') if any_of_distro_features: any_of_distro_features = any_of_distro_features.split() if set.isdisjoint(set(any_of_distro_features),set(distro_features)): raise bb.parse.SkipPackage("one of '%s' needs to be in DISTRO_FEATURES" % any_of_distro_features) - required_distro_features = d.getVar('REQUIRED_DISTRO_FEATURES', True) + required_distro_features = d.getVar('REQUIRED_DISTRO_FEATURES') if required_distro_features: required_distro_features = required_distro_features.split() for f in required_distro_features: @@ -28,7 +28,7 @@ python () { else: raise bb.parse.SkipPackage("missing required distro feature '%s' (not in DISTRO_FEATURES)" % f) - conflict_distro_features = d.getVar('CONFLICT_DISTRO_FEATURES', True) + conflict_distro_features = d.getVar('CONFLICT_DISTRO_FEATURES') if conflict_distro_features: conflict_distro_features = conflict_distro_features.split() for f in conflict_distro_features: diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass index 4168e43849..5e34441610 100644 --- a/meta/classes/distrodata.bbclass +++ b/meta/classes/distrodata.bbclass @@ -1,4 +1,3 @@ -include conf/distro/include/package_regex.inc include conf/distro/include/upstream_tracking.inc include conf/distro/include/distro_alias.inc include conf/distro/include/maintainers.inc @@ -26,75 +25,70 @@ addtask distrodata_np do_distrodata_np[nostamp] = "1" python do_distrodata_np() { localdata = bb.data.createCopy(d) - pn = d.getVar("PN", True) + pn = d.getVar("PN") bb.note("Package Name: %s" % pn) import oe.distro_check as dist_check - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') distro_check_dir = os.path.join(tmpdir, "distro_check") - datetime = localdata.getVar('DATETIME', True) - dist_check.update_distro_data(distro_check_dir, datetime) + datetime = localdata.getVar('DATETIME') + dist_check.update_distro_data(distro_check_dir, datetime, localdata) if pn.find("-native") != -1: pnstripped = pn.split("-native") bb.note("Native Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pn.find("-cross") != -1: pnstripped = pn.split("-cross") bb.note("cross Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pn.find("-crosssdk") != -1: pnstripped = pn.split("-crosssdk") bb.note("cross Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pn.startswith("nativesdk-"): pnstripped = pn.replace("nativesdk-", "") bb.note("NativeSDK Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES')) if pn.find("-initial") != -1: pnstripped = pn.split("-initial") bb.note("initial Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) """generate package information from .bb file""" - pname = localdata.getVar('PN', True) - pcurver = localdata.getVar('PV', True) - pdesc = localdata.getVar('DESCRIPTION', True) + pname = localdata.getVar('PN') + pcurver = localdata.getVar('PV') + pdesc = localdata.getVar('DESCRIPTION') if pdesc is not None: pdesc = pdesc.replace(',','') pdesc = pdesc.replace('\n','') - pgrp = localdata.getVar('SECTION', True) - plicense = localdata.getVar('LICENSE', True).replace(',','_') + pgrp = localdata.getVar('SECTION') + plicense = localdata.getVar('LICENSE').replace(',','_') - rstatus = localdata.getVar('RECIPE_COLOR', True) + rstatus = localdata.getVar('RECIPE_COLOR') if rstatus is not None: rstatus = rstatus.replace(',','') - pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION', True) + pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION') if pcurver == pupver: vermatch="1" else: vermatch="0" - noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON', True) + noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON') if noupdate_reason is None: noupdate="0" else: noupdate="1" noupdate_reason = noupdate_reason.replace(',','') - maintainer = localdata.getVar('RECIPE_MAINTAINER', True) - rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE', True) + maintainer = localdata.getVar('RECIPE_MAINTAINER') + rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE') result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) bb.note("DISTRO: %s,%s,%s,%s,%s,%s,%s,%s,%s\n" % \ @@ -104,85 +98,81 @@ python do_distrodata_np() { line = line + "," + i bb.note("%s\n" % line) } +do_distrodata_np[vardepsexclude] = "DATETIME" addtask distrodata do_distrodata[nostamp] = "1" python do_distrodata() { import csv - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) logfile = os.path.join(logpath, "distrodata.csv") import oe.distro_check as dist_check localdata = bb.data.createCopy(d) - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') distro_check_dir = os.path.join(tmpdir, "distro_check") - datetime = localdata.getVar('DATETIME', True) - dist_check.update_distro_data(distro_check_dir, datetime) + datetime = localdata.getVar('DATETIME') + dist_check.update_distro_data(distro_check_dir, datetime, localdata) - pn = d.getVar("PN", True) + pn = d.getVar("PN") bb.note("Package Name: %s" % pn) if pn.find("-native") != -1: pnstripped = pn.split("-native") bb.note("Native Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pn.startswith("nativesdk-"): pnstripped = pn.replace("nativesdk-", "") bb.note("NativeSDK Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES')) if pn.find("-cross") != -1: pnstripped = pn.split("-cross") bb.note("cross Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pn.find("-crosssdk") != -1: pnstripped = pn.split("-crosssdk") bb.note("cross Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pn.find("-initial") != -1: pnstripped = pn.split("-initial") bb.note("initial Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) """generate package information from .bb file""" - pname = localdata.getVar('PN', True) - pcurver = localdata.getVar('PV', True) - pdesc = localdata.getVar('DESCRIPTION', True) + pname = localdata.getVar('PN') + pcurver = localdata.getVar('PV') + pdesc = localdata.getVar('DESCRIPTION') if pdesc is not None: pdesc = pdesc.replace(',','') pdesc = pdesc.replace('\n','') - pgrp = localdata.getVar('SECTION', True) - plicense = localdata.getVar('LICENSE', True).replace(',','_') + pgrp = localdata.getVar('SECTION') + plicense = localdata.getVar('LICENSE').replace(',','_') - rstatus = localdata.getVar('RECIPE_COLOR', True) + rstatus = localdata.getVar('RECIPE_COLOR') if rstatus is not None: rstatus = rstatus.replace(',','') - pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION', True) + pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION') if pcurver == pupver: vermatch="1" else: vermatch="0" - noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON', True) + noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON') if noupdate_reason is None: noupdate="0" else: noupdate="1" noupdate_reason = noupdate_reason.replace(',','') - maintainer = localdata.getVar('RECIPE_MAINTAINER', True) - rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE', True) + maintainer = localdata.getVar('RECIPE_MAINTAINER') + rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE') # do the comparison result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) @@ -196,6 +186,7 @@ python do_distrodata() { f.close() bb.utils.unlockfile(lf) } +do_distrodata[vardepsexclude] = "DATETIME" addtask distrodataall after do_distrodata do_distrodataall[recrdeptask] = "do_distrodataall do_distrodata" @@ -271,59 +262,56 @@ python do_checkpkg() { from bb.fetch2 import FetchError, NoMethodError, decodeurl """first check whether a uri is provided""" - src_uri = d.getVar('SRC_URI', True) - if not src_uri: - return - uri_type, _, _, _, _, _ = decodeurl(src_uri) + src_uri = (d.getVar('SRC_URI') or '').split() + if src_uri: + uri_type, _, _, _, _, _ = decodeurl(src_uri[0]) + else: + uri_type = "none" """initialize log files.""" - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) logfile = os.path.join(logpath, "checkpkg.csv") """generate package information from .bb file""" - pname = d.getVar('PN', True) + pname = d.getVar('PN') if pname.find("-native") != -1: - if d.getVar('BBCLASSEXTEND', True): + if d.getVar('BBCLASSEXTEND'): return pnstripped = pname.split("-native") bb.note("Native Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pname.startswith("nativesdk-"): - if d.getVar('BBCLASSEXTEND', True): + if d.getVar('BBCLASSEXTEND'): return pnstripped = pname.replace("nativesdk-", "") bb.note("NativeSDK Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES')) if pname.find("-cross") != -1: pnstripped = pname.split("-cross") bb.note("cross Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) if pname.find("-initial") != -1: pnstripped = pname.split("-initial") bb.note("initial Split: %s" % pnstripped) - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) - - pdesc = localdata.getVar('DESCRIPTION', True) - pgrp = localdata.getVar('SECTION', True) - pversion = localdata.getVar('PV', True) - plicense = localdata.getVar('LICENSE', True) - psection = localdata.getVar('SECTION', True) - phome = localdata.getVar('HOMEPAGE', True) - prelease = localdata.getVar('PR', True) - pdepends = localdata.getVar('DEPENDS', True) - pbugtracker = localdata.getVar('BUGTRACKER', True) - ppe = localdata.getVar('PE', True) - psrcuri = localdata.getVar('SRC_URI', True) - maintainer = localdata.getVar('RECIPE_MAINTAINER', True) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + pdesc = localdata.getVar('DESCRIPTION') + pgrp = localdata.getVar('SECTION') + pversion = localdata.getVar('PV') + plicense = localdata.getVar('LICENSE') + psection = localdata.getVar('SECTION') + phome = localdata.getVar('HOMEPAGE') + prelease = localdata.getVar('PR') + pdepends = localdata.getVar('DEPENDS') + pbugtracker = localdata.getVar('BUGTRACKER') + ppe = localdata.getVar('PE') + psrcuri = localdata.getVar('SRC_URI') + maintainer = localdata.getVar('RECIPE_MAINTAINER') """ Get upstream version version """ pupver = "" @@ -354,10 +342,13 @@ python do_checkpkg() { elif cmp == 0: pstatus = "MATCH" - psrcuri = psrcuri.split()[0] + if psrcuri: + psrcuri = psrcuri.split()[0] + else: + psrcuri = "none" pdepends = "".join(pdepends.split("\t")) pdesc = "".join(pdesc.split("\t")) - no_upgr_reason = d.getVar('RECIPE_NO_UPDATE_REASON', True) + no_upgr_reason = d.getVar('RECIPE_NO_UPDATE_REASON') lf = bb.utils.lockfile("%s.lock" % logfile) with open(logfile, "a") as f: writer = csv.writer(f, delimiter='\t') @@ -387,6 +378,7 @@ python distro_check_eventhandler() { addtask distro_check do_distro_check[nostamp] = "1" +do_distro_check[vardepsexclude] += "DATETIME" python do_distro_check() { """checks if the package is present in other public Linux distros""" import oe.distro_check as dc @@ -395,14 +387,13 @@ python do_distro_check() { return localdata = bb.data.createCopy(d) - bb.data.update_data(localdata) - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') distro_check_dir = os.path.join(tmpdir, "distro_check") - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) result_file = os.path.join(logpath, "distrocheck.csv") - datetime = localdata.getVar('DATETIME', True) - dc.update_distro_data(distro_check_dir, datetime) + datetime = localdata.getVar('DATETIME') + dc.update_distro_data(distro_check_dir, datetime, localdata) # do the comparison result = dc.compare_in_distro_packages_list(distro_check_dir, d) @@ -444,12 +435,12 @@ do_checklicense[nostamp] = "1" python do_checklicense() { import csv import shutil - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) - pn = d.getVar('PN', True) + pn = d.getVar('PN') logfile = os.path.join(logpath, "missinglicense.csv") - generic_directory = d.getVar('COMMON_LICENSE_DIR', True) - license_types = d.getVar('LICENSE', True) + generic_directory = d.getVar('COMMON_LICENSE_DIR') + license_types = d.getVar('LICENSE') for license_type in ((license_types.replace('+', '').replace('|', '&') .replace('(', '').replace(')', '').replace(';', '') .replace(',', '').replace(" ", "").split("&"))): @@ -470,5 +461,3 @@ do_checklicenseall[nostamp] = "1" do_checklicenseall() { : } - - diff --git a/meta/classes/distutils-base.bbclass b/meta/classes/distutils-base.bbclass index aa18e8b292..9f398d7051 100644 --- a/meta/classes/distutils-base.bbclass +++ b/meta/classes/distutils-base.bbclass @@ -1,4 +1,4 @@ -DEPENDS += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES', True) == '')]}" +DEPENDS += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES') == '')]}" RDEPENDS_${PN} += "${@['', '${PYTHON_PN}-core']['${CLASSOVERRIDE}' == 'class-target']}" inherit distutils-common-base pythonnative diff --git a/meta/classes/distutils-common-base.bbclass b/meta/classes/distutils-common-base.bbclass index 427275b406..824a1b68b1 100644 --- a/meta/classes/distutils-common-base.bbclass +++ b/meta/classes/distutils-common-base.bbclass @@ -1,12 +1,6 @@ -inherit python-dir - -EXTRA_OEMAKE = "" - export STAGING_INCDIR export STAGING_LIBDIR -PACKAGES = "${PN}-staticdev ${PN}-dev ${PN}-dbg ${PN}-doc ${PN}" - FILES_${PN} += "${libdir}/* ${libdir}/${PYTHON_DIR}/*" FILES_${PN}-staticdev += "\ @@ -17,8 +11,3 @@ FILES_${PN}-dev += "\ ${libdir}/pkgconfig \ ${PYTHON_SITEPACKAGES_DIR}/*.la \ " -FILES_${PN}-dbg += "\ - ${PYTHON_SITEPACKAGES_DIR}/.debug \ - ${PYTHON_SITEPACKAGES_DIR}/*/.debug \ - ${PYTHON_SITEPACKAGES_DIR}/*/*/.debug \ -" diff --git a/meta/classes/distutils-native-base.bbclass b/meta/classes/distutils-native-base.bbclass deleted file mode 100644 index 509cb9551a..0000000000 --- a/meta/classes/distutils-native-base.bbclass +++ /dev/null @@ -1,3 +0,0 @@ -inherit distutils-common-base - -DEPENDS += "${@["${PYTHON_PN}-native", ""][(d.getVar('PACKAGES', True) == '')]}" diff --git a/meta/classes/distutils-tools.bbclass b/meta/classes/distutils-tools.bbclass index f43450e56f..6f2880ea01 100644 --- a/meta/classes/distutils-tools.bbclass +++ b/meta/classes/distutils-tools.bbclass @@ -8,16 +8,14 @@ DISTUTILS_INSTALL_ARGS ?= "--prefix=${D}/${prefix} \ distutils_do_compile() { STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py build ${DISTUTILS_BUILD_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py build_ext execution failed." + bbfatal_log "${PYTHON_PN} setup.py build_ext execution failed." } distutils_stage_headers() { install -d ${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install_headers ${DISTUTILS_STAGE_HEADERS_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install_headers execution failed." + bbfatal_log "${PYTHON_PN} setup.py install_headers execution failed." } distutils_stage_all() { @@ -25,9 +23,8 @@ distutils_stage_all() { STAGING_LIBDIR=${STAGING_LIBDIR} \ install -d ${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} PYTHONPATH=${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install ${DISTUTILS_STAGE_ALL_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install (stage) execution failed." + bbfatal_log "${PYTHON_PN} setup.py install (stage) execution failed." } distutils_do_install() { @@ -37,9 +34,8 @@ distutils_do_install() { STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ PYTHONPATH=${D}/${PYTHON_SITEPACKAGES_DIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install --install-lib=${D}/${PYTHON_SITEPACKAGES_DIR} ${DISTUTILS_INSTALL_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install execution failed." + bbfatal_log "${PYTHON_PN} setup.py install execution failed." echo "Step 3 of ${PN} Install ..." # support filenames with *spaces* @@ -67,7 +63,7 @@ distutils_do_install() { # # FIXME: Bandaid against wrong datadir computation # - if test -e ${D}${datadir}/share; then + if [ -e ${D}${datadir}/share ]; then mv -f ${D}${datadir}/share/* ${D}${datadir}/ fi } diff --git a/meta/classes/distutils.bbclass b/meta/classes/distutils.bbclass index 2498685d66..1930c35292 100644 --- a/meta/classes/distutils.bbclass +++ b/meta/classes/distutils.bbclass @@ -10,16 +10,14 @@ DISTUTILS_INSTALL_ARGS ?= "--prefix=${D}/${prefix} \ distutils_do_compile() { STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py build ${DISTUTILS_BUILD_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py build execution failed." + bbfatal_log "${PYTHON_PN} setup.py build execution failed." } distutils_stage_headers() { install -d ${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install_headers ${DISTUTILS_STAGE_HEADERS_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install_headers execution failed." + bbfatal_log "${PYTHON_PN} setup.py install_headers execution failed." } distutils_stage_all() { @@ -27,9 +25,8 @@ distutils_stage_all() { STAGING_LIBDIR=${STAGING_LIBDIR} \ install -d ${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} PYTHONPATH=${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install ${DISTUTILS_STAGE_ALL_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install (stage) execution failed." + bbfatal_log "${PYTHON_PN} setup.py install (stage) execution failed." } distutils_do_install() { @@ -37,9 +34,8 @@ distutils_do_install() { STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ PYTHONPATH=${D}${PYTHON_SITEPACKAGES_DIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install --install-lib=${D}/${PYTHON_SITEPACKAGES_DIR} ${DISTUTILS_INSTALL_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install execution failed." + bbfatal_log "${PYTHON_PN} setup.py install execution failed." # support filenames with *spaces* # only modify file if it contains path and recompile it @@ -48,16 +44,16 @@ distutils_do_install() { if test -e ${D}${bindir} ; then for i in ${D}${bindir}/* ; do \ if [ ${PN} != "${BPN}-native" ]; then - sed -i -e s:${STAGING_BINDIR_NATIVE}/python-native/python:${bindir}/env\ python:g $i + sed -i -e s:${STAGING_BINDIR_NATIVE}/python-native/python:${USRBINPATH}/env\ python:g $i fi sed -i -e s:${STAGING_BINDIR_NATIVE}:${bindir}:g $i done fi - if test -e ${D}${sbindir}; then + if [ -e ${D}${sbindir} ]; then for i in ${D}${sbindir}/* ; do \ if [ ${PN} != "${BPN}-native" ]; then - sed -i -e s:${STAGING_BINDIR_NATIVE}/python-native/python:${bindir}/env\ python:g $i + sed -i -e s:${STAGING_BINDIR_NATIVE}/python-native/python:${USRBINPATH}/env\ python:g $i fi sed -i -e s:${STAGING_BINDIR_NATIVE}:${bindir}:g $i done @@ -69,10 +65,16 @@ distutils_do_install() { # # FIXME: Bandaid against wrong datadir computation # - if test -e ${D}${datadir}/share; then + if [ -e ${D}${datadir}/share ]; then mv -f ${D}${datadir}/share/* ${D}${datadir}/ rmdir ${D}${datadir}/share fi + + # Fix backport modules + if [ -e ${STAGING_LIBDIR}/${PYTHON_DIR}/site-packages/backports/__init__.py ] && [ -e ${D}${PYTHON_SITEPACKAGES_DIR}/backports/__init__.py ]; then + rm ${D}${PYTHON_SITEPACKAGES_DIR}/backports/__init__.py; + rm ${D}${PYTHON_SITEPACKAGES_DIR}/backports/__init__.pyc; + fi } EXPORT_FUNCTIONS do_compile do_install diff --git a/meta/classes/distutils3-base.bbclass b/meta/classes/distutils3-base.bbclass index af3aa006fc..7dbf07ac4b 100644 --- a/meta/classes/distutils3-base.bbclass +++ b/meta/classes/distutils3-base.bbclass @@ -1,8 +1,5 @@ -DEPENDS += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES', True) == '')]}" +DEPENDS += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES') == '')]}" RDEPENDS_${PN} += "${@['', '${PYTHON_PN}-core']['${CLASSOVERRIDE}' == 'class-target']}" -PYTHON_BASEVERSION = "3.4" -PYTHON_ABI = "m" - inherit distutils-common-base python3native diff --git a/meta/classes/distutils3-native-base.bbclass b/meta/classes/distutils3-native-base.bbclass deleted file mode 100644 index 11171014f8..0000000000 --- a/meta/classes/distutils3-native-base.bbclass +++ /dev/null @@ -1,4 +0,0 @@ -PYTHON_BASEVERSION = "3.4" -PYTHON_ABI = "m" - -inherit distutils-native-base diff --git a/meta/classes/distutils3.bbclass b/meta/classes/distutils3.bbclass index e909ef41b6..6c30306882 100644 --- a/meta/classes/distutils3.bbclass +++ b/meta/classes/distutils3.bbclass @@ -9,73 +9,52 @@ DISTUTILS_INSTALL_ARGS ?= "--prefix=${D}/${prefix} \ --install-data=${D}/${datadir}" distutils3_do_compile() { - if [ ${BUILD_SYS} != ${HOST_SYS} ]; then - SYS=${MACHINE} - else - SYS=${HOST_SYS} - fi STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py \ build ${DISTUTILS_BUILD_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py build_ext execution failed." + bbfatal_log "${PYTHON_PN} setup.py build_ext execution failed." } +distutils3_do_compile[vardepsexclude] = "MACHINE" distutils3_stage_headers() { install -d ${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} - if [ ${BUILD_SYS} != ${HOST_SYS} ]; then - SYS=${MACHINE} - else - SYS=${HOST_SYS} - fi - BUILD_SYS=${BUILD_SYS} HOST_SYS=${SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install_headers ${DISTUTILS_STAGE_HEADERS_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install_headers execution failed." + bbfatal_log "${PYTHON_PN} setup.py install_headers execution failed." } +distutils3_stage_headers[vardepsexclude] = "MACHINE" distutils3_stage_all() { - if [ ${BUILD_SYS} != ${HOST_SYS} ]; then - SYS=${MACHINE} - else - SYS=${HOST_SYS} - fi STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ install -d ${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} PYTHONPATH=${STAGING_DIR_HOST}${PYTHON_SITEPACKAGES_DIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install ${DISTUTILS_STAGE_ALL_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install (stage) execution failed." + bbfatal_log "${PYTHON_PN} setup.py install (stage) execution failed." } +distutils3_stage_all[vardepsexclude] = "MACHINE" distutils3_do_install() { install -d ${D}${PYTHON_SITEPACKAGES_DIR} - if [ ${BUILD_SYS} != ${HOST_SYS} ]; then - SYS=${MACHINE} - else - SYS=${HOST_SYS} - fi STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ PYTHONPATH=${D}${PYTHON_SITEPACKAGES_DIR} \ - BUILD_SYS=${BUILD_SYS} HOST_SYS=${SYS} \ ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} setup.py install --install-lib=${D}/${PYTHON_SITEPACKAGES_DIR} ${DISTUTILS_INSTALL_ARGS} || \ - bbfatal "${PYTHON_PN} setup.py install execution failed." + bbfatal_log "${PYTHON_PN} setup.py install execution failed." # support filenames with *spaces* find ${D} -name "*.py" -exec grep -q ${D} {} \; -exec sed -i -e s:${D}::g {} \; if test -e ${D}${bindir} ; then for i in ${D}${bindir}/* ; do \ - sed -i -e s:${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}:${bindir}/env\ ${PYTHON_PN}:g $i + sed -i -e s:${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}:${USRBINPATH}/env\ ${PYTHON_PN}:g $i sed -i -e s:${STAGING_BINDIR_NATIVE}:${bindir}:g $i done fi if test -e ${D}${sbindir}; then for i in ${D}${sbindir}/* ; do \ - sed -i -e s:${STAGING_BINDIR_NATIVE}/python-${PYTHON_PN}/${PYTHON_PN}:${bindir}/env\ ${PYTHON_PN}:g $i + sed -i -e s:${STAGING_BINDIR_NATIVE}/python-${PYTHON_PN}/${PYTHON_PN}:${USRBINPATH}/env\ ${PYTHON_PN}:g $i sed -i -e s:${STAGING_BINDIR_NATIVE}:${bindir}:g $i done fi @@ -85,11 +64,12 @@ distutils3_do_install() { # # FIXME: Bandaid against wrong datadir computation # - if test -e ${D}${datadir}/share; then + if [ -e ${D}${datadir}/share ]; then mv -f ${D}${datadir}/share/* ${D}${datadir}/ rmdir ${D}${datadir}/share fi } +distutils3_do_install[vardepsexclude] = "MACHINE" EXPORT_FUNCTIONS do_compile do_install diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index 8f7f479293..d64af6a9c9 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass @@ -4,7 +4,7 @@ # Copyright (C) 2009 Chris Larson <clarson@kergoth.com> # Released under the MIT license (see COPYING.MIT for the terms) # -# externalsrc.bbclass enables use of an existing source tree, usually external to +# externalsrc.bbclass enables use of an existing source tree, usually external to # the build system to build a piece of software rather than the usual fetch/unpack/patch # process. # @@ -25,22 +25,44 @@ # SRCTREECOVEREDTASKS ?= "do_patch do_unpack do_fetch" +EXTERNALSRC_SYMLINKS ?= "oe-workdir:${WORKDIR} oe-logs:${T}" python () { - externalsrc = d.getVar('EXTERNALSRC', True) + externalsrc = d.getVar('EXTERNALSRC') + + # If this is the base recipe and EXTERNALSRC is set for it or any of its + # derivatives, then enable BB_DONT_CACHE to force the recipe to always be + # re-parsed so that the file-checksums function for do_compile is run every + # time. + bpn = d.getVar('BPN') + if bpn == d.getVar('PN'): + classextend = (d.getVar('BBCLASSEXTEND') or '').split() + if (externalsrc or + ('native' in classextend and + d.getVar('EXTERNALSRC_pn-%s-native' % bpn)) or + ('nativesdk' in classextend and + d.getVar('EXTERNALSRC_pn-nativesdk-%s' % bpn)) or + ('cross' in classextend and + d.getVar('EXTERNALSRC_pn-%s-cross' % bpn))): + d.setVar('BB_DONT_CACHE', '1') + if externalsrc: d.setVar('S', externalsrc) - externalsrcbuild = d.getVar('EXTERNALSRC_BUILD', True) + externalsrcbuild = d.getVar('EXTERNALSRC_BUILD') if externalsrcbuild: d.setVar('B', externalsrcbuild) else: d.setVar('B', '${WORKDIR}/${BPN}-${PV}/') - srcuri = (d.getVar('SRC_URI', True) or '').split() local_srcuri = [] - for uri in srcuri: - if uri.startswith('file://'): - local_srcuri.append(uri) + fetch = bb.fetch2.Fetch((d.getVar('SRC_URI') or '').split(), d) + for url in fetch.urls: + url_data = fetch.ud[url] + parm = url_data.parm + if (url_data.type == 'file' or + 'type' in parm and parm['type'] == 'kmeta'): + local_srcuri.append(url) + d.setVar('SRC_URI', ' '.join(local_srcuri)) if '{SRCPV}' in d.getVar('PV', False): @@ -58,35 +80,144 @@ python () { d.appendVarFlag(task, "lockfiles", " ${S}/singletask.lock") # We do not want our source to be wiped out, ever (kernel.bbclass does this for do_clean) - cleandirs = d.getVarFlag(task, 'cleandirs', False) - if cleandirs: - cleandirs = cleandirs.split() - setvalue = False - if '${S}' in cleandirs: - cleandirs.remove('${S}') + cleandirs = (d.getVarFlag(task, 'cleandirs', False) or '').split() + setvalue = False + for cleandir in cleandirs[:]: + if d.expand(cleandir) == externalsrc: + cleandirs.remove(cleandir) setvalue = True - if externalsrcbuild == externalsrc and '${B}' in cleandirs: - cleandirs.remove('${B}') - setvalue = True - if setvalue: - d.setVarFlag(task, 'cleandirs', ' '.join(cleandirs)) + if setvalue: + d.setVarFlag(task, 'cleandirs', ' '.join(cleandirs)) fetch_tasks = ['do_fetch', 'do_unpack'] # If we deltask do_patch, there's no dependency to ensure do_unpack gets run, so add one - d.appendVarFlag('do_configure', 'deps', ['do_unpack']) + # Note that we cannot use d.appendVarFlag() here because deps is expected to be a list object, not a string + d.setVarFlag('do_configure', 'deps', (d.getVarFlag('do_configure', 'deps', False) or []) + ['do_unpack']) - for task in d.getVar("SRCTREECOVEREDTASKS", True).split(): + for task in d.getVar("SRCTREECOVEREDTASKS").split(): if local_srcuri and task in fetch_tasks: continue bb.build.deltask(task, d) d.prependVarFlag('do_compile', 'prefuncs', "externalsrc_compile_prefunc ") + d.prependVarFlag('do_configure', 'prefuncs', "externalsrc_configure_prefunc ") + + d.setVarFlag('do_compile', 'file-checksums', '${@srctree_hash_files(d)}') + d.setVarFlag('do_configure', 'file-checksums', '${@srctree_configure_hash_files(d)}') + + # We don't want the workdir to go away + d.appendVar('RM_WORK_EXCLUDE', ' ' + d.getVar('PN')) + + bb.build.addtask('do_buildclean', + 'do_clean' if d.getVar('S') == d.getVar('B') else None, + None, d) + + # If B=S the same builddir is used even for different architectures. + # Thus, use a shared CONFIGURESTAMPFILE and STAMP directory so that + # change of do_configure task hash is correctly detected and stamps are + # invalidated if e.g. MACHINE changes. + if d.getVar('S') == d.getVar('B'): + configstamp = '${TMPDIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}/configure.sstate' + d.setVar('CONFIGURESTAMPFILE', configstamp) + d.setVar('STAMP', '${STAMPS_DIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}') + d.setVar('STAMPCLEAN', '${STAMPS_DIR}/work-shared/${PN}/*-*') +} - # Ensure compilation happens every time - d.setVarFlag('do_compile', 'nostamp', '1') +python externalsrc_configure_prefunc() { + s_dir = d.getVar('S') + # Create desired symlinks + symlinks = (d.getVar('EXTERNALSRC_SYMLINKS') or '').split() + newlinks = [] + for symlink in symlinks: + symsplit = symlink.split(':', 1) + lnkfile = os.path.join(s_dir, symsplit[0]) + target = d.expand(symsplit[1]) + if len(symsplit) > 1: + if os.path.islink(lnkfile): + # Link already exists, leave it if it points to the right location already + if os.readlink(lnkfile) == target: + continue + os.unlink(lnkfile) + elif os.path.exists(lnkfile): + # File/dir exists with same name as link, just leave it alone + continue + os.symlink(target, lnkfile) + newlinks.append(symsplit[0]) + # Hide the symlinks from git + try: + git_exclude_file = os.path.join(s_dir, '.git/info/exclude') + if os.path.exists(git_exclude_file): + with open(git_exclude_file, 'r+') as efile: + elines = efile.readlines() + for link in newlinks: + if link in elines or '/'+link in elines: + continue + efile.write('/' + link + '\n') + except IOError as ioe: + bb.note('Failed to hide EXTERNALSRC_SYMLINKS from git') } python externalsrc_compile_prefunc() { # Make it obvious that this is happening, since forgetting about it could lead to much confusion - bb.warn('Compiling %s from external source %s' % (d.getVar('PN', True), d.getVar('EXTERNALSRC', True))) + bb.plain('NOTE: %s: compiling from external source tree %s' % (d.getVar('PN'), d.getVar('EXTERNALSRC'))) +} + +do_buildclean[dirs] = "${S} ${B}" +do_buildclean[nostamp] = "1" +do_buildclean[doc] = "Call 'make clean' or equivalent in ${B}" +externalsrc_do_buildclean() { + if [ -e Makefile -o -e makefile -o -e GNUmakefile ]; then + oe_runmake clean || die "make failed" + else + bbnote "nothing to do - no makefile found" + fi } + +def srctree_hash_files(d, srcdir=None): + import shutil + import subprocess + import tempfile + + s_dir = srcdir or d.getVar('EXTERNALSRC') + git_dir = os.path.join(s_dir, '.git') + oe_hash_file = os.path.join(git_dir, 'oe-devtool-tree-sha1') + + ret = " " + if os.path.exists(git_dir): + with tempfile.NamedTemporaryFile(dir=git_dir, prefix='oe-devtool-index') as tmp_index: + # Clone index + shutil.copy2(os.path.join(git_dir, 'index'), tmp_index.name) + # Update our custom index + env = os.environ.copy() + env['GIT_INDEX_FILE'] = tmp_index.name + subprocess.check_output(['git', 'add', '-A', '.'], cwd=s_dir, env=env) + sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8") + with open(oe_hash_file, 'w') as fobj: + fobj.write(sha1) + ret = oe_hash_file + ':True' + else: + ret = s_dir + '/*:True' + return ret + +def srctree_configure_hash_files(d): + """ + Get the list of files that should trigger do_configure to re-execute, + based on the value of CONFIGURE_FILES + """ + in_files = (d.getVar('CONFIGURE_FILES') or '').split() + out_items = [] + search_files = [] + for entry in in_files: + if entry.startswith('/'): + out_items.append('%s:%s' % (entry, os.path.exists(entry))) + else: + search_files.append(entry) + if search_files: + s_dir = d.getVar('EXTERNALSRC') + for root, _, files in os.walk(s_dir): + for f in files: + if f in search_files: + out_items.append('%s:True' % os.path.join(root, f)) + return ' '.join(out_items) + +EXPORT_FUNCTIONS do_buildclean diff --git a/meta/classes/extrausers.bbclass b/meta/classes/extrausers.bbclass index faf57b108e..7709407b69 100644 --- a/meta/classes/extrausers.bbclass +++ b/meta/classes/extrausers.bbclass @@ -15,7 +15,7 @@ inherit useradd_base -IMAGE_INSTALL_append = " ${@['', 'base-passwd shadow'][bool(d.getVar('EXTRA_USERS_PARAMS', True))]}" +PACKAGE_INSTALL_append = " ${@['', 'base-passwd shadow'][bool(d.getVar('EXTRA_USERS_PARAMS'))]}" # Image level user / group settings ROOTFS_POSTPROCESS_COMMAND_append = " set_user_group;" @@ -33,22 +33,22 @@ set_user_group () { # this setting is actually a serial process. So we only retry once. case $cmd in useradd) - perform_useradd "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" 1 + perform_useradd "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" ;; groupadd) - perform_groupadd "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" 1 + perform_groupadd "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" ;; userdel) - perform_userdel "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" 1 + perform_userdel "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" ;; groupdel) - perform_groupdel "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" 1 + perform_groupdel "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" ;; usermod) - perform_usermod "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" 1 + perform_usermod "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" ;; groupmod) - perform_groupmod "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" 1 + perform_groupmod "${IMAGE_ROOTFS}" "-R ${IMAGE_ROOTFS} $opts" ;; *) bbfatal "Invalid command in EXTRA_USERS_PARAMS: $cmd" @@ -63,3 +63,7 @@ set_user_group () { remaining=`echo $remaining | cut -d ';' -f2-` done } + +USERADDEXTENSION ?= "" + +inherit ${USERADDEXTENSION} diff --git a/meta/classes/fontcache.bbclass b/meta/classes/fontcache.bbclass index d122387ffd..e76331131e 100644 --- a/meta/classes/fontcache.bbclass +++ b/meta/classes/fontcache.bbclass @@ -3,42 +3,53 @@ # packages. # -DEPENDS += "qemu-native" +PACKAGE_WRITE_DEPS += "qemu-native" inherit qemu FONT_PACKAGES ??= "${PN}" FONT_EXTRA_RDEPENDS ?= "fontconfig-utils" FONTCONFIG_CACHE_DIR ?= "${localstatedir}/cache/fontconfig" +FONTCONFIG_CACHE_PARAMS ?= "-v" +# You can change this to e.g. FC_DEBUG=16 to debug fc-cache issues, +# something has to be set, because qemuwrapper is using this variable after -E +# multiple variables aren't allowed because for qemu they are separated +# by comma and in -n "$D" case they should be separated by space +FONTCONFIG_CACHE_ENV ?= "FC_DEBUG=1" fontcache_common() { -if [ "x$D" != "x" ] ; then - $INTERCEPT_DIR/postinst_intercept update_font_cache ${PKG} mlprefix=${MLPREFIX} bindir=${bindir} \ - libdir=${libdir} base_libdir=${base_libdir} fontconfigcachedir=${FONTCONFIG_CACHE_DIR} +if [ -n "$D" ] ; then + $INTERCEPT_DIR/postinst_intercept update_font_cache ${PKG} mlprefix=${MLPREFIX} \ + 'bindir="${bindir}"' \ + 'libdir="${libdir}"' \ + 'base_libdir="${base_libdir}"' \ + 'fontconfigcachedir="${FONTCONFIG_CACHE_DIR}"' \ + 'fontconfigcacheparams="${FONTCONFIG_CACHE_PARAMS}"' \ + 'fontconfigcacheenv="${FONTCONFIG_CACHE_ENV}"' else - fc-cache + ${FONTCONFIG_CACHE_ENV} fc-cache ${FONTCONFIG_CACHE_PARAMS} fi } python () { - font_pkgs = d.getVar('FONT_PACKAGES', True).split() - deps = d.getVar("FONT_EXTRA_RDEPENDS", True) + font_pkgs = d.getVar('FONT_PACKAGES').split() + deps = d.getVar("FONT_EXTRA_RDEPENDS") for pkg in font_pkgs: if deps: d.appendVar('RDEPENDS_' + pkg, ' '+deps) } python add_fontcache_postinsts() { - for pkg in d.getVar('FONT_PACKAGES', True).split(): + for pkg in d.getVar('FONT_PACKAGES').split(): bb.note("adding fonts postinst and postrm scripts to %s" % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True) + postinst = d.getVar('pkg_postinst_%s' % pkg) or d.getVar('pkg_postinst') if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('fontcache_common', True) + postinst += d.getVar('fontcache_common') d.setVar('pkg_postinst_%s' % pkg, postinst) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True) + postrm = d.getVar('pkg_postrm_%s' % pkg) or d.getVar('pkg_postrm') if not postrm: postrm = '#!/bin/sh\n' - postrm += d.getVar('fontcache_common', True) + postrm += d.getVar('fontcache_common') d.setVar('pkg_postrm_%s' % pkg, postrm) } diff --git a/meta/classes/fs-uuid.bbclass b/meta/classes/fs-uuid.bbclass new file mode 100644 index 0000000000..9b53dfba7a --- /dev/null +++ b/meta/classes/fs-uuid.bbclass @@ -0,0 +1,24 @@ +# Extract UUID from ${ROOTFS}, which must have been built +# by the time that this function gets called. Only works +# on ext file systems and depends on tune2fs. +def get_rootfs_uuid(d): + import subprocess + rootfs = d.getVar('ROOTFS') + output = subprocess.check_output(['tune2fs', '-l', rootfs]) + for line in output.split('\n'): + if line.startswith('Filesystem UUID:'): + uuid = line.split()[-1] + bb.note('UUID of %s: %s' % (rootfs, uuid)) + return uuid + bb.fatal('Could not determine filesystem UUID of %s' % rootfs) + +# Replace the special <<uuid-of-rootfs>> inside a string (like the +# root= APPEND string in a syslinux.cfg or systemd-boot entry) with the +# actual UUID of the rootfs. Does nothing if the special string +# is not used. +def replace_rootfs_uuid(d, string): + UUID_PLACEHOLDER = '<<uuid-of-rootfs>>' + if UUID_PLACEHOLDER in string: + uuid = get_rootfs_uuid(d) + string = string.replace(UUID_PLACEHOLDER, uuid) + return string diff --git a/meta/classes/gconf.bbclass b/meta/classes/gconf.bbclass index d7afa7282f..4e0ee2e7d5 100644 --- a/meta/classes/gconf.bbclass +++ b/meta/classes/gconf.bbclass @@ -1,4 +1,5 @@ -DEPENDS += "gconf gconf-native" +DEPENDS += "gconf" +PACKAGE_WRITE_DEPS += "gconf-native" # These are for when gconftool is used natively and the prefix isn't necessarily # the sysroot. TODO: replicate the postinst logic for -native packages going @@ -42,8 +43,8 @@ done python populate_packages_append () { import re - packages = d.getVar('PACKAGES', True).split() - pkgdest = d.getVar('PKGDEST', True) + packages = d.getVar('PACKAGES').split() + pkgdest = d.getVar('PKGDEST') for pkg in packages: schema_dir = '%s/%s/etc/gconf/schemas' % (pkgdest, pkg) @@ -56,15 +57,15 @@ python populate_packages_append () { if schemas != []: bb.note("adding gconf postinst and prerm scripts to %s" % pkg) d.setVar('SCHEMA_FILES', " ".join(schemas)) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('gconf_postinst', True) + postinst += d.getVar('gconf_postinst') d.setVar('pkg_postinst_%s' % pkg, postinst) - prerm = d.getVar('pkg_prerm_%s' % pkg, True) + prerm = d.getVar('pkg_prerm_%s' % pkg) if not prerm: prerm = '#!/bin/sh\n' - prerm += d.getVar('gconf_prerm', True) + prerm += d.getVar('gconf_prerm') d.setVar('pkg_prerm_%s' % pkg, prerm) d.appendVar("RDEPENDS_%s" % pkg, ' ' + d.getVar('MLPREFIX', False) + 'gconf') } diff --git a/meta/classes/gettext.bbclass b/meta/classes/gettext.bbclass index 03b89b2455..0be14246bf 100644 --- a/meta/classes/gettext.bbclass +++ b/meta/classes/gettext.bbclass @@ -1,15 +1,15 @@ def gettext_dependencies(d): - if d.getVar('INHIBIT_DEFAULT_DEPS', True) and not oe.utils.inherits(d, 'cross-canadian'): + if d.getVar('INHIBIT_DEFAULT_DEPS') and not oe.utils.inherits(d, 'cross-canadian'): return "" - if d.getVar('USE_NLS', True) == 'no': + if d.getVar('USE_NLS') == 'no': return "gettext-minimal-native" return d.getVar('DEPENDS_GETTEXT', False) def gettext_oeconf(d): - if d.getVar('USE_NLS', True) == 'no': + if d.getVar('USE_NLS') == 'no': return '--disable-nls' # Remove the NLS bits if USE_NLS is no or INHIBIT_DEFAULT_DEPS is set - if d.getVar('INHIBIT_DEFAULT_DEPS', True) and not oe.utils.inherits(d, 'cross-canadian'): + if d.getVar('INHIBIT_DEFAULT_DEPS') and not oe.utils.inherits(d, 'cross-canadian'): return '--disable-nls' return "--enable-nls" diff --git a/meta/classes/gio-module-cache.bbclass b/meta/classes/gio-module-cache.bbclass new file mode 100644 index 0000000000..a8190b7b89 --- /dev/null +++ b/meta/classes/gio-module-cache.bbclass @@ -0,0 +1,37 @@ +PACKAGE_WRITE_DEPS += "qemu-native" +inherit qemu + +GIO_MODULE_PACKAGES ??= "${PN}" + +gio_module_cache_common() { +if [ "x$D" != "x" ]; then + $INTERCEPT_DIR/postinst_intercept update_gio_module_cache ${PKG} \ + mlprefix=${MLPREFIX} \ + binprefix=${MLPREFIX} \ + libdir=${libdir} \ + base_libdir=${base_libdir} \ + bindir=${bindir} +else + ${libexecdir}/${MLPREFIX}gio-querymodules ${libdir}/gio/modules/ +fi +} + +python populate_packages_append () { + packages = d.getVar('GIO_MODULE_PACKAGES').split() + + for pkg in packages: + bb.note("adding gio-module-cache postinst and postrm scripts to %s" % pkg) + + postinst = d.getVar('pkg_postinst_%s' % pkg) + if not postinst: + postinst = '#!/bin/sh\n' + postinst += d.getVar('gio_module_cache_common') + d.setVar('pkg_postinst_%s' % pkg, postinst) + + postrm = d.getVar('pkg_postrm_%s' % pkg) + if not postrm: + postrm = '#!/bin/sh\n' + postrm += d.getVar('gio_module_cache_common') + d.setVar('pkg_postrm_%s' % pkg, postrm) +} + diff --git a/meta/classes/gnomebase.bbclass b/meta/classes/gnomebase.bbclass index d22ba5a5cc..54aa45f174 100644 --- a/meta/classes/gnomebase.bbclass +++ b/meta/classes/gnomebase.bbclass @@ -6,8 +6,6 @@ SECTION ?= "x11/gnome" GNOMEBN ?= "${BPN}" SRC_URI = "${GNOME_MIRROR}/${GNOMEBN}/${@gnome_verdir("${PV}")}/${GNOMEBN}-${PV}.tar.${GNOME_COMPRESS_TYPE};name=archive" -DEPENDS += "gnome-common-native" - FILES_${PN} += "${datadir}/application-registry \ ${datadir}/mime-info \ ${datadir}/mime/packages \ @@ -27,7 +25,3 @@ do_install_append() { rm -rf ${D}${localstatedir}/scrollkeeper/* rm -f ${D}${datadir}/applications/*.cache } - -EXTRA_OECONF += "--disable-introspection" - -UNKNOWN_CONFIGURE_WHITELIST += "--disable-introspection" diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass new file mode 100644 index 0000000000..85f71a2e9a --- /dev/null +++ b/meta/classes/go.bbclass @@ -0,0 +1,77 @@ +inherit goarch + +# x32 ABI is not supported on go compiler so far +COMPATIBLE_HOST_linux-gnux32 = "null" +# ppc32 is not supported in go compilers +COMPATIBLE_HOST_powerpc = "null" + +GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" +GOROOT = "${STAGING_LIBDIR_NATIVE}/${TARGET_SYS}/go" +GOBIN_FINAL_class-native = "${GOROOT_FINAL}/bin" +GOBIN_FINAL = "${GOROOT_FINAL}/bin/${GOOS}_${GOARCH}" + +export GOOS = "${TARGET_GOOS}" +export GOARCH = "${TARGET_GOARCH}" +export GOARM = "${TARGET_GOARM}" +export CGO_ENABLED = "1" +export GOROOT +export GOROOT_FINAL = "${libdir}/${TARGET_SYS}/go" +export GOBIN_FINAL +export GOPKG_FINAL = "${GOROOT_FINAL}/pkg/${GOOS}_${GOARCH}" +export GOSRC_FINAL = "${GOROOT_FINAL}/src" +export GO_GCFLAGS = "${TARGET_CFLAGS}" +export GO_LDFLAGS = "${TARGET_LDFLAGS}" +export CGO_CFLAGS = "${TARGET_CC_ARCH}${TOOLCHAIN_OPTIONS} ${TARGET_CFLAGS}" +export CGO_CPPFLAGS = "${TARGET_CPPFLAGS}" +export CGO_CXXFLAGS = "${TARGET_CC_ARCH}${TOOLCHAIN_OPTIONS} ${TARGET_CXXFLAGS}" +export CGO_LDFLAGS = "${TARGET_CC_ARCH}${TOOLCHAIN_OPTIONS} ${TARGET_LDFLAGS}" + +DEPENDS += "go-cross-${TARGET_ARCH}" +DEPENDS_class-native += "go-native" + +FILES_${PN}-staticdev += "${GOSRC_FINAL}/${GO_IMPORT}" +FILES_${PN}-staticdev += "${GOPKG_FINAL}/${GO_IMPORT}*" + +GO_INSTALL ?= "${GO_IMPORT}/..." + +do_go_compile() { + GOPATH=${S}:${STAGING_LIBDIR}/${TARGET_SYS}/go go env + if [ -n "${GO_INSTALL}" ]; then + GOPATH=${S}:${STAGING_LIBDIR}/${TARGET_SYS}/go go install -v ${GO_INSTALL} + fi +} + +do_go_install() { + rm -rf ${WORKDIR}/staging + install -d ${WORKDIR}/staging${GOROOT_FINAL} ${D}${GOROOT_FINAL} + tar -C ${S} -cf - . | tar -C ${WORKDIR}/staging${GOROOT_FINAL} -xpvf - + + find ${WORKDIR}/staging${GOROOT_FINAL} \( \ + -name \*.indirectionsymlink -o \ + -name .git\* -o \ + -name .hg -o \ + -name .svn -o \ + -name .pc\* -o \ + -name patches\* \ + \) -print0 | \ + xargs -r0 rm -rf + + tar -C ${WORKDIR}/staging${GOROOT_FINAL} -cf - . | \ + tar -C ${D}${GOROOT_FINAL} -xpvf - + + chown -R root:root "${D}${GOROOT_FINAL}" + + if [ -e "${D}${GOBIN_FINAL}" ]; then + install -d -m 0755 "${D}${bindir}" + find "${D}${GOBIN_FINAL}" ! -type d -print0 | xargs -r0 mv --target-directory="${D}${bindir}" + rmdir -p "${D}${GOBIN_FINAL}" || true + fi +} + +do_compile() { + do_go_compile +} + +do_install() { + do_go_install +} diff --git a/meta/classes/goarch.bbclass b/meta/classes/goarch.bbclass new file mode 100644 index 0000000000..4a5b2ec787 --- /dev/null +++ b/meta/classes/goarch.bbclass @@ -0,0 +1,50 @@ +BUILD_GOOS = "${@go_map_os(d.getVar('BUILD_OS', True), d)}" +BUILD_GOARCH = "${@go_map_arch(d.getVar('BUILD_ARCH', True), d)}" +BUILD_GOTUPLE = "${BUILD_GOOS}_${BUILD_GOARCH}" +HOST_GOOS = "${@go_map_os(d.getVar('HOST_OS', True), d)}" +HOST_GOARCH = "${@go_map_arch(d.getVar('HOST_ARCH', True), d)}" +HOST_GOARM = "${@go_map_arm(d.getVar('HOST_ARCH', True), d.getVar('TUNE_FEATURES', True), d)}" +HOST_GOTUPLE = "${HOST_GOOS}_${HOST_GOARCH}" +TARGET_GOOS = "${@go_map_os(d.getVar('TARGET_OS', True), d)}" +TARGET_GOARCH = "${@go_map_arch(d.getVar('TARGET_ARCH', True), d)}" +TARGET_GOARM = "${@go_map_arm(d.getVar('TARGET_ARCH', True), d.getVar('TUNE_FEATURES', True), d)}" +TARGET_GOTUPLE = "${TARGET_GOOS}_${TARGET_GOARCH}" +GO_BUILD_BINDIR = "${@['bin/${HOST_GOTUPLE}','bin'][d.getVar('BUILD_GOTUPLE',True) == d.getVar('HOST_GOTUPLE',True)]}" + +def go_map_arch(a, d): + import re + if re.match('i.86', a): + return '386' + elif a == 'x86_64': + return 'amd64' + elif re.match('arm.*', a): + return 'arm' + elif re.match('aarch64.*', a): + return 'arm64' + elif re.match('mips64el*', a): + return 'mips64le' + elif re.match('mips64*', a): + return 'mips64' + elif re.match('mipsel*', a): + return 'mipsle' + elif re.match('mips*', a): + return 'mips' + elif re.match('p(pc|owerpc)(64)', a): + return 'ppc64' + elif re.match('p(pc|owerpc)(64el)', a): + return 'ppc64le' + else: + raise bb.parse.SkipPackage("Unsupported CPU architecture: %s" % a) + +def go_map_arm(a, f, d): + import re + if re.match('arm.*', a) and re.match('arm.*7.*', f): + return '7' + return '' + +def go_map_os(o, d): + if o.startswith('linux'): + return 'linux' + return o + + diff --git a/meta/classes/gobject-introspection-data.bbclass b/meta/classes/gobject-introspection-data.bbclass new file mode 100644 index 0000000000..2ef684626a --- /dev/null +++ b/meta/classes/gobject-introspection-data.bbclass @@ -0,0 +1,7 @@ +# This variable is set to True if gobject-introspection-data is in +# DISTRO_FEATURES and qemu-usermode is in MACHINE_FEATURES, and False otherwise. +# +# It should be used in recipes to determine whether introspection data should be built, +# so that qemu use can be avoided when necessary. +GI_DATA_ENABLED ?= "${@bb.utils.contains('DISTRO_FEATURES', 'gobject-introspection-data', \ + bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'True', 'False', d), 'False', d)}" diff --git a/meta/classes/gobject-introspection.bbclass b/meta/classes/gobject-introspection.bbclass new file mode 100644 index 0000000000..b6160b88b6 --- /dev/null +++ b/meta/classes/gobject-introspection.bbclass @@ -0,0 +1,43 @@ +# Inherit this class in recipes to enable building their introspection files + +# python3native is inherited to prevent introspection tools being run with +# host's python 3 (they need to be run with native python 3) +# +# This also sets up autoconf-based recipes to build introspection data (or not), +# depending on distro and machine features (see gobject-introspection-data class). +inherit python3native gobject-introspection-data +EXTRA_OECONF_prepend_class-target = "${@bb.utils.contains('GI_DATA_ENABLED', 'True', '--enable-introspection', '--disable-introspection', d)} " + +# When building native recipes, disable introspection, as it is not necessary, +# pulls in additional dependencies, and makes build times longer +EXTRA_OECONF_prepend_class-native = "--disable-introspection " +EXTRA_OECONF_prepend_class-nativesdk = "--disable-introspection " + +UNKNOWN_CONFIGURE_WHITELIST_append = " --enable-introspection --disable-introspection" + +# Generating introspection data depends on a combination of native and target +# introspection tools, and qemu to run the target tools. +DEPENDS_append_class-target = " gobject-introspection gobject-introspection-native qemu-native prelink-native" + +# Even though introspection is disabled on -native, gobject-introspection package is still +# needed for m4 macros. +DEPENDS_append_class-native = " gobject-introspection-native" +DEPENDS_append_class-nativesdk = " gobject-introspection-native" + +# This is used by introspection tools to find .gir includes +export XDG_DATA_DIRS = "${STAGING_DATADIR}" + +do_configure_prepend_class-target () { + # introspection.m4 pre-packaged with upstream tarballs does not yet + # have our fixes + mkdir -p ${S}/m4 + cp ${STAGING_DIR_TARGET}/${datadir}/aclocal/introspection.m4 ${S}/m4 +} + +# .typelib files are needed at runtime and so they go to the main package (so +# they'll be together with libraries they support). +FILES_${PN}_append = " ${libdir}/girepository-*/*.typelib" + +# .gir files go to dev package, as they're needed for developing (but not for +# running) things that depends on introspection. +FILES_${PN}-dev_append = " ${datadir}/gir-*/*.gir" diff --git a/meta/classes/grub-efi.bbclass b/meta/classes/grub-efi.bbclass index 6d58d48704..df7fe18a79 100644 --- a/meta/classes/grub-efi.bbclass +++ b/meta/classes/grub-efi.bbclass @@ -14,17 +14,24 @@ # ${APPEND} - an override list of append strings for each label # ${GRUB_OPTS} - additional options to add to the config, ';' delimited # (optional) # ${GRUB_TIMEOUT} - timeout before executing the deault label (optional) +# ${GRUB_ROOT} - grub's root device. do_bootimg[depends] += "${MLPREFIX}grub-efi:do_deploy" do_bootdirectdisk[depends] += "${MLPREFIX}grub-efi:do_deploy" GRUB_SERIAL ?= "console=ttyS0,115200" -GRUBCFG = "${S}/grub.cfg" +GRUB_CFG_VM = "${S}/grub_vm.cfg" +GRUB_CFG_LIVE = "${S}/grub_live.cfg" GRUB_TIMEOUT ?= "10" #FIXME: build this from the machine config GRUB_OPTS ?= "serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1" EFIDIR = "/EFI/BOOT" +GRUB_ROOT ?= "${ROOT}" +APPEND ?= "" + +# Need UUID utility code. +inherit fs-uuid efi_populate() { # DEST must be the root of the image so that EFIDIR is not @@ -33,13 +40,17 @@ efi_populate() { install -d ${DEST}${EFIDIR} - GRUB_IMAGE="bootia32.efi" + GRUB_IMAGE="grub-efi-bootia32.efi" + DEST_IMAGE="bootia32.efi" if [ "${TARGET_ARCH}" = "x86_64" ]; then - GRUB_IMAGE="bootx64.efi" + GRUB_IMAGE="grub-efi-bootx64.efi" + DEST_IMAGE="bootx64.efi" fi - install -m 0644 ${DEPLOY_DIR_IMAGE}/${GRUB_IMAGE} ${DEST}${EFIDIR} + install -m 0644 ${DEPLOY_DIR_IMAGE}/${GRUB_IMAGE} ${DEST}${EFIDIR}/${DEST_IMAGE} + EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g') + printf 'fs0:%s\%s\n' "$EFIPATH" "$DEST_IMAGE" >${DEST}/startup.nsh - install -m 0644 ${GRUBCFG} ${DEST}${EFIDIR}/grub.cfg + install -m 0644 ${GRUB_CFG} ${DEST}${EFIDIR}/grub.cfg } efi_iso_populate() { @@ -49,7 +60,8 @@ efi_iso_populate() { mkdir -p ${EFIIMGDIR}/${EFIDIR} cp $iso_dir/${EFIDIR}/* ${EFIIMGDIR}${EFIDIR} cp $iso_dir/vmlinuz ${EFIIMGDIR} - echo "${GRUB_IMAGE}" > ${EFIIMGDIR}/startup.nsh + EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g') + printf 'fs0:%s\%s\n' "$EFIPATH" "$GRUB_IMAGE" > ${EFIIMGDIR}/startup.nsh if [ -f "$iso_dir/initrd" ] ; then cp $iso_dir/initrd ${EFIIMGDIR} fi @@ -62,14 +74,14 @@ efi_hddimg_populate() { python build_efi_cfg() { import sys - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') if not workdir: bb.error("WORKDIR not defined, unable to package") return - gfxserial = d.getVar('GRUB_GFXSERIAL', True) or "" + gfxserial = d.getVar('GRUB_GFXSERIAL') or "" - labels = d.getVar('LABELS', True) + labels = d.getVar('LABELS') if not labels: bb.debug(1, "LABELS not defined, nothing to do") return @@ -78,46 +90,49 @@ python build_efi_cfg() { bb.debug(1, "No labels, nothing to do") return - cfile = d.getVar('GRUBCFG', True) + cfile = d.getVar('GRUB_CFG') if not cfile: - raise bb.build.FuncFailed('Unable to read GRUBCFG') + bb.fatal('Unable to read GRUB_CFG') try: - cfgfile = file(cfile, 'w') + cfgfile = open(cfile, 'w') except OSError: - raise bb.build.funcFailed('Unable to open %s' % (cfile)) + bb.fatal('Unable to open %s' % cfile) cfgfile.write('# Automatically created by OE\n') - opts = d.getVar('GRUB_OPTS', True) + opts = d.getVar('GRUB_OPTS') if opts: for opt in opts.split(';'): cfgfile.write('%s\n' % opt) cfgfile.write('default=%s\n' % (labels.split()[0])) - timeout = d.getVar('GRUB_TIMEOUT', True) + timeout = d.getVar('GRUB_TIMEOUT') if timeout: cfgfile.write('timeout=%s\n' % timeout) else: cfgfile.write('timeout=50\n') + root = d.getVar('GRUB_ROOT') + if not root: + bb.fatal('GRUB_ROOT not defined') + if gfxserial == "1": btypes = [ [ " graphics console", "" ], - [ " serial console", d.getVar('GRUB_SERIAL', True) or "" ] ] + [ " serial console", d.getVar('GRUB_SERIAL') or "" ] ] else: btypes = [ [ "", "" ] ] for label in labels.split(): localdata = d.createCopy() - overrides = localdata.getVar('OVERRIDES', True) + overrides = localdata.getVar('OVERRIDES') if not overrides: - raise bb.build.FuncFailed('OVERRIDES not defined') + bb.fatal('OVERRIDES not defined') for btype in btypes: localdata.setVar('OVERRIDES', label + ':' + overrides) - bb.data.update_data(localdata) cfgfile.write('\nmenuentry \'%s%s\'{\n' % (label, btype[0])) lb = label @@ -125,11 +140,15 @@ python build_efi_cfg() { lb = "install-efi" cfgfile.write('linux /vmlinuz LABEL=%s' % (lb)) - append = localdata.getVar('APPEND', True) - initrd = localdata.getVar('INITRD', True) + cfgfile.write(' %s' % replace_rootfs_uuid(d, root)) + + append = localdata.getVar('APPEND') + initrd = localdata.getVar('INITRD') if append: - cfgfile.write('%s' % (append)) + append = replace_rootfs_uuid(d, append) + cfgfile.write(' %s' % (append)) + cfgfile.write(' %s' % btype[1]) cfgfile.write('\n') diff --git a/meta/classes/gsettings.bbclass b/meta/classes/gsettings.bbclass index dec5abc026..eae3dc7999 100644 --- a/meta/classes/gsettings.bbclass +++ b/meta/classes/gsettings.bbclass @@ -7,31 +7,32 @@ # TODO use a trigger so that this runs once per package operation run -DEPENDS += "glib-2.0-native" RDEPENDS_${PN} += "glib-2.0-utils" FILES_${PN} += "${datadir}/glib-2.0/schemas" +PACKAGE_WRITE_DEPS += "glib-2.0-native" + gsettings_postinstrm () { glib-compile-schemas $D${datadir}/glib-2.0/schemas } python populate_packages_append () { - pkg = d.getVar('PN', True) + pkg = d.getVar('PN') bb.note("adding gsettings postinst scripts to %s" % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True) + postinst = d.getVar('pkg_postinst_%s' % pkg) or d.getVar('pkg_postinst') if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('gsettings_postinstrm', True) + postinst += d.getVar('gsettings_postinstrm') d.setVar('pkg_postinst_%s' % pkg, postinst) bb.note("adding gsettings postrm scripts to %s" % pkg) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True) + postrm = d.getVar('pkg_postrm_%s' % pkg) or d.getVar('pkg_postrm') if not postrm: postrm = '#!/bin/sh\n' - postrm += d.getVar('gsettings_postinstrm', True) + postrm += d.getVar('gsettings_postinstrm') d.setVar('pkg_postrm_%s' % pkg, postrm) } diff --git a/meta/classes/gtk-doc.bbclass b/meta/classes/gtk-doc.bbclass index e32f98dcfc..0ae2729c0a 100644 --- a/meta/classes/gtk-doc.bbclass +++ b/meta/classes/gtk-doc.bbclass @@ -1,25 +1,69 @@ -# Helper class to pull in the right gtk-doc dependencies and disable -# gtk-doc. +# Helper class to pull in the right gtk-doc dependencies and configure +# gtk-doc to enable or disable documentation building (which requries the +# use of usermode qemu). + +# This variable is set to True if api-documentation is in +# DISTRO_FEATURES and qemu-usermode is in MACHINE_FEATURES, and False otherwise. # -# Long-term it would be great if this class could be toggled between -# gtk-doc-stub-native and the real gtk-doc-native, which would enable -# re-generation of documentation. For now, we'll make do with this which -# packages up any existing documentation (so from tarball builds). +# It should be used in recipes to determine whether gtk-doc based documentation should be built, +# so that qemu use can be avoided when necessary. +GTKDOC_ENABLED ?= "${@bb.utils.contains('DISTRO_FEATURES', 'api-documentation', \ + bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'True', 'False', d), 'False', d)}" + +EXTRA_OECONF_prepend_class-target = "${@bb.utils.contains('GTKDOC_ENABLED', 'True', '--enable-gtk-doc --enable-gtk-doc-html --disable-gtk-doc-pdf', \ + '--disable-gtk-doc', d)} " + +# When building native recipes, disable gtkdoc, as it is not necessary, +# pulls in additional dependencies, and makes build times longer +EXTRA_OECONF_prepend_class-native = "--disable-gtk-doc " +EXTRA_OECONF_prepend_class-nativesdk = "--disable-gtk-doc " + +DEPENDS_append_class-target = " gtk-doc-native qemu-native" + +# Even though gtkdoc is disabled on -native, gtk-doc package is still +# needed for m4 macros. +DEPENDS_append_class-native = " gtk-doc-native" +DEPENDS_append_class-nativesdk = " gtk-doc-native" # The documentation directory, where the infrastructure will be copied. # gtkdocize has a default of "." so to handle out-of-tree builds set this to $S. GTKDOC_DOCDIR ?= "${S}" -DEPENDS_append = " gtk-doc-stub-native" +do_configure_prepend () { + ( cd ${S}; gtkdocize --docdir ${GTKDOC_DOCDIR} || true ) +} + +inherit qemu -EXTRA_OECONF_append = "\ - --disable-gtk-doc \ - --disable-gtk-doc-html \ - --disable-gtk-doc-pdf \ -" +export STAGING_DIR_HOST -do_configure_prepend () { - ( cd ${S}; gtkdocize --docdir ${GTKDOC_DOCDIR} ) +do_compile_prepend_class-target () { + + # Write out a qemu wrapper that will be given to gtkdoc-scangobj so that it + # can run target helper binaries through that. + qemu_binary="${@qemu_wrapper_cmdline(d, '$STAGING_DIR_HOST', ['\$GIR_EXTRA_LIBS_PATH','$STAGING_DIR_HOST/${libdir}','$STAGING_DIR_HOST/${base_libdir}'])}" + cat > ${B}/gtkdoc-qemuwrapper << EOF +#!/bin/sh +# Use a modules directory which doesn't exist so we don't load random things +# which may then get deleted (or their dependencies) and potentially segfault +export GIO_MODULE_DIR=${STAGING_LIBDIR}/gio/modules-dummy + +GIR_EXTRA_LIBS_PATH=\`find ${B} -name .libs| tr '\n' ':'\`\$GIR_EXTRA_LIBS_PATH + +if [ -d ".libs" ]; then + $qemu_binary ".libs/\$@" +else + $qemu_binary "\$@" +fi + +if [ \$? -ne 0 ]; then + echo "If the above error message is about missing .so libraries, then setting up GIR_EXTRA_LIBS_PATH in the recipe should help." + echo "(typically like this: GIR_EXTRA_LIBS_PATH=\"$""{B}/something/.libs\" )" + exit 1 +fi +EOF + chmod +x ${B}/gtkdoc-qemuwrapper } + inherit pkgconfig diff --git a/meta/classes/gtk-icon-cache.bbclass b/meta/classes/gtk-icon-cache.bbclass index 12358e3aed..d87167aec0 100644 --- a/meta/classes/gtk-icon-cache.bbclass +++ b/meta/classes/gtk-icon-cache.bbclass @@ -2,14 +2,17 @@ FILES_${PN} += "${datadir}/icons/hicolor" DEPENDS += "${@['hicolor-icon-theme', '']['${BPN}' == 'hicolor-icon-theme']} gtk-icon-utils-native" +PACKAGE_WRITE_DEPS += "gtk-icon-utils-native gdk-pixbuf-native" + gtk_icon_cache_postinst() { if [ "x$D" != "x" ]; then - $INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} mlprefix=${MLPREFIX} libdir=${libdir} \ - base_libdir=${base_libdir} + $INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} \ + mlprefix=${MLPREFIX} \ + libdir_native=${libdir_native} else # Update the pixbuf loaders in case they haven't been registered yet - GDK_PIXBUF_MODULEDIR=${libdir}/gdk-pixbuf-2.0/2.10.0/loaders gdk-pixbuf-query-loaders --update-cache + ${libdir}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders --update-cache for icondir in /usr/share/icons/* ; do if [ -d $icondir ] ; then @@ -21,8 +24,9 @@ fi gtk_icon_cache_postrm() { if [ "x$D" != "x" ]; then - $INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} mlprefix=${MLPREFIX} libdir=${libdir} \ - base_libdir=${base_libdir} + $INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} \ + mlprefix=${MLPREFIX} \ + libdir=${libdir} else for icondir in /usr/share/icons/* ; do if [ -d $icondir ] ; then @@ -33,11 +37,11 @@ fi } python populate_packages_append () { - packages = d.getVar('PACKAGES', True).split() - pkgdest = d.getVar('PKGDEST', True) + packages = d.getVar('PACKAGES').split() + pkgdest = d.getVar('PKGDEST') for pkg in packages: - icon_dir = '%s/%s/%s/icons' % (pkgdest, pkg, d.getVar('datadir', True)) + icon_dir = '%s/%s/%s/icons' % (pkgdest, pkg, d.getVar('datadir')) if not os.path.exists(icon_dir): continue @@ -47,16 +51,16 @@ python populate_packages_append () { bb.note("adding gtk-icon-cache postinst and postrm scripts to %s" % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('gtk_icon_cache_postinst', True) + postinst += d.getVar('gtk_icon_cache_postinst') d.setVar('pkg_postinst_%s' % pkg, postinst) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) + postrm = d.getVar('pkg_postrm_%s' % pkg) if not postrm: postrm = '#!/bin/sh\n' - postrm += d.getVar('gtk_icon_cache_postrm', True) + postrm += d.getVar('gtk_icon_cache_postrm') d.setVar('pkg_postrm_%s' % pkg, postrm) } diff --git a/meta/classes/gtk-immodules-cache.bbclass b/meta/classes/gtk-immodules-cache.bbclass index 0a6316da47..3d82dbe9e3 100644 --- a/meta/classes/gtk-immodules-cache.bbclass +++ b/meta/classes/gtk-immodules-cache.bbclass @@ -2,7 +2,7 @@ # # Usage: Set GTKIMMODULES_PACKAGES to the packages that needs to update the inputmethod modules -DEPENDS =+ "qemu-native" +PACKAGE_WRITE_DEPS += "qemu-native" inherit qemu @@ -10,66 +10,72 @@ GTKIMMODULES_PACKAGES ?= "${PN}" gtk_immodule_cache_postinst() { if [ "x$D" != "x" ]; then - for maj_ver in 2 3; do - if [ -x $D${bindir}/gtk-query-immodules-$maj_ver.0 ]; then - IMFILES=$(ls $D${libdir}/gtk-$maj_ver.0/*/immodules/*.so) - ${@qemu_run_binary(d, '$D', '${bindir}/gtk-query-immodules-$maj_ver.0')} \ - $IMFILES > $D/etc/gtk-$maj_ver.0/gtk.immodules 2>/dev/null && - sed -i -e "s:$D::" $D/etc/gtk-$maj_ver.0/gtk.immodules - - [ $? -ne 0 ] && exit 1 + if [ -x $D${bindir}/gtk-query-immodules-2.0 ]; then + IMFILES=$(ls $D${libdir}/gtk-2.0/*/immodules/*.so) + ${@qemu_run_binary(d, '$D', '${bindir}/gtk-query-immodules-2.0')} \ + $IMFILES > $D${libdir}/gtk-2.0/2.10.0/immodules.cache 2>/dev/null && + sed -i -e "s:$D::" $D${libdir}/gtk-2.0/2.10.0/immodules.cache + fi + if [ -x $D${bindir}/gtk-query-immodules-3.0 ]; then + IMFILES=$(ls $D${libdir}/gtk-3.0/*/immodules/*.so) + ${@qemu_run_binary(d, '$D', '${bindir}/gtk-query-immodules-3.0')} \ + $IMFILES > $D${libdir}/gtk-3.0/3.0.0/immodules.cache 2>/dev/null && + sed -i -e "s:$D::" $D${libdir}/gtk-3.0/3.0.0/immodules.cache fi - done + [ $? -ne 0 ] && exit 1 exit 0 fi if [ ! -z `which gtk-query-immodules-2.0` ]; then - gtk-query-immodules-2.0 > /etc/gtk-2.0/gtk.immodules + gtk-query-immodules-2.0 > ${libdir}/gtk-2.0/2.10.0/immodules.cache fi if [ ! -z `which gtk-query-immodules-3.0` ]; then - gtk-query-immodules-3.0 > /etc/gtk-3.0/gtk.immodules + gtk-query-immodules-3.0 > ${libdir}/gtk-3.0/3.0.0/immodules.cache fi } gtk_immodule_cache_postrm() { if [ "x$D" != "x" ]; then - for maj_ver in 2 3; do - if [ -x $D${bindir}/gtk-query-immodules-$maj_ver.0 ]; then - IMFILES=$(ls $D${libdir}/gtk-$maj_ver.0/*/immodules/*.so) - ${@qemu_run_binary(d, '$D', '${bindir}/gtk-query-immodules-$maj_ver.0')} \ - $IMFILES > $D/etc/gtk-$maj_ver.0/gtk.immodules 2>/dev/null && - sed -i -e "s:$D::" $D/etc/gtk-$maj_ver.0/gtk.immodules - - [ $? -ne 0 ] && exit 1 + if [ -x $D${bindir}/gtk-query-immodules-2.0 ]; then + IMFILES=$(ls $D${libdir}/gtk-2.0/*/immodules/*.so) + ${@qemu_run_binary(d, '$D', '${bindir}/gtk-query-immodules-2.0')} \ + $IMFILES > $D${libdir}/gtk-2.0/2.10.0/immodules.cache 2>/dev/null && + sed -i -e "s:$D::" $D${libdir}/gtk-2.0/2.10.0/immodules.cache + fi + if [ -x $D${bindir}/gtk-query-immodules-3.0 ]; then + IMFILES=$(ls $D${libdir}/gtk-3.0/*/immodules/*.so) + ${@qemu_run_binary(d, '$D', '${bindir}/gtk-query-immodules-3.0')} \ + $IMFILES > $D${libdir}/gtk-3.0/3.0.0/immodules.cache 2>/dev/null && + sed -i -e "s:$D::" $D${libdir}/gtk-3.0/3.0.0/immodules.cache fi - done + [ $? -ne 0 ] && exit 1 exit 0 fi if [ ! -z `which gtk-query-immodules-2.0` ]; then - gtk-query-immodules-2.0 > /etc/gtk-2.0/gtk.immodules + gtk-query-immodules-2.0 > ${libdir}/gtk-2.0/2.10.0/immodules.cache fi if [ ! -z `which gtk-query-immodules-3.0` ]; then - gtk-query-immodules-3.0 > /etc/gtk-3.0/gtk.immodules + gtk-query-immodules-3.0 > ${libdir}/gtk-3.0/3.0.0/immodules.cache fi } python populate_packages_append () { - gtkimmodules_pkgs = d.getVar('GTKIMMODULES_PACKAGES', True).split() + gtkimmodules_pkgs = d.getVar('GTKIMMODULES_PACKAGES').split() for pkg in gtkimmodules_pkgs: bb.note("adding gtk-immodule-cache postinst and postrm scripts to %s" % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('gtk_immodule_cache_postinst', True) + postinst += d.getVar('gtk_immodule_cache_postinst') d.setVar('pkg_postinst_%s' % pkg, postinst) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) + postrm = d.getVar('pkg_postrm_%s' % pkg) if not postrm: postrm = '#!/bin/sh\n' - postrm += d.getVar('gtk_immodule_cache_postrm', True) + postrm += d.getVar('gtk_immodule_cache_postrm') d.setVar('pkg_postrm_%s' % pkg, postrm) } @@ -78,6 +84,6 @@ python __anonymous() { gtkimmodules_check = d.getVar('GTKIMMODULES_PACKAGES', False) if not gtkimmodules_check: bb_filename = d.getVar('FILE', False) - raise bb.build.FuncFailed("ERROR: %s inherits gtk-immodules-cache but doesn't set GTKIMMODULES_PACKAGES" % bb_filename) + bb.fatal("ERROR: %s inherits gtk-immodules-cache but doesn't set GTKIMMODULES_PACKAGES" % bb_filename) } diff --git a/meta/classes/gzipnative.bbclass b/meta/classes/gzipnative.bbclass deleted file mode 100644 index 326cbbb6f6..0000000000 --- a/meta/classes/gzipnative.bbclass +++ /dev/null @@ -1,5 +0,0 @@ -EXTRANATIVEPATH += "pigz-native gzip-native" -DEPENDS += "gzip-native" - -# tar may get run by do_unpack or do_populate_lic which could call gzip -do_unpack[depends] += "gzip-native:do_populate_sysroot" diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass index 61b8bb1a11..8a351cf3b8 100644 --- a/meta/classes/icecc.bbclass +++ b/meta/classes/icecc.bbclass @@ -47,7 +47,8 @@ def get_cross_kernel_cc(bb,d): # evaluate the expression by the shell if necessary if '`' in kernel_cc or '$(' in kernel_cc: - kernel_cc = os.popen("echo %s" % kernel_cc).read()[:-1] + import subprocess + kernel_cc = subprocess.check_output("echo %s" % kernel_cc, shell=True).decode("utf-8")[:-1] kernel_cc = d.expand(kernel_cc) kernel_cc = kernel_cc.replace('ccache', '').strip() @@ -63,7 +64,7 @@ def create_path(compilers, bb, d): Create Symlinks for the icecc in the staging directory """ staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice") - if icc_is_kernel(bb, d): + if icecc_is_kernel(bb, d): staging += "-kernel" #check if the icecc path is set by the user @@ -90,16 +91,16 @@ def create_path(compilers, bb, d): return staging -def use_icc(bb,d): +def use_icecc(bb,d): if d.getVar('ICECC_DISABLED', False) == "1": # don't even try it, when explicitly disabled return "no" # allarch recipes don't use compiler - if icc_is_allarch(bb, d): + if icecc_is_allarch(bb, d): return "no" - pn = d.getVar('PN', True) + pn = d.getVar('PN') system_class_blacklist = [] user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL', False) or "none").split() @@ -137,29 +138,29 @@ def use_icc(bb,d): return "yes" -def icc_is_allarch(bb, d): - return d.getVar("PACKAGE_ARCH", False) == "all" +def icecc_is_allarch(bb, d): + return d.getVar("PACKAGE_ARCH") == "all" or bb.data.inherits_class('allarch', d) -def icc_is_kernel(bb, d): +def icecc_is_kernel(bb, d): return \ bb.data.inherits_class("kernel", d); -def icc_is_native(bb, d): +def icecc_is_native(bb, d): return \ bb.data.inherits_class("cross", d) or \ bb.data.inherits_class("native", d); # Don't pollute allarch signatures with TARGET_FPU -icc_version[vardepsexclude] += "TARGET_FPU" -def icc_version(bb, d): - if use_icc(bb, d) == "no": +icecc_version[vardepsexclude] += "TARGET_FPU" +def icecc_version(bb, d): + if use_icecc(bb, d) == "no": return "" parallel = d.getVar('ICECC_PARALLEL_MAKE', False) or "" if not d.getVar('PARALLEL_MAKE', False) == "" and parallel: d.setVar("PARALLEL_MAKE", parallel) - if icc_is_native(bb, d): + if icecc_is_native(bb, d): archive_name = "local-host-env" elif d.expand('${HOST_PREFIX}') == "": bb.fatal(d.expand("${PN}"), " NULL prefix") @@ -169,7 +170,7 @@ def icc_version(bb, d): target_sys = d.expand('${TARGET_SYS}') float = d.getVar('TARGET_FPU', False) or "hard" archive_name = prefix + distro + "-" + target_sys + "-" + float - if icc_is_kernel(bb, d): + if icecc_is_kernel(bb, d): archive_name += "-kernel" import socket @@ -178,29 +179,29 @@ def icc_version(bb, d): return tar_file -def icc_path(bb,d): - if use_icc(bb, d) == "no": +def icecc_path(bb,d): + if use_icecc(bb, d) == "no": # don't create unnecessary directories when icecc is disabled return - if icc_is_kernel(bb, d): + if icecc_is_kernel(bb, d): return create_path( [get_cross_kernel_cc(bb,d), ], bb, d) else: prefix = d.expand('${HOST_PREFIX}') return create_path( [prefix+"gcc", prefix+"g++"], bb, d) -def icc_get_external_tool(bb, d, tool): +def icecc_get_external_tool(bb, d, tool): external_toolchain_bindir = d.expand('${EXTERNAL_TOOLCHAIN}${bindir_cross}') target_prefix = d.expand('${TARGET_PREFIX}') return os.path.join(external_toolchain_bindir, '%s%s' % (target_prefix, tool)) # Don't pollute native signatures with target TUNE_PKGARCH through STAGING_BINDIR_TOOLCHAIN -icc_get_tool[vardepsexclude] += "STAGING_BINDIR_TOOLCHAIN" -def icc_get_tool(bb, d, tool): - if icc_is_native(bb, d): +icecc_get_tool[vardepsexclude] += "STAGING_BINDIR_TOOLCHAIN" +def icecc_get_tool(bb, d, tool): + if icecc_is_native(bb, d): return bb.utils.which(os.getenv("PATH"), tool) - elif icc_is_kernel(bb, d): + elif icecc_is_kernel(bb, d): return bb.utils.which(os.getenv("PATH"), get_cross_kernel_cc(bb, d)) else: ice_dir = d.expand('${STAGING_BINDIR_TOOLCHAIN}') @@ -209,20 +210,25 @@ def icc_get_tool(bb, d, tool): if os.path.isfile(tool_bin): return tool_bin else: - external_tool_bin = icc_get_external_tool(bb, d, tool) + external_tool_bin = icecc_get_external_tool(bb, d, tool) if os.path.isfile(external_tool_bin): return external_tool_bin else: return "" -def icc_get_and_check_tool(bb, d, tool): +def icecc_get_and_check_tool(bb, d, tool): # Check that g++ or gcc is not a symbolic link to icecc binary in # PATH or icecc-create-env script will silently create an invalid # compiler environment package. - t = icc_get_tool(bb, d, tool) - if t and os.popen("readlink -f %s" % t).read()[:-1] == get_icecc(d): - bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, get_icecc(d))) - return "" + t = icecc_get_tool(bb, d, tool) + if t: + import subprocess + link_path = subprocess.check_output("readlink -f %s" % t, shell=True).decode("utf-8")[:-1] + if link_path == get_icecc(d): + bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, get_icecc(d))) + return "" + else: + return t else: return t @@ -246,27 +252,27 @@ def set_icecc_env(): return set_icecc_env() { - if [ "${@use_icc(bb, d)}" = "no" ] + if [ "${@use_icecc(bb, d)}" = "no" ] then return fi - ICECC_VERSION="${@icc_version(bb, d)}" + ICECC_VERSION="${@icecc_version(bb, d)}" if [ "x${ICECC_VERSION}" = "x" ] then bbwarn "Cannot use icecc: could not get ICECC_VERSION" return fi - ICE_PATH="${@icc_path(bb, d)}" + ICE_PATH="${@icecc_path(bb, d)}" if [ "x${ICE_PATH}" = "x" ] then bbwarn "Cannot use icecc: could not get ICE_PATH" return fi - ICECC_CC="${@icc_get_and_check_tool(bb, d, "gcc")}" - ICECC_CXX="${@icc_get_and_check_tool(bb, d, "g++")}" - # cannot use icc_get_and_check_tool here because it assumes as without target_sys prefix + ICECC_CC="${@icecc_get_and_check_tool(bb, d, "gcc")}" + ICECC_CXX="${@icecc_get_and_check_tool(bb, d, "g++")}" + # cannot use icecc_get_and_check_tool here because it assumes as without target_sys prefix ICECC_WHICH_AS="${@bb.utils.which(os.getenv('PATH'), 'as')}" if [ ! -x "${ICECC_CC}" -o ! -x "${ICECC_CXX}" ] then diff --git a/meta/classes/image-buildinfo.bbclass b/meta/classes/image-buildinfo.bbclass index aa17cc8f9e..213fb9cf9b 100644 --- a/meta/classes/image-buildinfo.bbclass +++ b/meta/classes/image-buildinfo.bbclass @@ -12,13 +12,16 @@ # Desired variables to display IMAGE_BUILDINFO_VARS ?= "DISTRO DISTRO_VERSION" +# Desired location of the output file in the image. +IMAGE_BUILDINFO_FILE ??= "${sysconfdir}/build" + # From buildhistory.bbclass def image_buildinfo_outputvars(vars, listvars, d): vars = vars.split() listvars = listvars.split() ret = "" for var in vars: - value = d.getVar(var, True) or "" + value = d.getVar(var) or "" if (d.getVarFlag(var, 'type') == "list"): value = oe.utils.squashspaces(value) ret += "%s = %s\n" % (var, value) @@ -26,16 +29,23 @@ def image_buildinfo_outputvars(vars, listvars, d): # Gets git branch's status (clean or dirty) def get_layer_git_status(path): - f = os.popen("cd %s; git diff --stat 2>&1 | tail -n 1" % path) - data = f.read() - if f.close() is None: - if len(data) != 0: - return "-- modified" - return "" + import subprocess + try: + subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e; + git diff --quiet --no-ext-diff + git diff --quiet --no-ext-diff --cached""" % path, + shell=True, + stderr=subprocess.STDOUT) + return "" + except subprocess.CalledProcessError as ex: + # Silently treat errors as "modified", without checking for the + # (expected) return code 1 in a modified git repo. For example, we get + # output and a 129 return code when a layer isn't a git repo at all. + return "-- modified" # Returns layer revisions along with their respective status def get_layer_revs(d): - layers = (d.getVar("BBLAYERS", True) or "").split() + layers = (d.getVar("BBLAYERS") or "").split() medadata_revs = ["%-17s = %s:%s %s" % (os.path.basename(i), \ base_get_metadata_git_branch(i, None).strip(), \ base_get_metadata_git_revision(i, None), \ @@ -45,25 +55,31 @@ def get_layer_revs(d): def buildinfo_target(d): # Get context - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" # Single and list variables to be read - vars = (d.getVar("IMAGE_BUILDINFO_VARS", True) or "") - listvars = (d.getVar("IMAGE_BUILDINFO_LVARS", True) or "") + vars = (d.getVar("IMAGE_BUILDINFO_VARS") or "") + listvars = (d.getVar("IMAGE_BUILDINFO_LVARS") or "") return image_buildinfo_outputvars(vars, listvars, d) # Write build information to target filesystem -buildinfo () { -cat > ${IMAGE_ROOTFS}${sysconfdir}/build << END ------------------------ +python buildinfo () { + with open(d.expand('${IMAGE_ROOTFS}${IMAGE_BUILDINFO_FILE}'), 'w') as build: + build.writelines(( + '''----------------------- Build Configuration: | ----------------------- -${@buildinfo_target(d)} +''', + buildinfo_target(d), + ''' ----------------------- -Layer Revisions: | +Layer Revisions: | ----------------------- -${@get_layer_revs(d)} -END +''', + get_layer_revs(d), + ''' +''' + )) } IMAGE_PREPROCESS_COMMAND += "buildinfo;" diff --git a/meta/classes/image-container.bbclass b/meta/classes/image-container.bbclass new file mode 100644 index 0000000000..f002858bd2 --- /dev/null +++ b/meta/classes/image-container.bbclass @@ -0,0 +1,21 @@ +ROOTFS_BOOTSTRAP_INSTALL = "" +IMAGE_TYPES_MASKED += "container" +IMAGE_TYPEDEP_container = "tar.bz2" + +python __anonymous() { + if "container" in d.getVar("IMAGE_FSTYPES") and \ + d.getVar("IMAGE_CONTAINER_NO_DUMMY") != "1" and \ + "linux-dummy" not in d.getVar("PREFERRED_PROVIDER_virtual/kernel"): + msg = '"container" is in IMAGE_FSTYPES, but ' \ + 'PREFERRED_PROVIDER_virtual/kernel is not "linux-dummy". ' \ + 'Unless a particular kernel is needed, using linux-dummy will ' \ + 'prevent a kernel from being built, which can reduce ' \ + 'build times. If you don\'t want to use "linux-dummy", set ' \ + '"IMAGE_CONTAINER_NO_DUMMY" to "1".' + + # Raising skip recipe was Paul's clever idea. It causes the error to + # only be shown for the recipes actually requested to build, rather + # than bb.fatal which would appear for all recipes inheriting the + # class. + raise bb.parse.SkipRecipe(msg) +} diff --git a/meta/classes/image-live.bbclass b/meta/classes/image-live.bbclass index fa7a131eda..a3d1b4e567 100644 --- a/meta/classes/image-live.bbclass +++ b/meta/classes/image-live.bbclass @@ -1,18 +1,278 @@ +# Copyright (C) 2004, Advanced Micro Devices, Inc. All Rights Reserved +# Released under the MIT license (see packages/COPYING) -AUTO_SYSLINUXCFG = "1" -INITRD_IMAGE ?= "core-image-minimal-initramfs" -INITRD ?= "${DEPLOY_DIR_IMAGE}/${INITRD_IMAGE}-${MACHINE}.cpio.gz" -SYSLINUX_ROOT = "root=/dev/ram0" -SYSLINUX_TIMEOUT ?= "50" -SYSLINUX_LABELS ?= "boot install" -LABELS_append = " ${SYSLINUX_LABELS} " +# Creates a bootable image using syslinux, your kernel and an optional +# initrd -ROOTFS ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_BASENAME}-${MACHINE}.ext4" +# +# End result is two things: +# +# 1. A .hddimg file which is an msdos filesystem containing syslinux, a kernel, +# an initrd and a rootfs image. These can be written to harddisks directly and +# also booted on USB flash disks (write them there with dd). +# +# 2. A CD .iso image -do_bootimg[depends] += "${INITRD_IMAGE}:do_rootfs" -do_bootimg[depends] += "${PN}:do_rootfs" +# Boot process is that the initrd will boot and process which label was selected +# in syslinux. Actions based on the label are then performed (e.g. installing to +# an hdd) -inherit bootimg +# External variables (also used by syslinux.bbclass) +# ${INITRD} - indicates a list of filesystem images to concatenate and use as an initrd (optional) +# ${COMPRESSISO} - Transparent compress ISO, reduce size ~40% if set to 1 +# ${NOISO} - skip building the ISO image if set to 1 +# ${NOHDD} - skip building the HDD image if set to 1 +# ${HDDIMG_ID} - FAT image volume-id +# ${ROOTFS} - indicates a filesystem image to include as the root filesystem (optional) + +inherit live-vm-common + +do_bootimg[depends] += "dosfstools-native:do_populate_sysroot \ + mtools-native:do_populate_sysroot \ + cdrtools-native:do_populate_sysroot \ + virtual/kernel:do_deploy \ + ${MLPREFIX}syslinux:do_populate_sysroot \ + syslinux-native:do_populate_sysroot \ + ${@oe.utils.ifelse(d.getVar('COMPRESSISO', False),'zisofs-tools-native:do_populate_sysroot','')} \ + ${PN}:do_image_ext4 \ + " + + +LABELS_LIVE ?= "boot install" +ROOT_LIVE ?= "root=/dev/ram0" +INITRD_IMAGE_LIVE ?= "core-image-minimal-initramfs" +INITRD_LIVE ?= "${DEPLOY_DIR_IMAGE}/${INITRD_IMAGE_LIVE}-${MACHINE}.cpio.gz" + +ROOTFS ?= "${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.ext4" IMAGE_TYPEDEP_live = "ext4" -IMAGE_TYPES_MASKED += "live" +IMAGE_TYPEDEP_iso = "ext4" +IMAGE_TYPEDEP_hddimg = "ext4" +IMAGE_TYPES_MASKED += "live hddimg iso" + +python() { + image_b = d.getVar('IMAGE_BASENAME') + initrd_i = d.getVar('INITRD_IMAGE_LIVE') + if image_b == initrd_i: + bb.error('INITRD_IMAGE_LIVE %s cannot use image live, hddimg or iso.' % initrd_i) + bb.fatal('Check IMAGE_FSTYPES and INITRAMFS_FSTYPES settings.') + elif initrd_i: + d.appendVarFlag('do_bootimg', 'depends', ' %s:do_image_complete' % initrd_i) +} + +HDDDIR = "${S}/hddimg" +ISODIR = "${S}/iso" +EFIIMGDIR = "${S}/efi_img" +COMPACT_ISODIR = "${S}/iso.z" +COMPRESSISO ?= "0" + +ISOLINUXDIR ?= "/isolinux" +ISO_BOOTIMG = "isolinux/isolinux.bin" +ISO_BOOTCAT = "isolinux/boot.cat" +MKISOFS_OPTIONS = "-no-emul-boot -boot-load-size 4 -boot-info-table" + +BOOTIMG_VOLUME_ID ?= "boot" +BOOTIMG_EXTRA_SPACE ?= "512" + +populate_live() { + populate_kernel $1 + if [ -s "${ROOTFS}" ]; then + install -m 0644 ${ROOTFS} $1/rootfs.img + fi +} + +build_iso() { + # Only create an ISO if we have an INITRD and NOISO was not set + if [ -z "${INITRD}" ] || [ "${NOISO}" = "1" ]; then + bbnote "ISO image will not be created." + return + fi + # ${INITRD} is a list of multiple filesystem images + for fs in ${INITRD} + do + if [ ! -s "$fs" ]; then + bbnote "ISO image will not be created. $fs is invalid." + return + fi + done + + populate_live ${ISODIR} + + if [ "${PCBIOS}" = "1" ]; then + syslinux_iso_populate ${ISODIR} + fi + if [ "${EFI}" = "1" ]; then + efi_iso_populate ${ISODIR} + build_fat_img ${EFIIMGDIR} ${ISODIR}/efi.img + fi + + # EFI only + if [ "${PCBIOS}" != "1" ] && [ "${EFI}" = "1" ] ; then + # Work around bug in isohybrid where it requires isolinux.bin + # In the boot catalog, even though it is not used + mkdir -p ${ISODIR}/${ISOLINUXDIR} + install -m 0644 ${STAGING_DATADIR}/syslinux/isolinux.bin ${ISODIR}${ISOLINUXDIR} + fi + + if [ "${COMPRESSISO}" = "1" ] ; then + # create compact directory, compress iso + mkdir -p ${COMPACT_ISODIR} + mkzftree -z 9 -p 4 -F ${ISODIR}/rootfs.img ${COMPACT_ISODIR}/rootfs.img + + # move compact iso to iso, then remove compact directory + mv ${COMPACT_ISODIR}/rootfs.img ${ISODIR}/rootfs.img + rm -Rf ${COMPACT_ISODIR} + mkisofs_compress_opts="-R -z -D -l" + else + mkisofs_compress_opts="-r" + fi + + # Check the size of ${ISODIR}/rootfs.img, use mkisofs -iso-level 3 + # when it exceeds 3.8GB, the specification is 4G - 1 bytes, we need + # leave a few space for other files. + mkisofs_iso_level="" + + if [ -n "${ROOTFS}" ] && [ -s "${ROOTFS}" ]; then + rootfs_img_size=`stat -c '%s' ${ISODIR}/rootfs.img` + # 4080218931 = 3.8 * 1024 * 1024 * 1024 + if [ $rootfs_img_size -gt 4080218931 ]; then + bbnote "${ISODIR}/rootfs.img execeeds 3.8GB, using '-iso-level 3' for mkisofs" + mkisofs_iso_level="-iso-level 3" + fi + fi + + if [ "${PCBIOS}" = "1" ] && [ "${EFI}" != "1" ] ; then + # PCBIOS only media + mkisofs -V ${BOOTIMG_VOLUME_ID} \ + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ + -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ + $mkisofs_compress_opts \ + ${MKISOFS_OPTIONS} $mkisofs_iso_level ${ISODIR} + else + # EFI only OR EFI+PCBIOS + mkisofs -A ${BOOTIMG_VOLUME_ID} -V ${BOOTIMG_VOLUME_ID} \ + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ + -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ + $mkisofs_compress_opts ${MKISOFS_OPTIONS} $mkisofs_iso_level \ + -eltorito-alt-boot -eltorito-platform efi \ + -b efi.img -no-emul-boot \ + ${ISODIR} + isohybrid_args="-u" + fi + + isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso +} + +build_fat_img() { + FATSOURCEDIR=$1 + FATIMG=$2 + + # Calculate the size required for the final image including the + # data and filesystem overhead. + # Sectors: 512 bytes + # Blocks: 1024 bytes + + # Determine the sector count just for the data + SECTORS=$(expr $(du --apparent-size -ks ${FATSOURCEDIR} | cut -f 1) \* 2) + + # Account for the filesystem overhead. This includes directory + # entries in the clusters as well as the FAT itself. + # Assumptions: + # FAT32 (12 or 16 may be selected by mkdosfs, but the extra + # padding will be minimal on those smaller images and not + # worth the logic here to caclulate the smaller FAT sizes) + # < 16 entries per directory + # 8.3 filenames only + + # 32 bytes per dir entry + DIR_BYTES=$(expr $(find ${FATSOURCEDIR} | tail -n +2 | wc -l) \* 32) + # 32 bytes for every end-of-directory dir entry + DIR_BYTES=$(expr $DIR_BYTES + $(expr $(find ${FATSOURCEDIR} -type d | tail -n +2 | wc -l) \* 32)) + # 4 bytes per FAT entry per sector of data + FAT_BYTES=$(expr $SECTORS \* 4) + # 4 bytes per FAT entry per end-of-cluster list + FAT_BYTES=$(expr $FAT_BYTES + $(expr $(find ${FATSOURCEDIR} -type d | tail -n +2 | wc -l) \* 4)) + + # Use a ceiling function to determine FS overhead in sectors + DIR_SECTORS=$(expr $(expr $DIR_BYTES + 511) / 512) + # There are two FATs on the image + FAT_SECTORS=$(expr $(expr $(expr $FAT_BYTES + 511) / 512) \* 2) + SECTORS=$(expr $SECTORS + $(expr $DIR_SECTORS + $FAT_SECTORS)) + + # Determine the final size in blocks accounting for some padding + BLOCKS=$(expr $(expr $SECTORS / 2) + ${BOOTIMG_EXTRA_SPACE}) + + # mkdosfs will sometimes use FAT16 when it is not appropriate, + # resulting in a boot failure from SYSLINUX. Use FAT32 for + # images larger than 512MB, otherwise let mkdosfs decide. + if [ $(expr $BLOCKS / 1024) -gt 512 ]; then + FATSIZE="-F 32" + fi + + # mkdosfs will fail if ${FATIMG} exists. Since we are creating an + # new image, it is safe to delete any previous image. + if [ -e ${FATIMG} ]; then + rm ${FATIMG} + fi + + if [ -z "${HDDIMG_ID}" ]; then + mkdosfs ${FATSIZE} -n ${BOOTIMG_VOLUME_ID} -S 512 -C ${FATIMG} \ + ${BLOCKS} + else + mkdosfs ${FATSIZE} -n ${BOOTIMG_VOLUME_ID} -S 512 -C ${FATIMG} \ + ${BLOCKS} -i ${HDDIMG_ID} + fi + + # Copy FATSOURCEDIR recursively into the image file directly + mcopy -i ${FATIMG} -s ${FATSOURCEDIR}/* ::/ +} + +build_hddimg() { + # Create an HDD image + if [ "${NOHDD}" != "1" ] ; then + populate_live ${HDDDIR} + + if [ "${PCBIOS}" = "1" ]; then + syslinux_hddimg_populate ${HDDDIR} + fi + if [ "${EFI}" = "1" ]; then + efi_hddimg_populate ${HDDDIR} + fi + + # Check the size of ${HDDDIR}/rootfs.img, error out if it + # exceeds 4GB, it is the single file's max size of FAT fs. + if [ -f ${HDDDIR}/rootfs.img ]; then + rootfs_img_size=`stat -c '%s' ${HDDDIR}/rootfs.img` + max_size=`expr 4 \* 1024 \* 1024 \* 1024` + if [ $rootfs_img_size -gt $max_size ]; then + bberror "${HDDDIR}/rootfs.img execeeds 4GB," + bberror "this doesn't work on FAT filesystem, you can try either of:" + bberror "1) Reduce the size of rootfs.img" + bbfatal "2) Use iso, vmdk or vdi to instead of hddimg\n" + fi + fi + + build_fat_img ${HDDDIR} ${IMGDEPLOYDIR}/${IMAGE_NAME}.hddimg + + if [ "${PCBIOS}" = "1" ]; then + syslinux_hddimg_install + fi + + chmod 644 ${IMGDEPLOYDIR}/${IMAGE_NAME}.hddimg + fi +} + +python do_bootimg() { + set_live_vm_vars(d, 'LIVE') + if d.getVar("PCBIOS") == "1": + bb.build.exec_func('build_syslinux_cfg', d) + if d.getVar("EFI") == "1": + bb.build.exec_func('build_efi_cfg', d) + bb.build.exec_func('build_hddimg', d) + bb.build.exec_func('build_iso', d) + bb.build.exec_func('create_symlinks', d) +} +do_bootimg[subimages] = "hddimg iso" +do_bootimg[imgsuffix] = "." + +addtask bootimg before do_image_complete diff --git a/meta/classes/image-mklibs.bbclass b/meta/classes/image-mklibs.bbclass index c455a8e2d4..5f6df1b17f 100644 --- a/meta/classes/image-mklibs.bbclass +++ b/meta/classes/image-mklibs.bbclass @@ -2,39 +2,24 @@ do_rootfs[depends] += "mklibs-native:do_populate_sysroot" IMAGE_PREPROCESS_COMMAND += "mklibs_optimize_image; " +inherit linuxloader + mklibs_optimize_image_doit() { rm -rf ${WORKDIR}/mklibs mkdir -p ${WORKDIR}/mklibs/dest cd ${IMAGE_ROOTFS} du -bs > ${WORKDIR}/mklibs/du.before.mklibs.txt - for i in `find .`; do file $i; done \ - | grep ELF \ - | grep "LSB *executable" \ - | grep "dynamically linked" \ - | sed "s/:.*//" \ - | sed "s+^\./++" \ - > ${WORKDIR}/mklibs/executables.list - case ${TARGET_ARCH} in - powerpc | mips | mipsel | microblaze ) - dynamic_loader="${base_libdir}/ld.so.1" - ;; - powerpc64) - dynamic_loader="${base_libdir}/ld64.so.1" - ;; - x86_64) - dynamic_loader="${base_libdir}/ld-linux-x86-64.so.2" - ;; - i586 ) - dynamic_loader="${base_libdir}/ld-linux.so.2" - ;; - arm ) - dynamic_loader="${base_libdir}/ld-linux.so.3" - ;; - * ) - dynamic_loader="/unknown_dynamic_linker" - ;; - esac + # Build a list of dynamically linked executable ELF files. + # Omit libc/libpthread as a special case because it has an interpreter + # but is primarily what we intend to strip down. + for i in `find . -type f -executable ! -name 'libc-*' ! -name 'libpthread-*'`; do + file $i | grep -q ELF || continue + ${HOST_PREFIX}readelf -l $i | grep -q INTERP || continue + echo $i + done > ${WORKDIR}/mklibs/executables.list + + dynamic_loader=$(linuxloader) mklibs -v \ --ldlib ${dynamic_loader} \ diff --git a/meta/classes/image-prelink.bbclass b/meta/classes/image-prelink.bbclass index d4bb3aec39..4157df021a 100644 --- a/meta/classes/image-prelink.bbclass +++ b/meta/classes/image-prelink.bbclass @@ -1,6 +1,12 @@ do_rootfs[depends] += "prelink-native:do_populate_sysroot" -IMAGE_PREPROCESS_COMMAND += "prelink_image; " +IMAGE_PREPROCESS_COMMAND += "prelink_setup; prelink_image; " + +python prelink_setup () { + oe.utils.write_ld_so_conf(d) +} + +inherit linuxloader prelink_image () { # export PSEUDO_DEBUG=4 @@ -13,21 +19,36 @@ prelink_image () { # We need a prelink conf on the filesystem, add one if it's missing if [ ! -e ${IMAGE_ROOTFS}${sysconfdir}/prelink.conf ]; then - cp ${STAGING_DIR_NATIVE}${sysconfdir_native}/prelink.conf \ + cp ${STAGING_ETCDIR_NATIVE}/prelink.conf \ ${IMAGE_ROOTFS}${sysconfdir}/prelink.conf dummy_prelink_conf=true; else dummy_prelink_conf=false; fi + # We need a ld.so.conf with pathnames in,prelink conf on the filesystem, add one if it's missing + ldsoconf=${IMAGE_ROOTFS}${sysconfdir}/ld.so.conf + if [ -e $ldsoconf ]; then + cp $ldsoconf $ldsoconf.prelink + fi + cat ${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf >> $ldsoconf + + dynamic_loader=$(linuxloader) + # prelink! - ${STAGING_DIR_NATIVE}${sbindir_native}/prelink --root ${IMAGE_ROOTFS} -amR -N -c ${sysconfdir}/prelink.conf + ${STAGING_SBINDIR_NATIVE}/prelink --root ${IMAGE_ROOTFS} -amR -N -c ${sysconfdir}/prelink.conf --dynamic-linker $dynamic_loader # Remove the prelink.conf if we had to add it. if [ "$dummy_prelink_conf" = "true" ]; then rm -f ${IMAGE_ROOTFS}${sysconfdir}/prelink.conf fi + if [ -e $ldsoconf.prelink ]; then + mv $ldsoconf.prelink $ldsoconf + else + rm $ldsoconf + fi + pre_prelink_size=`du -ks ${IMAGE_ROOTFS} | awk '{size = $1 ; print size }'` echo "Size after prelinking $pre_prelink_size." } diff --git a/meta/classes/image-swab.bbclass b/meta/classes/image-swab.bbclass deleted file mode 100644 index 89318560db..0000000000 --- a/meta/classes/image-swab.bbclass +++ /dev/null @@ -1,94 +0,0 @@ -HOST_DATA ?= "${TMPDIR}/host-contamination-data/" -SWABBER_REPORT ?= "${LOG_DIR}/swabber/" -SWABBER_LOGS ?= "${LOG_DIR}/contamination-logs" -TRACE_LOGDIR ?= "${SWABBER_LOGS}/${PACKAGE_ARCH}" -TRACE_LOGFILE = "${TRACE_LOGDIR}/${PN}-${PV}" - -SWAB_ORIG_TASK := "${BB_DEFAULT_TASK}" -BB_DEFAULT_TASK = "generate_swabber_report" - -# Several recipes don't build with parallel make when run under strace -# Ideally these should be fixed but as a temporary measure disable parallel -# builds for troublesome recipes -PARALLEL_MAKE_pn-openssl = "" -PARALLEL_MAKE_pn-glibc = "" -PARALLEL_MAKE_pn-glib-2.0 = "" -PARALLEL_MAKE_pn-libxml2 = "" -PARALLEL_MAKE_pn-readline = "" -PARALLEL_MAKE_pn-util-linux = "" -PARALLEL_MAKE_pn-binutils = "" -PARALLEL_MAKE_pn-bison = "" -PARALLEL_MAKE_pn-cmake = "" -PARALLEL_MAKE_pn-elfutils = "" -PARALLEL_MAKE_pn-gcc = "" -PARALLEL_MAKE_pn-gcc-runtime = "" -PARALLEL_MAKE_pn-m4 = "" -PARALLEL_MAKE_pn-opkg = "" -PARALLEL_MAKE_pn-pkgconfig = "" -PARALLEL_MAKE_pn-prelink = "" -PARALLEL_MAKE_pn-rpm = "" -PARALLEL_MAKE_pn-tcl = "" -PARALLEL_MAKE_pn-beecrypt = "" -PARALLEL_MAKE_pn-curl = "" -PARALLEL_MAKE_pn-gmp = "" -PARALLEL_MAKE_pn-libmpc = "" -PARALLEL_MAKE_pn-libxslt = "" -PARALLEL_MAKE_pn-lzo = "" -PARALLEL_MAKE_pn-popt = "" -PARALLEL_MAKE_pn-linux-wrs = "" -PARALLEL_MAKE_pn-libgcrypt = "" -PARALLEL_MAKE_pn-gpgme = "" -PARALLEL_MAKE_pn-udev = "" -PARALLEL_MAKE_pn-gnutls = "" - -python() { - # NOTE: It might be useful to detect host infection on native and cross - # packages but as it turns out to be pretty hard to do this for all native - # and cross packages which aren't swabber-native or one of its dependencies - # I have ignored them for now... - if not bb.data.inherits_class('native', d) and not bb.data.inherits_class('nativesdk', d) and not bb.data.inherits_class('cross', d): - deps = (d.getVarFlag('do_setscene', 'depends') or "").split() - deps.append('strace-native:do_populate_sysroot') - d.setVarFlag('do_setscene', 'depends', " ".join(deps)) - logdir = d.expand("${TRACE_LOGDIR}") - bb.utils.mkdirhier(logdir) - else: - d.setVar('STRACEFUNC', '') -} - -STRACEPID = "${@os.getpid()}" -STRACEFUNC = "imageswab_attachstrace" - -do_configure[prefuncs] += "${STRACEFUNC}" -do_compile[prefuncs] += "${STRACEFUNC}" - -imageswab_attachstrace () { - STRACE=`which strace` - - if [ -x "$STRACE" ]; then - swabber-strace-attach "$STRACE -f -o ${TRACE_LOGFILE}-${BB_CURRENTTASK}.log -e trace=open,execve -p ${STRACEPID}" "${TRACE_LOGFILE}-traceattach-${BB_CURRENTTASK}.log" - fi -} - -do_generate_swabber_report () { - - update_distro ${HOST_DATA} - - # Swabber can't create the directory for us - mkdir -p ${SWABBER_REPORT} - - REPORTSTAMP=${SWAB_ORIG_TASK}-`date +%2m%2d%2H%2M%Y` - - if [ `which ccache` ] ; then - CCACHE_DIR=`( ccache -s | grep "cache directory" | grep -o '[^ ]*$' 2> /dev/null )` - fi - - if [ "$(ls -A ${HOST_DATA})" ]; then - echo "Generating swabber report" - swabber -d ${HOST_DATA} -l ${SWABBER_LOGS} -o ${SWABBER_REPORT}/report-${REPORTSTAMP}.txt -r ${SWABBER_REPORT}/extra_report-${REPORTSTAMP}.txt -c all -p ${TOPDIR} -f ${OEROOT}/meta/conf/swabber ${TOPDIR} ${OEROOT} ${CCACHE_DIR} - else - echo "No host data, cannot generate swabber report." - fi -} -addtask generate_swabber_report after do_${SWAB_ORIG_TASK} -do_generate_swabber_report[depends] = "swabber-native:do_populate_sysroot" diff --git a/meta/classes/image-vm.bbclass b/meta/classes/image-vm.bbclass index bc0503bfc0..35c9244e9b 100644 --- a/meta/classes/image-vm.bbclass +++ b/meta/classes/image-vm.bbclass @@ -1,46 +1,171 @@ +# image-vm.bbclass +# (loosly based off image-live.bbclass Copyright (C) 2004, Advanced Micro Devices, Inc.) +# +# Create an image which can be placed directly onto a harddisk using dd and then +# booted. +# +# This uses syslinux. extlinux would have been nice but required the ext2/3 +# partition to be mounted. grub requires to run itself as part of the install +# process. +# +# The end result is a 512 boot sector populated with an MBR and partition table +# followed by an msdos fat16 partition containing syslinux and a linux kernel +# completed by the ext2/3 rootfs. +# +# We have to push the msdos parition table size > 16MB so fat 16 is used as parted +# won't touch fat12 partitions. -SYSLINUX_ROOT ?= "root=/dev/sda2" -SYSLINUX_PROMPT ?= "0" -SYSLINUX_TIMEOUT ?= "10" -SYSLINUX_LABELS = "boot" -LABELS_append = " ${SYSLINUX_LABELS} " +inherit live-vm-common -# need to define the dependency and the ROOTFS for directdisk -do_bootdirectdisk[depends] += "${PN}:do_rootfs" -ROOTFS ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_BASENAME}-${MACHINE}.ext4" +do_bootdirectdisk[depends] += "dosfstools-native:do_populate_sysroot \ + virtual/kernel:do_deploy \ + syslinux:do_populate_sysroot \ + syslinux-native:do_populate_sysroot \ + parted-native:do_populate_sysroot \ + mtools-native:do_populate_sysroot \ + ${PN}:do_image_${VM_ROOTFS_TYPE} \ + " -# creating VM images relies on having a hddimg so ensure we inherit it here. -inherit boot-directdisk +IMAGE_TYPEDEP_vmdk = "${VM_ROOTFS_TYPE}" +IMAGE_TYPEDEP_vdi = "${VM_ROOTFS_TYPE}" +IMAGE_TYPEDEP_qcow2 = "${VM_ROOTFS_TYPE}" +IMAGE_TYPEDEP_hdddirect = "${VM_ROOTFS_TYPE}" +IMAGE_TYPES_MASKED += "vmdk vdi qcow2 hdddirect" -IMAGE_TYPEDEP_vmdk = "ext4" -IMAGE_TYPEDEP_vdi = "ext4" -IMAGE_TYPEDEP_qcow2 = "ext4" -IMAGE_TYPES_MASKED += "vmdk vdi qcow2" +VM_ROOTFS_TYPE ?= "ext4" +ROOTFS ?= "${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.${VM_ROOTFS_TYPE}" +# Used by bootloader +LABELS_VM ?= "boot" +ROOT_VM ?= "root=/dev/sda2" +# Using an initramfs is optional. Enable it by setting INITRD_IMAGE_VM. +INITRD_IMAGE_VM ?= "" +INITRD_VM ?= "${@'${IMGDEPLOYDIR}/${INITRD_IMAGE_VM}-${MACHINE}.cpio.gz' if '${INITRD_IMAGE_VM}' else ''}" +do_bootdirectdisk[depends] += "${@'${INITRD_IMAGE_VM}:do_image_complete' if '${INITRD_IMAGE_VM}' else ''}" + +BOOTDD_VOLUME_ID ?= "boot" +BOOTDD_EXTRA_SPACE ?= "16384" + +DISK_SIGNATURE ?= "${DISK_SIGNATURE_GENERATED}" +DISK_SIGNATURE[vardepsexclude] = "DISK_SIGNATURE_GENERATED" + +build_boot_dd() { + HDDDIR="${S}/hdd/boot" + HDDIMG="${S}/hdd.image" + IMAGE=${IMGDEPLOYDIR}/${IMAGE_NAME}.hdddirect + + populate_kernel $HDDDIR + + if [ "${PCBIOS}" = "1" ]; then + syslinux_hddimg_populate $HDDDIR + fi + if [ "${EFI}" = "1" ]; then + efi_hddimg_populate $HDDDIR + fi + + BLOCKS=`du -bks $HDDDIR | cut -f 1` + BLOCKS=`expr $BLOCKS + ${BOOTDD_EXTRA_SPACE}` + + # Remove it since mkdosfs would fail when it exists + rm -f $HDDIMG + mkdosfs -n ${BOOTDD_VOLUME_ID} -S 512 -C $HDDIMG $BLOCKS + mcopy -i $HDDIMG -s $HDDDIR/* ::/ + + if [ "${PCBIOS}" = "1" ]; then + syslinux_hdddirect_install $HDDIMG + fi + chmod 644 $HDDIMG + + ROOTFSBLOCKS=`du -Lbks ${ROOTFS} | cut -f 1` + TOTALSIZE=`expr $BLOCKS + $ROOTFSBLOCKS` + END1=`expr $BLOCKS \* 1024` + END2=`expr $END1 + 512` + END3=`expr \( $ROOTFSBLOCKS \* 1024 \) + $END1` + + echo $ROOTFSBLOCKS $TOTALSIZE $END1 $END2 $END3 + rm -rf $IMAGE + dd if=/dev/zero of=$IMAGE bs=1024 seek=$TOTALSIZE count=1 + + parted $IMAGE mklabel msdos + parted $IMAGE mkpart primary fat16 0 ${END1}B + parted $IMAGE unit B mkpart primary ext2 ${END2}B ${END3}B + parted $IMAGE set 1 boot on + + parted $IMAGE print + + awk "BEGIN { printf \"$(echo ${DISK_SIGNATURE} | fold -w 2 | tac | paste -sd '' | sed 's/\(..\)/\\x&/g')\" }" | \ + dd of=$IMAGE bs=1 seek=440 conv=notrunc + + OFFSET=`expr $END2 / 512` + if [ "${PCBIOS}" = "1" ]; then + dd if=${STAGING_DATADIR}/syslinux/mbr.bin of=$IMAGE conv=notrunc + fi + + dd if=$HDDIMG of=$IMAGE conv=notrunc seek=1 bs=512 + dd if=${ROOTFS} of=$IMAGE conv=notrunc seek=$OFFSET bs=512 + + cd ${IMGDEPLOYDIR} + + ln -sf ${IMAGE_NAME}.hdddirect ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.hdddirect +} + +python do_bootdirectdisk() { + validate_disk_signature(d) + set_live_vm_vars(d, 'VM') + if d.getVar("PCBIOS") == "1": + bb.build.exec_func('build_syslinux_cfg', d) + if d.getVar("EFI") == "1": + bb.build.exec_func('build_efi_cfg', d) + bb.build.exec_func('build_boot_dd', d) +} + +def generate_disk_signature(): + import uuid + + signature = str(uuid.uuid4())[:8] + + if signature != '00000000': + return signature + else: + return 'ffffffff' + +def validate_disk_signature(d): + import re + + disk_signature = d.getVar("DISK_SIGNATURE") + + if not re.match(r'^[0-9a-fA-F]{8}$', disk_signature): + bb.fatal("DISK_SIGNATURE '%s' must be an 8 digit hex string" % disk_signature) + +DISK_SIGNATURE_GENERATED := "${@generate_disk_signature()}" + +run_qemu_img (){ + type="$1" + qemu-img convert -O $type ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.hdddirect ${IMGDEPLOYDIR}/${IMAGE_NAME}.$type + + ln -sf ${IMAGE_NAME}.$type ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.$type +} create_vmdk_image () { - qemu-img convert -O vmdk ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hdddirect ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.vmdk - ln -sf ${IMAGE_NAME}.vmdk ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.vmdk + run_qemu_img vmdk } create_vdi_image () { - qemu-img convert -O vdi ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hdddirect ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.vdi - ln -sf ${IMAGE_NAME}.vdi ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.vdi + run_qemu_img vdi } create_qcow2_image () { - qemu-img convert -O qcow2 ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hdddirect ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.qcow2 - ln -sf ${IMAGE_NAME}.qcow2 ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.qcow2 + run_qemu_img qcow2 } python do_vmimg() { - if 'vmdk' in d.getVar('IMAGE_FSTYPES', True): + if 'vmdk' in d.getVar('IMAGE_FSTYPES'): bb.build.exec_func('create_vmdk_image', d) - if 'vdi' in d.getVar('IMAGE_FSTYPES', True): + if 'vdi' in d.getVar('IMAGE_FSTYPES'): bb.build.exec_func('create_vdi_image', d) - if 'qcow2' in d.getVar('IMAGE_FSTYPES', True): + if 'qcow2' in d.getVar('IMAGE_FSTYPES'): bb.build.exec_func('create_qcow2_image', d) } -addtask vmimg after do_bootdirectdisk before do_build +addtask bootdirectdisk before do_vmimg +addtask vmimg after do_bootdirectdisk before do_image_complete do_vmimg[depends] += "qemu-native:do_populate_sysroot" - diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass index 4e66535407..405fd73c04 100644 --- a/meta/classes/image.bbclass +++ b/meta/classes/image.bbclass @@ -1,16 +1,17 @@ inherit rootfs_${IMAGE_PKGTYPE} -inherit populate_sdk_ext +# Only Linux SDKs support populate_sdk_ext, fall back to populate_sdk_base +# in the non-Linux SDK_OS case, such as mingw32 +SDKEXTCLASS ?= "${@['populate_sdk_base', 'populate_sdk_ext']['linux' in d.getVar("SDK_OS")]}" +inherit ${SDKEXTCLASS} TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}" TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY}" POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; " -inherit gzipnative - LICENSE = "MIT" PACKAGES = "" -DEPENDS += "${MLPREFIX}qemuwrapper-cross ${MLPREFIX}depmodwrapper-cross" +DEPENDS += "${MLPREFIX}qemuwrapper-cross depmodwrapper-cross" RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}" RRECOMMENDS += "${PACKAGE_INSTALL_ATTEMPTONLY}" @@ -28,7 +29,11 @@ IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs empty-root-password IMAGE_GEN_DEBUGFS ?= "0" # rootfs bootstrap install -ROOTFS_BOOTSTRAP_INSTALL = "${@bb.utils.contains("IMAGE_FEATURES", "package-management", "", "${ROOTFS_PKGMANAGE_BOOTSTRAP}",d)}" +ROOTFS_BOOTSTRAP_INSTALL = "run-postinsts" + +# These packages will be removed from a read-only rootfs after all other +# packages have been installed +ROOTFS_RO_UNNEEDED = "update-rc.d base-passwd shadow ${VIRTUAL-RUNTIME_update-alternatives} ${ROOTFS_BOOTSTRAP_INSTALL}" # packages to install from features FEATURE_INSTALL = "${@' '.join(oe.packagegroup.required_packages(oe.data.typed_value('IMAGE_FEATURES', d), d))}" @@ -44,7 +49,7 @@ FEATURE_PACKAGES_splash = "${SPLASH}" IMAGE_INSTALL_COMPLEMENTARY = '${@complementary_globs("IMAGE_FEATURES", d)}' def check_image_features(d): - valid_features = (d.getVarFlag('IMAGE_FEATURES', 'validitems', True) or "").split() + valid_features = (d.getVarFlag('IMAGE_FEATURES', 'validitems') or "").split() valid_features += d.getVarFlags('COMPLEMENTARY_GLOB').keys() for var in d: if var.startswith("PACKAGE_GROUP_"): @@ -67,6 +72,8 @@ IMAGE_INSTALL[type] = "list" export PACKAGE_INSTALL ?= "${IMAGE_INSTALL} ${ROOTFS_BOOTSTRAP_INSTALL} ${FEATURE_INSTALL}" PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}" +IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete" + # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1" @@ -94,13 +101,12 @@ do_rootfs[depends] += " \ " do_rootfs[recrdeptask] += "do_packagedata" -def command_variables(d): - return ['ROOTFS_POSTPROCESS_COMMAND','ROOTFS_PREPROCESS_COMMAND','ROOTFS_POSTINSTALL_COMMAND','OPKG_PREPROCESS_COMMANDS','OPKG_POSTPROCESS_COMMANDS','IMAGE_POSTPROCESS_COMMAND', - 'IMAGE_PREPROCESS_COMMAND','ROOTFS_POSTPROCESS_COMMAND','POPULATE_SDK_POST_HOST_COMMAND','POPULATE_SDK_POST_TARGET_COMMAND','SDK_POSTPROCESS_COMMAND','RPM_PREPROCESS_COMMANDS', - 'RPM_POSTPROCESS_COMMANDS'] +def rootfs_command_variables(d): + return ['ROOTFS_POSTPROCESS_COMMAND','ROOTFS_PREPROCESS_COMMAND','ROOTFS_POSTINSTALL_COMMAND','ROOTFS_POSTUNINSTALL_COMMAND','OPKG_PREPROCESS_COMMANDS','OPKG_POSTPROCESS_COMMANDS','IMAGE_POSTPROCESS_COMMAND', + 'IMAGE_PREPROCESS_COMMAND','RPM_PREPROCESS_COMMANDS','RPM_POSTPROCESS_COMMANDS','DEB_PREPROCESS_COMMANDS','DEB_POSTPROCESS_COMMANDS'] python () { - variables = command_variables(d) + variables = rootfs_command_variables(d) + sdk_command_variables(d) for var in variables: if d.getVar(var, False): d.setVarFlag(var, 'func', '1') @@ -108,13 +114,12 @@ python () { def rootfs_variables(d): from oe.rootfs import variable_depends - variables = ['IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPEDEP_','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE', - 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','RM_OLD_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS','SDK_OS', - 'SDK_OUTPUT','SDKPATHNATIVE','SDKTARGETSYSROOT','SDK_DIR','SDK_VENDOR','SDKIMAGE_INSTALL_COMPLEMENTARY','SDK_PACKAGE_ARCHS','SDK_OUTPUT','SDKTARGETSYSROOT','MULTILIBRE_ALLOW_REP', - 'MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS', - 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', - 'COMPRESSIONTYPES', 'IMAGE_GEN_DEBUGFS'] - variables.extend(command_variables(d)) + variables = ['IMAGE_DEVICE_TABLE','IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE', + 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', + 'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS', + 'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', + 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY'] + variables.extend(rootfs_command_variables(d)) variables.extend(variable_depends(d)) return " ".join(variables) @@ -126,7 +131,7 @@ def build_live(d): if bb.utils.contains("IMAGE_FSTYPES", "live", "live", "0", d) == "0": # live is not set but hob might set iso or hddimg d.setVar('NOISO', bb.utils.contains('IMAGE_FSTYPES', "iso", "0", "1", d)) d.setVar('NOHDD', bb.utils.contains('IMAGE_FSTYPES', "hddimg", "0", "1", d)) - if d.getVar('NOISO', True) == "0" or d.getVar('NOHDD', True) == "0": + if d.getVar('NOISO') == "0" or d.getVar('NOHDD') == "0": return "image-live" return "" return "image-live" @@ -134,40 +139,55 @@ def build_live(d): IMAGE_TYPE_live = "${@build_live(d)}" inherit ${IMAGE_TYPE_live} -IMAGE_TYPE_vm = '${@bb.utils.contains_any("IMAGE_FSTYPES", ["vmdk", "vdi", "qcow2"], "image-vm", "", d)}' +IMAGE_TYPE_vm = '${@bb.utils.contains_any("IMAGE_FSTYPES", ["vmdk", "vdi", "qcow2", "hdddirect"], "image-vm", "", d)}' inherit ${IMAGE_TYPE_vm} +IMAGE_TYPE_container = '${@bb.utils.contains("IMAGE_FSTYPES", "container", "image-container", "", d)}' +inherit ${IMAGE_TYPE_container} + +def build_uboot(d): + if 'u-boot' in (d.getVar('IMAGE_FSTYPES') or ''): + return "image_types_uboot" + else: + return "" + +IMAGE_TYPE_uboot = "${@build_uboot(d)}" +inherit ${IMAGE_TYPE_uboot} + +IMAGE_TYPE_wic = "image_types_wic" +inherit ${IMAGE_TYPE_wic} + python () { deps = " " + imagetypes_getdepends(d) d.appendVarFlag('do_rootfs', 'depends', deps) deps = "" - for dep in (d.getVar('EXTRA_IMAGEDEPENDS', True) or "").split(): + for dep in (d.getVar('EXTRA_IMAGEDEPENDS') or "").split(): deps += " %s:do_populate_sysroot" % dep - d.appendVarFlag('do_build', 'depends', deps) + d.appendVarFlag('do_image_complete', 'depends', deps) #process IMAGE_FEATURES, we must do this before runtime_mapping_rename #Check for replaces image features features = set(oe.data.typed_value('IMAGE_FEATURES', d)) remain_features = features.copy() for feature in features: - replaces = set((d.getVar("IMAGE_FEATURES_REPLACES_%s" % feature, True) or "").split()) + replaces = set((d.getVar("IMAGE_FEATURES_REPLACES_%s" % feature) or "").split()) remain_features -= replaces #Check for conflict image features for feature in remain_features: - conflicts = set((d.getVar("IMAGE_FEATURES_CONFLICTS_%s" % feature, True) or "").split()) + conflicts = set((d.getVar("IMAGE_FEATURES_CONFLICTS_%s" % feature) or "").split()) temp = conflicts & remain_features if temp: - bb.fatal("%s contains conflicting IMAGE_FEATURES %s %s" % (d.getVar('PN', True), feature, ' '.join(list(temp)))) + bb.fatal("%s contains conflicting IMAGE_FEATURES %s %s" % (d.getVar('PN'), feature, ' '.join(list(temp)))) - d.setVar('IMAGE_FEATURES', ' '.join(list(remain_features))) + d.setVar('IMAGE_FEATURES', ' '.join(sorted(list(remain_features)))) check_image_features(d) - initramfs_image = d.getVar('INITRAMFS_IMAGE', True) or "" + initramfs_image = d.getVar('INITRAMFS_IMAGE') or "" if initramfs_image != "": - d.appendVarFlag('do_build', 'depends', " %s:do_bundle_initramfs" % d.getVar('PN', True)) - d.appendVarFlag('do_bundle_initramfs', 'depends', " %s:do_rootfs" % initramfs_image) + d.appendVarFlag('do_build', 'depends', " %s:do_bundle_initramfs" % d.getVar('PN')) + d.appendVarFlag('do_bundle_initramfs', 'depends', " %s:do_image_complete" % initramfs_image) } IMAGE_CLASSES += "image_types" @@ -175,295 +195,405 @@ inherit ${IMAGE_CLASSES} IMAGE_POSTPROCESS_COMMAND ?= "" -# Zap the root password if debug-tweaks feature is not enabled -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'empty-root-password' ], "", "zap_empty_root_password ; ",d)}' - -# Allow dropbear/openssh to accept logins from accounts with an empty password string if debug-tweaks is enabled -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'allow-empty-password' ], "ssh_allow_empty_password; ", "",d)}' - -# Enable postinst logging if debug-tweaks is enabled -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'post-install-logging' ], "postinst_enable_logging; ", "",d)}' - -# Create /etc/timestamp during image construction to give a reasonably sane default time setting -ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp ; " - -# Tweak the mount options for rootfs in /etc/fstab if read-only-rootfs is enabled -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", "read_only_rootfs_hook; ", "",d)}' - -# Write manifest -IMAGE_MANIFEST = "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.manifest" -ROOTFS_POSTUNINSTALL_COMMAND =+ "write_image_manifest ; " -# Set default postinst log file -POSTINST_LOGFILE ?= "${localstatedir}/log/postinstall.log" -# Set default target for systemd images -SYSTEMD_DEFAULT_TARGET ?= '${@bb.utils.contains("IMAGE_FEATURES", "x11-base", "graphical.target", "multi-user.target", d)}' -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("DISTRO_FEATURES", "systemd", "set_systemd_default_target; ", "", d)}' - -ROOTFS_POSTPROCESS_COMMAND += 'empty_var_volatile;' - # some default locales IMAGE_LINGUAS ?= "de-de fr-fr en-gb" -LINGUAS_INSTALL ?= "${@" ".join(map(lambda s: "locale-base-%s" % s, d.getVar('IMAGE_LINGUAS', True).split()))}" +LINGUAS_INSTALL ?= "${@" ".join(map(lambda s: "locale-base-%s" % s, d.getVar('IMAGE_LINGUAS').split()))}" # Prefer image, but use the fallback files for lookups if the image ones # aren't yet available. PSEUDO_PASSWD = "${IMAGE_ROOTFS}:${STAGING_DIR_NATIVE}" -do_rootfs[dirs] = "${TOPDIR}" -do_rootfs[lockfiles] += "${IMAGE_ROOTFS}.lock" -do_rootfs[cleandirs] += "${S}" - -# Must call real_do_rootfs() from inside here, rather than as a separate -# task, so that we have a single fakeroot context for the whole process. -do_rootfs[umask] = "022" - -# A hook function to support read-only-rootfs IMAGE_FEATURES -read_only_rootfs_hook () { - # Tweak the mount option and fs_passno for rootfs in fstab - sed -i -e '/^[#[:space:]]*\/dev\/root/{s/defaults/ro/;s/\([[:space:]]*[[:digit:]]\)\([[:space:]]*\)[[:digit:]]$/\1\20/}' ${IMAGE_ROOTFS}/etc/fstab - - if ${@bb.utils.contains("DISTRO_FEATURES", "sysvinit", "true", "false", d)}; then - # Change the value of ROOTFS_READ_ONLY in /etc/default/rcS to yes - if [ -e ${IMAGE_ROOTFS}/etc/default/rcS ]; then - sed -i 's/ROOTFS_READ_ONLY=no/ROOTFS_READ_ONLY=yes/' ${IMAGE_ROOTFS}/etc/default/rcS - fi - # Run populate-volatile.sh at rootfs time to set up basic files - # and directories to support read-only rootfs. - if [ -x ${IMAGE_ROOTFS}/etc/init.d/populate-volatile.sh ]; then - ${IMAGE_ROOTFS}/etc/init.d/populate-volatile.sh - fi - # If we're using openssh and the /etc/ssh directory has no pre-generated keys, - # we should configure openssh to use the configuration file /etc/ssh/sshd_config_readonly - # and the keys under /var/run/ssh. - if [ -d ${IMAGE_ROOTFS}/etc/ssh ]; then - if [ -e ${IMAGE_ROOTFS}/etc/ssh/ssh_host_rsa_key ]; then - echo "SYSCONFDIR=/etc/ssh" >> ${IMAGE_ROOTFS}/etc/default/ssh - echo "SSHD_OPTS=" >> ${IMAGE_ROOTFS}/etc/default/ssh - else - echo "SYSCONFDIR=/var/run/ssh" >> ${IMAGE_ROOTFS}/etc/default/ssh - echo "SSHD_OPTS='-f /etc/ssh/sshd_config_readonly'" >> ${IMAGE_ROOTFS}/etc/default/ssh - fi - fi - fi - - if ${@bb.utils.contains("DISTRO_FEATURES", "systemd", "true", "false", d)}; then - # Update user database files so that services don't fail for a read-only systemd system - for conffile in ${IMAGE_ROOTFS}/usr/lib/sysusers.d/systemd.conf ${IMAGE_ROOTFS}/usr/lib/sysusers.d/systemd-remote.conf; do - [ -e $conffile ] || continue - grep -v "^#" $conffile | sed -e '/^$/d' | while read type name id comment; do - if [ "$type" = "u" ]; then - useradd_params="" - [ "$id" != "-" ] && useradd_params="$useradd_params --uid $id" - [ "$comment" != "-" ] && useradd_params="$useradd_params --comment $comment" - useradd_params="$useradd_params --system $name" - eval useradd --root ${IMAGE_ROOTFS} $useradd_params || true - elif [ "$type" = "g" ]; then - groupadd_params="" - [ "$id" != "-" ] && groupadd_params="$groupadd_params --gid $id" - groupadd_params="$groupadd_params --system $name" - eval groupadd --root ${IMAGE_ROOTFS} $groupadd_params || true - fi - done - done - fi -} +inherit rootfs-postcommands PACKAGE_EXCLUDE ??= "" PACKAGE_EXCLUDE[type] = "list" -python rootfs_process_ignore() { - excl_pkgs = d.getVar("PACKAGE_EXCLUDE", True).split() - inst_pkgs = d.getVar("PACKAGE_INSTALL", True).split() - inst_attempt_pkgs = d.getVar("PACKAGE_INSTALL_ATTEMPTONLY", True).split() +fakeroot python do_rootfs () { + from oe.rootfs import create_rootfs + from oe.manifest import create_manifest + import logging + + logger = d.getVar('BB_TASK_LOGGER', False) + if logger: + logcatcher = bb.utils.LogCatcher() + logger.addHandler(logcatcher) + else: + logcatcher = None + + # NOTE: if you add, remove or significantly refactor the stages of this + # process then you should recalculate the weightings here. This is quite + # easy to do - just change the MultiStageProgressReporter line temporarily + # to pass debug=True as the last parameter and you'll get a printout of + # the weightings as well as a map to the lines where next_stage() was + # called. Of course this isn't critical, but it helps to keep the progress + # reporting accurate. + stage_weights = [1, 203, 354, 186, 65, 4228, 1, 353, 49, 330, 382, 23, 1] + progress_reporter = bb.progress.MultiStageProgressReporter(d, stage_weights) + progress_reporter.next_stage() + + # Handle package exclusions + excl_pkgs = d.getVar("PACKAGE_EXCLUDE").split() + inst_pkgs = d.getVar("PACKAGE_INSTALL").split() + inst_attempt_pkgs = d.getVar("PACKAGE_INSTALL_ATTEMPTONLY").split() d.setVar('PACKAGE_INSTALL_ORIG', ' '.join(inst_pkgs)) d.setVar('PACKAGE_INSTALL_ATTEMPTONLY', ' '.join(inst_attempt_pkgs)) for pkg in excl_pkgs: if pkg in inst_pkgs: - bb.warn("Package %s, set to be excluded, is in %s PACKAGE_INSTALL (%s). It will be removed from the list." % (pkg, d.getVar('PN', True), inst_pkgs)) + bb.warn("Package %s, set to be excluded, is in %s PACKAGE_INSTALL (%s). It will be removed from the list." % (pkg, d.getVar('PN'), inst_pkgs)) inst_pkgs.remove(pkg) if pkg in inst_attempt_pkgs: - bb.warn("Package %s, set to be excluded, is in %s PACKAGE_INSTALL_ATTEMPTONLY (%s). It will be removed from the list." % (pkg, d.getVar('PN', True), inst_pkgs)) + bb.warn("Package %s, set to be excluded, is in %s PACKAGE_INSTALL_ATTEMPTONLY (%s). It will be removed from the list." % (pkg, d.getVar('PN'), inst_pkgs)) inst_attempt_pkgs.remove(pkg) d.setVar("PACKAGE_INSTALL", ' '.join(inst_pkgs)) d.setVar("PACKAGE_INSTALL_ATTEMPTONLY", ' '.join(inst_attempt_pkgs)) -} -do_rootfs[prefuncs] += "rootfs_process_ignore" -# We have to delay the runtime_mapping_rename until just before rootfs runs -# otherwise, the multilib renaming could step in and squash any fixups that -# may have occurred. -python rootfs_runtime_mapping() { - pn = d.getVar('PN', True) + # Ensure we handle package name remapping + # We have to delay the runtime_mapping_rename until just before rootfs runs + # otherwise, the multilib renaming could step in and squash any fixups that + # may have occurred. + pn = d.getVar('PN') runtime_mapping_rename("PACKAGE_INSTALL", pn, d) runtime_mapping_rename("PACKAGE_INSTALL_ATTEMPTONLY", pn, d) runtime_mapping_rename("BAD_RECOMMENDATIONS", pn, d) -} -do_rootfs[prefuncs] += "rootfs_runtime_mapping" -fakeroot python do_rootfs () { - from oe.rootfs import create_rootfs - from oe.image import create_image - from oe.manifest import create_manifest - - # generate the initial manifest + # Generate the initial manifest create_manifest(d) + progress_reporter.next_stage() + # generate rootfs - create_rootfs(d) + create_rootfs(d, progress_reporter=progress_reporter, logcatcher=logcatcher) - # generate final images - create_image(d) + progress_reporter.finish() } +do_rootfs[dirs] = "${TOPDIR}" +do_rootfs[cleandirs] += "${S} ${IMGDEPLOYDIR}" +do_rootfs[umask] = "022" +addtask rootfs before do_build after do_prepare_recipe_sysroot -insert_feed_uris () { - - echo "Building feeds for [${DISTRO}].." - - for line in ${FEED_URIS} - do - # strip leading and trailing spaces/tabs, then split into name and uri - line_clean="`echo "$line"|sed 's/^[ \t]*//;s/[ \t]*$//'`" - feed_name="`echo "$line_clean" | sed -n 's/\(.*\)##\(.*\)/\1/p'`" - feed_uri="`echo "$line_clean" | sed -n 's/\(.*\)##\(.*\)/\2/p'`" - - echo "Added $feed_name feed with URL $feed_uri" - - # insert new feed-sources - echo "src/gz $feed_name $feed_uri" >> ${IMAGE_ROOTFS}/etc/opkg/${feed_name}-feed.conf - done -} +fakeroot python do_image () { + from oe.utils import execute_pre_post_process -MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${libexecdir}|${sysconfdir}|${nonarch_base_libdir}/udev|/lib/modules/[^/]*/modules.*|" -MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" -MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib" + pre_process_cmds = d.getVar("IMAGE_PREPROCESS_COMMAND") -# This function is intended to disallow empty root password if 'debug-tweaks' is not in IMAGE_FEATURES. -zap_empty_root_password () { - if [ -e ${IMAGE_ROOTFS}/etc/shadow ]; then - sed -i 's%^root::%root:*:%' ${IMAGE_ROOTFS}/etc/shadow - fi - if [ -e ${IMAGE_ROOTFS}/etc/passwd ]; then - sed -i 's%^root::%root:*:%' ${IMAGE_ROOTFS}/etc/passwd - fi -} - -# allow dropbear/openssh to accept root logins and logins from accounts with an empty password string -ssh_allow_empty_password () { - if [ -e ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config ]; then - sed -i 's/^[#[:space:]]*PermitRootLogin.*/PermitRootLogin yes/' ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config - sed -i 's/^[#[:space:]]*PermitEmptyPasswords.*/PermitEmptyPasswords yes/' ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config - fi - - if [ -e ${IMAGE_ROOTFS}${sbindir}/dropbear ] ; then - if grep -q DROPBEAR_EXTRA_ARGS ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear 2>/dev/null ; then - if ! grep -q "DROPBEAR_EXTRA_ARGS=.*-B" ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear ; then - sed -i 's/^DROPBEAR_EXTRA_ARGS="*\([^"]*\)"*/DROPBEAR_EXTRA_ARGS="\1 -B"/' ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear - fi - else - printf '\nDROPBEAR_EXTRA_ARGS="-B"\n' >> ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear - fi - fi - - if [ -d ${IMAGE_ROOTFS}${sysconfdir}/pam.d ] ; then - sed -i 's/nullok_secure/nullok/' ${IMAGE_ROOTFS}${sysconfdir}/pam.d/* - fi + execute_pre_post_process(d, pre_process_cmds) } +do_image[dirs] = "${TOPDIR}" +do_image[umask] = "022" +addtask do_image after do_rootfs before do_build -# Disable DNS lookups, the SSH_DISABLE_DNS_LOOKUP can be overridden to allow -# distros to choose not to take this change -SSH_DISABLE_DNS_LOOKUP ?= " ssh_disable_dns_lookup ; " -ROOTFS_POSTPROCESS_COMMAND_append_qemuall = "${SSH_DISABLE_DNS_LOOKUP}" -ssh_disable_dns_lookup () { - if [ -e ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config ]; then - sed -i -e 's:#UseDNS yes:UseDNS no:' ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config - fi -} +fakeroot python do_image_complete () { + from oe.utils import execute_pre_post_process -# Enable postinst logging if debug-tweaks is enabled -postinst_enable_logging () { - mkdir -p ${IMAGE_ROOTFS}${sysconfdir}/default - echo "POSTINST_LOGGING=1" >> ${IMAGE_ROOTFS}${sysconfdir}/default/postinst - echo "LOGFILE=${POSTINST_LOGFILE}" >> ${IMAGE_ROOTFS}${sysconfdir}/default/postinst -} + post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND") -# Modify systemd default target -set_systemd_default_target () { - if [ -d ${IMAGE_ROOTFS}${sysconfdir}/systemd/system -a -e ${IMAGE_ROOTFS}${systemd_unitdir}/system/${SYSTEMD_DEFAULT_TARGET} ]; then - ln -sf ${systemd_unitdir}/system/${SYSTEMD_DEFAULT_TARGET} ${IMAGE_ROOTFS}${sysconfdir}/systemd/system/default.target - fi + execute_pre_post_process(d, post_process_cmds) } - -# If /var/volatile is not empty, we have seen problems where programs such as the -# journal make assumptions based on the contents of /var/volatile. The journal -# would then write to /var/volatile before it was mounted, thus hiding the -# items previously written. +do_image_complete[dirs] = "${TOPDIR}" +do_image_complete[umask] = "022" +SSTATETASKS += "do_image_complete" +SSTATE_SKIP_CREATION_task-image-complete = '1' +do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" +do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" +do_image_complete[stamp-extra-info] = "${MACHINE}" +addtask do_image_complete after do_image before do_build + +# Add image-level QA/sanity checks to IMAGE_QA_COMMANDS # -# This change is to attempt to fix those types of issues in a way that doesn't -# affect users that may not be using /var/volatile. -empty_var_volatile () { - if [ -e ${IMAGE_ROOTFS}/etc/fstab ]; then - match=`awk '$1 !~ "#" && $2 ~ /\/var\/volatile/{print $2}' ${IMAGE_ROOTFS}/etc/fstab 2> /dev/null` - if [ -n "$match" ]; then - find ${IMAGE_ROOTFS}/var/volatile -mindepth 1 -delete - fi - fi -} - -# Turn any symbolic /sbin/init link into a file -remove_init_link () { - if [ -h ${IMAGE_ROOTFS}/sbin/init ]; then - LINKFILE=${IMAGE_ROOTFS}`readlink ${IMAGE_ROOTFS}/sbin/init` - rm ${IMAGE_ROOTFS}/sbin/init - cp $LINKFILE ${IMAGE_ROOTFS}/sbin/init - fi -} - -make_zimage_symlink_relative () { - if [ -L ${IMAGE_ROOTFS}/boot/zImage ]; then - (cd ${IMAGE_ROOTFS}/boot/ && for i in `ls zImage-* | sort`; do ln -sf $i zImage; done) - fi +# IMAGE_QA_COMMANDS += " \ +# image_check_everything_ok \ +# " +# This task runs all functions in IMAGE_QA_COMMANDS after the image +# construction has completed in order to validate the resulting image. +fakeroot python do_image_qa () { + from oe.utils import ImageQAFailed + + qa_cmds = (d.getVar('IMAGE_QA_COMMANDS') or '').split() + qamsg = "" + + for cmd in qa_cmds: + try: + bb.build.exec_func(cmd, d) + except oe.utils.ImageQAFailed as e: + qamsg = qamsg + '\tImage QA function %s failed: %s\n' % (e.name, e.description) + except bb.build.FuncFailed as e: + qamsg = qamsg + '\tImage QA function %s failed' % e.name + if e.logfile: + qamsg = qamsg + ' (log file is located at %s)' % e.logfile + qamsg = qamsg + '\n' + + if qamsg: + imgname = d.getVar('IMAGE_NAME') + bb.fatal("QA errors found whilst validating image: %s\n%s" % (imgname, qamsg)) } - -python write_image_manifest () { - from oe.rootfs import image_list_installed_packages - with open(d.getVar('IMAGE_MANIFEST', True), 'w+') as image_manifest: - image_manifest.write(image_list_installed_packages(d, 'ver')) - image_manifest.write("\n") +addtask do_image_qa after do_image_complete before do_build + +def setup_debugfs_variables(d): + d.appendVar('IMAGE_ROOTFS', '-dbg') + d.appendVar('IMAGE_LINK_NAME', '-dbg') + d.appendVar('IMAGE_NAME','-dbg') + d.setVar('IMAGE_BUILDING_DEBUGFS', 'true') + debugfs_image_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS') + if debugfs_image_fstypes: + d.setVar('IMAGE_FSTYPES', debugfs_image_fstypes) + +python setup_debugfs () { + setup_debugfs_variables(d) } -# Can be use to create /etc/timestamp during image construction to give a reasonably -# sane default time setting -rootfs_update_timestamp () { - date -u +%4Y%2m%2d%2H%2M%2S >${IMAGE_ROOTFS}/etc/timestamp +python () { + vardeps = set() + # We allow CONVERSIONTYPES to have duplicates. That avoids breaking + # derived distros when OE-core or some other layer independently adds + # the same type. There is still only one command for each type, but + # presumably the commands will do the same when the type is the same, + # even when added in different places. + # + # Without de-duplication, gen_conversion_cmds() below + # would create the same compression command multiple times. + ctypes = set(d.getVar('CONVERSIONTYPES').split()) + old_overrides = d.getVar('OVERRIDES', False) + + def _image_base_type(type): + basetype = type + for ctype in ctypes: + if type.endswith("." + ctype): + basetype = type[:-len("." + ctype)] + break + + if basetype != type: + # New base type itself might be generated by a conversion command. + basetype = _image_base_type(basetype) + + return basetype + + basetypes = {} + alltypes = d.getVar('IMAGE_FSTYPES').split() + typedeps = {} + + if d.getVar('IMAGE_GEN_DEBUGFS') == "1": + debugfs_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS').split() + for t in debugfs_fstypes: + alltypes.append("debugfs_" + t) + + def _add_type(t): + baset = _image_base_type(t) + input_t = t + if baset not in basetypes: + basetypes[baset]= [] + if t not in basetypes[baset]: + basetypes[baset].append(t) + debug = "" + if t.startswith("debugfs_"): + t = t[8:] + debug = "debugfs_" + deps = (d.getVar('IMAGE_TYPEDEP_' + t) or "").split() + vardeps.add('IMAGE_TYPEDEP_' + t) + if baset not in typedeps: + typedeps[baset] = set() + deps = [debug + dep for dep in deps] + for dep in deps: + if dep not in alltypes: + alltypes.append(dep) + _add_type(dep) + basedep = _image_base_type(dep) + typedeps[baset].add(basedep) + + if baset != input_t: + _add_type(baset) + + for t in alltypes[:]: + _add_type(t) + + d.appendVarFlag('do_image', 'vardeps', ' '.join(vardeps)) + + maskedtypes = (d.getVar('IMAGE_TYPES_MASKED') or "").split() + maskedtypes = [dbg + t for t in maskedtypes for dbg in ("", "debugfs_")] + + for t in basetypes: + vardeps = set() + cmds = [] + subimages = [] + realt = t + + if t in maskedtypes: + continue + + localdata = bb.data.createCopy(d) + debug = "" + if t.startswith("debugfs_"): + setup_debugfs_variables(localdata) + debug = "setup_debugfs " + realt = t[8:] + localdata.setVar('OVERRIDES', '%s:%s' % (realt, old_overrides)) + localdata.setVar('type', realt) + # Delete DATETIME so we don't expand any references to it now + # This means the task's hash can be stable rather than having hardcoded + # date/time values. It will get expanded at execution time. + # Similarly TMPDIR since otherwise we see QA stamp comparision problems + localdata.delVar('DATETIME') + localdata.delVar('TMPDIR') + + image_cmd = localdata.getVar("IMAGE_CMD") + vardeps.add('IMAGE_CMD_' + realt) + if image_cmd: + cmds.append("\t" + image_cmd) + else: + bb.fatal("No IMAGE_CMD defined for IMAGE_FSTYPES entry '%s' - possibly invalid type name or missing support class" % t) + cmds.append(localdata.expand("\tcd ${IMGDEPLOYDIR}")) + + # Since a copy of IMAGE_CMD_xxx will be inlined within do_image_xxx, + # prevent a redundant copy of IMAGE_CMD_xxx being emitted as a function. + d.delVarFlag('IMAGE_CMD_' + realt, 'func') + + rm_tmp_images = set() + def gen_conversion_cmds(bt): + for ctype in ctypes: + if bt[bt.find('.') + 1:] == ctype: + type = bt[0:-len(ctype) - 1] + if type.startswith("debugfs_"): + type = type[8:] + # Create input image first. + gen_conversion_cmds(type) + localdata.setVar('type', type) + cmd = "\t" + (localdata.getVar("CONVERSION_CMD_" + ctype) or localdata.getVar("COMPRESS_CMD_" + ctype)) + if cmd not in cmds: + cmds.append(cmd) + vardeps.add('CONVERSION_CMD_' + ctype) + vardeps.add('COMPRESS_CMD_' + ctype) + subimage = type + "." + ctype + if subimage not in subimages: + subimages.append(subimage) + if type not in alltypes: + rm_tmp_images.add(localdata.expand("${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}")) + + for bt in basetypes[t]: + gen_conversion_cmds(bt) + + localdata.setVar('type', realt) + if t not in alltypes: + rm_tmp_images.add(localdata.expand("${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}")) + else: + subimages.append(realt) + + # Clean up after applying all conversion commands. Some of them might + # use the same input, therefore we cannot delete sooner without applying + # some complex dependency analysis. + for image in rm_tmp_images: + cmds.append("\trm " + image) + + after = 'do_image' + for dep in typedeps[t]: + after += ' do_image_%s' % dep.replace("-", "_").replace(".", "_") + + t = t.replace("-", "_").replace(".", "_") + + d.setVar('do_image_%s' % t, '\n'.join(cmds)) + d.setVarFlag('do_image_%s' % t, 'func', '1') + d.setVarFlag('do_image_%s' % t, 'fakeroot', '1') + d.setVarFlag('do_image_%s' % t, 'prefuncs', debug + 'set_image_size') + d.setVarFlag('do_image_%s' % t, 'postfuncs', 'create_symlinks') + d.setVarFlag('do_image_%s' % t, 'subimages', ' '.join(subimages)) + d.appendVarFlag('do_image_%s' % t, 'vardeps', ' '.join(vardeps)) + d.appendVarFlag('do_image_%s' % t, 'vardepsexclude', 'DATETIME') + + bb.debug(2, "Adding type %s before %s, after %s" % (t, 'do_image_complete', after)) + bb.build.addtask('do_image_%s' % t, 'do_image_complete', after, d) } -# Prevent X from being started -rootfs_no_x_startup () { - if [ -f ${IMAGE_ROOTFS}/etc/init.d/xserver-nodm ]; then - chmod a-x ${IMAGE_ROOTFS}/etc/init.d/xserver-nodm - fi +# +# Compute the rootfs size +# +def get_rootfs_size(d): + import subprocess + + rootfs_alignment = int(d.getVar('IMAGE_ROOTFS_ALIGNMENT')) + overhead_factor = float(d.getVar('IMAGE_OVERHEAD_FACTOR')) + rootfs_req_size = int(d.getVar('IMAGE_ROOTFS_SIZE')) + rootfs_extra_space = eval(d.getVar('IMAGE_ROOTFS_EXTRA_SPACE')) + rootfs_maxsize = d.getVar('IMAGE_ROOTFS_MAXSIZE') + image_fstypes = d.getVar('IMAGE_FSTYPES') or '' + initramfs_fstypes = d.getVar('INITRAMFS_FSTYPES') or '' + initramfs_maxsize = d.getVar('INITRAMFS_MAXSIZE') + + output = subprocess.check_output(['du', '-ks', + d.getVar('IMAGE_ROOTFS')]) + size_kb = int(output.split()[0]) + base_size = size_kb * overhead_factor + base_size = max(base_size, rootfs_req_size) + rootfs_extra_space + + if base_size != int(base_size): + base_size = int(base_size + 1) + else: + base_size = int(base_size) + + base_size += rootfs_alignment - 1 + base_size -= base_size % rootfs_alignment + + # Do not check image size of the debugfs image. This is not supposed + # to be deployed, etc. so it doesn't make sense to limit the size + # of the debug. + if (d.getVar('IMAGE_BUILDING_DEBUGFS') or "") == "true": + return base_size + + # Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set) + if rootfs_maxsize: + rootfs_maxsize_int = int(rootfs_maxsize) + if base_size > rootfs_maxsize_int: + bb.fatal("The rootfs size %d(K) overrides IMAGE_ROOTFS_MAXSIZE: %d(K)" % \ + (base_size, rootfs_maxsize_int)) + + # Check the initramfs size against INITRAMFS_MAXSIZE (if set) + if image_fstypes == initramfs_fstypes != '' and initramfs_maxsize: + initramfs_maxsize_int = int(initramfs_maxsize) + if base_size > initramfs_maxsize_int: + bb.error("The initramfs size %d(K) overrides INITRAMFS_MAXSIZE: %d(K)" % \ + (base_size, initramfs_maxsize_int)) + bb.error("You can set INITRAMFS_MAXSIZE a larger value. Usually, it should") + bb.fatal("be less than 1/2 of ram size, or you may fail to boot it.\n") + return base_size + +python set_image_size () { + rootfs_size = get_rootfs_size(d) + d.setVar('ROOTFS_SIZE', str(rootfs_size)) + d.setVarFlag('ROOTFS_SIZE', 'export', '1') } -rootfs_trim_schemas () { - for schema in ${IMAGE_ROOTFS}/etc/gconf/schemas/*.schemas - do - # Need this in case no files exist - if [ -e $schema ]; then - oe-trim-schemas $schema > $schema.new - mv $schema.new $schema - fi - done +# +# Create symlinks to the newly created image +# +python create_symlinks() { + + deploy_dir = d.getVar('IMGDEPLOYDIR') + img_name = d.getVar('IMAGE_NAME') + link_name = d.getVar('IMAGE_LINK_NAME') + manifest_name = d.getVar('IMAGE_MANIFEST') + taskname = d.getVar("BB_CURRENTTASK") + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split() + imgsuffix = d.getVarFlag("do_" + taskname, 'imgsuffix') or d.expand("${IMAGE_NAME_SUFFIX}.") + + if not link_name: + return + for type in subimages: + dst = os.path.join(deploy_dir, link_name + "." + type) + src = img_name + imgsuffix + type + if os.path.exists(os.path.join(deploy_dir, src)): + bb.note("Creating symlink: %s -> %s" % (dst, src)) + if os.path.islink(dst): + os.remove(dst) + os.symlink(src, dst) + else: + bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src)) } -# Make any absolute links in a sysroot relative -rootfs_sysroot_relativelinks () { - sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT} -} +MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${libexecdir}|${sysconfdir}|${nonarch_base_libdir}/udev|/lib/modules/[^/]*/modules.*|" +MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" +MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib" do_fetch[noexec] = "1" do_unpack[noexec] = "1" @@ -471,15 +601,14 @@ do_patch[noexec] = "1" do_configure[noexec] = "1" do_compile[noexec] = "1" do_install[noexec] = "1" -do_populate_sysroot[noexec] = "1" +deltask do_populate_sysroot do_package[noexec] = "1" -do_package_qa[noexec] = "1" +deltask do_package_qa do_packagedata[noexec] = "1" do_package_write_ipk[noexec] = "1" do_package_write_deb[noexec] = "1" do_package_write_rpm[noexec] = "1" -addtask rootfs before do_build # Allow the kernel to be repacked with the initramfs and boot image file as a single file do_bundle_initramfs[depends] += "virtual/kernel:do_bundle_initramfs" do_bundle_initramfs[nostamp] = "1" @@ -487,4 +616,4 @@ do_bundle_initramfs[noexec] = "1" do_bundle_initramfs () { : } -addtask bundle_initramfs after do_rootfs +addtask bundle_initramfs after do_image_complete diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass index ecb066bf99..7749b00098 100644 --- a/meta/classes/image_types.bbclass +++ b/meta/classes/image_types.bbclass @@ -1,3 +1,8 @@ +# IMAGE_NAME is the base name for everything produced when building images. +# The actual image that contains the rootfs has an additional suffix (.rootfs +# by default) followed by additional suffices which describe the format (.ext4, +# .ext4.xz, etc.). +IMAGE_NAME_SUFFIX ??= ".rootfs" # The default aligment of the size of the rootfs is set to 1KiB. In case # you're using the SD card emulation of a QEMU system simulator you may @@ -6,39 +11,48 @@ IMAGE_ROOTFS_ALIGNMENT ?= "1" def imagetypes_getdepends(d): def adddep(depstr, deps): - for i in (depstr or "").split(): - if i not in deps: - deps.append(i) - - deps = [] - ctypes = d.getVar('COMPRESSIONTYPES', True).split() - for type in (d.getVar('IMAGE_FSTYPES', True) or "").split(): - if type in ["vmdk", "vdi", "qcow2", "live", "iso", "hddimg"]: - type = "ext4" - basetype = type - for ctype in ctypes: - if type.endswith("." + ctype): - basetype = type[:-len("." + ctype)] - adddep(d.getVar("COMPRESS_DEPENDS_%s" % ctype, True), deps) - break - for typedepends in (d.getVar("IMAGE_TYPEDEP_%s" % basetype, True) or "").split(): - adddep(d.getVar('IMAGE_DEPENDS_%s' % typedepends, True) , deps) - adddep(d.getVar('IMAGE_DEPENDS_%s' % basetype, True) , deps) - - depstr = "" - for dep in deps: - depstr += " " + dep + ":do_populate_sysroot" - return depstr - - -XZ_COMPRESSION_LEVEL ?= "-e -6" + for d in (depstr or "").split(): + # Add task dependency if not already present + if ":" not in d: + d += ":do_populate_sysroot" + deps.add(d) + + # Take a type in the form of foo.bar.car and split it into the items + # needed for the image deps "foo", and the conversion deps ["bar", "car"] + def split_types(typestring): + types = typestring.split(".") + return types[0], types[1:] + + fstypes = set((d.getVar('IMAGE_FSTYPES') or "").split()) + fstypes |= set((d.getVar('IMAGE_FSTYPES_DEBUGFS') or "").split()) + + deps = set() + for typestring in fstypes: + basetype, resttypes = split_types(typestring) + adddep(d.getVar('IMAGE_DEPENDS_%s' % basetype) , deps) + + for typedepends in (d.getVar("IMAGE_TYPEDEP_%s" % basetype) or "").split(): + base, rest = split_types(typedepends) + adddep(d.getVar('IMAGE_DEPENDS_%s' % base) , deps) + resttypes += rest + + for ctype in resttypes: + adddep(d.getVar("CONVERSION_DEPENDS_%s" % ctype), deps) + adddep(d.getVar("COMPRESS_DEPENDS_%s" % ctype), deps) + + # Sort the set so that ordering is consistant + return " ".join(sorted(deps)) + +XZ_COMPRESSION_LEVEL ?= "-3" XZ_INTEGRITY_CHECK ?= "crc32" XZ_THREADS ?= "-T 0" +ZIP_COMPRESSION_LEVEL ?= "-9" + JFFS2_SUM_EXTRA_ARGS ?= "" -IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.jffs2 ${EXTRA_IMAGECMD}" +IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}" -IMAGE_CMD_cramfs = "mkfs.cramfs ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cramfs ${EXTRA_IMAGECMD}" +IMAGE_CMD_cramfs = "mkfs.cramfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cramfs ${EXTRA_IMAGECMD}" oe_mkext234fs () { fstype=$1 @@ -58,8 +72,10 @@ oe_mkext234fs () { eval COUNT=\"$MIN_COUNT\" fi # Create a sparse image block - dd if=/dev/zero of=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.$fstype seek=$ROOTFS_SIZE count=$COUNT bs=1024 - mkfs.$fstype -F $extra_imagecmd ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.$fstype -d ${IMAGE_ROOTFS} + dd if=/dev/zero of=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.$fstype seek=$ROOTFS_SIZE count=$COUNT bs=1024 + mkfs.$fstype -F $extra_imagecmd ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.$fstype -d ${IMAGE_ROOTFS} + # Error codes 0-3 indicate successfull operation of fsck (no errors or errors corrected) + fsck.$fstype -pvfD ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.$fstype || [ $? -le 3 ] } IMAGE_CMD_ext2 = "oe_mkext234fs ext2 ${EXTRA_IMAGECMD}" @@ -68,17 +84,18 @@ IMAGE_CMD_ext4 = "oe_mkext234fs ext4 ${EXTRA_IMAGECMD}" MIN_BTRFS_SIZE ?= "16384" IMAGE_CMD_btrfs () { - if [ ${ROOTFS_SIZE} -gt ${MIN_BTRFS_SIZE} ]; then - dd if=/dev/zero of=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.btrfs count=${ROOTFS_SIZE} bs=1024 - mkfs.btrfs ${EXTRA_IMAGECMD} -r ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.btrfs - else - bbfatal "Rootfs is too small for BTRFS (Rootfs Actual Size: ${ROOTFS_SIZE}, BTRFS Minimum Size: ${MIN_BTRFS_SIZE})" + size=${ROOTFS_SIZE} + if [ ${size} -lt ${MIN_BTRFS_SIZE} ] ; then + size=${MIN_BTRFS_SIZE} + bbwarn "Rootfs size is too small for BTRFS. Filesystem will be extended to ${size}K" fi + dd if=/dev/zero of=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.btrfs count=${size} bs=1024 + mkfs.btrfs ${EXTRA_IMAGECMD} -r ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.btrfs } -IMAGE_CMD_squashfs = "mksquashfs ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.squashfs ${EXTRA_IMAGECMD} -noappend" -IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.squashfs-xz ${EXTRA_IMAGECMD} -noappend -comp xz" -IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo" +IMAGE_CMD_squashfs = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs ${EXTRA_IMAGECMD} -noappend" +IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-xz ${EXTRA_IMAGECMD} -noappend -comp xz" +IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo" # By default, tar from the host is used, which can be quite old. If # you need special parameters (like --xattrs) which are only supported @@ -91,28 +108,36 @@ IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE # In practice, it turned out to be not needed when creating archives and # required when extracting, but it seems prudent to use it in both cases. IMAGE_CMD_TAR ?= "tar" -IMAGE_CMD_tar = "${IMAGE_CMD_TAR} -cvf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.tar -C ${IMAGE_ROOTFS} ." +IMAGE_CMD_tar = "${IMAGE_CMD_TAR} -cvf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} ." -do_rootfs[cleandirs] += "${WORKDIR}/cpio_append" +do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append" IMAGE_CMD_cpio () { - (cd ${IMAGE_ROOTFS} && find . | cpio -o -H newc >${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio) - if [ ! -L ${IMAGE_ROOTFS}/init -a ! -e ${IMAGE_ROOTFS}/init ]; then - if [ -L ${IMAGE_ROOTFS}/sbin/init -o -e ${IMAGE_ROOTFS}/sbin/init ]; then - ln -sf /sbin/init ${WORKDIR}/cpio_append/init - else - touch ${WORKDIR}/cpio_append/init + (cd ${IMAGE_ROOTFS} && find . | cpio -o -H newc >${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cpio) + # We only need the /init symlink if we're building the real + # image. The -dbg image doesn't need it! By being clever + # about this we also avoid 'touch' below failing, as it + # might be trying to touch /sbin/init on the host since both + # the normal and the -dbg image share the same WORKDIR + if [ "${IMAGE_BUILDING_DEBUGFS}" != "true" ]; then + if [ ! -L ${IMAGE_ROOTFS}/init ] && [ ! -e ${IMAGE_ROOTFS}/init ]; then + if [ -L ${IMAGE_ROOTFS}/sbin/init ] || [ -e ${IMAGE_ROOTFS}/sbin/init ]; then + ln -sf /sbin/init ${WORKDIR}/cpio_append/init + else + touch ${WORKDIR}/cpio_append/init + fi + (cd ${WORKDIR}/cpio_append && echo ./init | cpio -oA -H newc -F ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cpio) fi - (cd ${WORKDIR}/cpio_append && echo ./init | cpio -oA -H newc -F ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio) fi } -ELF_KERNEL ?= "${STAGING_DIR_HOST}/usr/src/kernel/${KERNEL_IMAGETYPE}" +ELF_KERNEL ?= "${DEPLOY_DIR_IMAGE}/${KERNEL_IMAGETYPE}" ELF_APPEND ?= "ramdisk_size=32768 root=/dev/ram0 rw console=" IMAGE_CMD_elf () { - test -f ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.elf && rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.elf - mkelfImage --kernel=${ELF_KERNEL} --initrd=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio.gz --output=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.elf --append='${ELF_APPEND}' ${EXTRA_IMAGECMD} + test -f ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.elf && rm -f ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.elf + mkelfImage --kernel=${ELF_KERNEL} --initrd=${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.cpio.gz --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.elf --append='${ELF_APPEND}' ${EXTRA_IMAGECMD} } + IMAGE_TYPEDEP_elf = "cpio.gz" UBI_VOLNAME ?= "${MACHINE}-rootfs" @@ -120,32 +145,38 @@ UBI_VOLNAME ?= "${MACHINE}-rootfs" multiubi_mkfs() { local mkubifs_args="$1" local ubinize_args="$2" - local vname="_$3" - - echo \[ubifs\] > ubinize${vname}.cfg - echo mode=ubi >> ubinize${vname}.cfg - echo image=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${vname}.rootfs.ubifs >> ubinize${vname}.cfg - echo vol_id=0 >> ubinize${vname}.cfg - echo vol_type=dynamic >> ubinize${vname}.cfg - echo vol_name=${UBI_VOLNAME} >> ubinize${vname}.cfg - echo vol_flags=autoresize >> ubinize${vname}.cfg - mkfs.ubifs -r ${IMAGE_ROOTFS} -o ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${vname}.rootfs.ubifs ${mkubifs_args} - ubinize -o ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${vname}.rootfs.ubi ${ubinize_args} ubinize${vname}.cfg + if [ -z "$3" ]; then + local vname="" + else + local vname="_$3" + fi - # Cleanup cfg file - mv ubinize${vname}.cfg ${DEPLOY_DIR_IMAGE}/ + echo \[ubifs\] > ubinize${vname}-${IMAGE_NAME}.cfg + echo mode=ubi >> ubinize${vname}-${IMAGE_NAME}.cfg + echo image=${IMGDEPLOYDIR}/${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubifs >> ubinize${vname}-${IMAGE_NAME}.cfg + echo vol_id=0 >> ubinize${vname}-${IMAGE_NAME}.cfg + echo vol_type=dynamic >> ubinize${vname}-${IMAGE_NAME}.cfg + echo vol_name=${UBI_VOLNAME} >> ubinize${vname}-${IMAGE_NAME}.cfg + echo vol_flags=autoresize >> ubinize${vname}-${IMAGE_NAME}.cfg + mkfs.ubifs -r ${IMAGE_ROOTFS} -o ${IMGDEPLOYDIR}/${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubifs ${mkubifs_args} + ubinize -o ${IMGDEPLOYDIR}/${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubi ${ubinize_args} ubinize${vname}-${IMAGE_NAME}.cfg - # Create own symlink - cd ${DEPLOY_DIR_IMAGE} - if [ -e ${IMAGE_NAME}${vname}.rootfs.ubifs ]; then - ln -sf ${IMAGE_NAME}${vname}.rootfs.ubifs \ - ${IMAGE_LINK_NAME}${vname}.ubifs - fi - if [ -e ${IMAGE_NAME}${vname}.rootfs.ubi ]; then - ln -sf ${IMAGE_NAME}${vname}.rootfs.ubi \ - ${IMAGE_LINK_NAME}${vname}.ubi + # Cleanup cfg file + mv ubinize${vname}-${IMAGE_NAME}.cfg ${IMGDEPLOYDIR}/ + + # Create own symlinks for 'named' volumes + if [ -n "$vname" ]; then + cd ${IMGDEPLOYDIR} + if [ -e ${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubifs ]; then + ln -sf ${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubifs \ + ${IMAGE_LINK_NAME}${vname}.ubifs + fi + if [ -e ${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubi ]; then + ln -sf ${IMAGE_NAME}${vname}${IMAGE_NAME_SUFFIX}.ubi \ + ${IMAGE_LINK_NAME}${vname}.ubi + fi + cd - fi - cd - } IMAGE_CMD_multiubi () { @@ -158,20 +189,12 @@ IMAGE_CMD_multiubi () { done } -IMAGE_CMD_ubi = "multiubi_mkfs "${MKUBIFS_ARGS}" "${UBINIZE_ARGS}" "${UBI_VOLNAME}"" - -IMAGE_CMD_ubifs = "mkfs.ubifs -r ${IMAGE_ROOTFS} -o ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ubifs ${MKUBIFS_ARGS}" - -IMAGE_CMD_wic () { - out=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME} - wks=${FILE_DIRNAME}/${IMAGE_BASENAME}.${MACHINE}.wks - [ -e $wks ] || wks=${FILE_DIRNAME}/${IMAGE_BASENAME}.wks - [ -e $wks ] || bbfatal "Kiskstart file $wks doesn't exist" - BUILDDIR=${TOPDIR} wic create $wks --vars ${STAGING_DIR_TARGET}/imgdata/ -e ${IMAGE_BASENAME} -o $out/ - mv $out/build/${IMAGE_BASENAME}*.direct $out.rootfs.wic - rm -rf $out/ +IMAGE_CMD_ubi () { + multiubi_mkfs "${MKUBIFS_ARGS}" "${UBINIZE_ARGS}" } +IMAGE_CMD_ubifs = "mkfs.ubifs -r ${IMAGE_ROOTFS} -o ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ubifs ${MKUBIFS_ARGS}" + EXTRA_IMAGECMD = "" inherit siteinfo @@ -183,7 +206,7 @@ EXTRA_IMAGECMD_jffs2 ?= "--pad ${JFFS2_ENDIANNESS} --eraseblock=${JFFS2_ERASEBLO EXTRA_IMAGECMD_ext2 ?= "-i 4096" EXTRA_IMAGECMD_ext3 ?= "-i 4096" EXTRA_IMAGECMD_ext4 ?= "-i 4096" -EXTRA_IMAGECMD_btrfs ?= "" +EXTRA_IMAGECMD_btrfs ?= "-n 4096" EXTRA_IMAGECMD_elf ?= "" IMAGE_DEPENDS = "" @@ -219,23 +242,44 @@ IMAGE_TYPES = " \ vmdk \ vdi \ qcow2 \ + hdddirect \ elf \ wic wic.gz wic.bz2 wic.lzma \ + container \ " -COMPRESSIONTYPES = "gz bz2 lzma xz lz4 sum" -COMPRESS_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}.rootfs.${type}" -COMPRESS_CMD_gz = "gzip -f -9 -c ${IMAGE_NAME}.rootfs.${type} > ${IMAGE_NAME}.rootfs.${type}.gz" -COMPRESS_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}.rootfs.${type}" -COMPRESS_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}.rootfs.${type} > ${IMAGE_NAME}.rootfs.${type}.xz" -COMPRESS_CMD_lz4 = "lz4c -9 -c ${IMAGE_NAME}.rootfs.${type} > ${IMAGE_NAME}.rootfs.${type}.lz4" -COMPRESS_CMD_sum = "sumtool -i ${IMAGE_NAME}.rootfs.${type} -o ${IMAGE_NAME}.rootfs.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" -COMPRESS_DEPENDS_lzma = "xz-native" -COMPRESS_DEPENDS_gz = "" -COMPRESS_DEPENDS_bz2 = "pbzip2-native" -COMPRESS_DEPENDS_xz = "xz-native" -COMPRESS_DEPENDS_lz4 = "lz4-native" -COMPRESS_DEPENDS_sum = "mtd-utils-native" +# Compression is a special case of conversion. The old variable +# names are still supported for backward-compatibility. When defining +# new compression or conversion commands, use CONVERSIONTYPES and +# CONVERSION_CMD/DEPENDS. +COMPRESSIONTYPES ?= "" + +CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap ${COMPRESSIONTYPES}" +CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_gz = "gzip -f -9 -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" +CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" +CONVERSION_CMD_lz4 = "lz4 -9 -z ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" +CONVERSION_CMD_lz4_legacy = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" +CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" +CONVERSION_CMD_sum = "sumtool -i ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}" +CONVERSION_CMD_md5sum = "md5sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.md5sum" +CONVERSION_CMD_sha1sum = "sha1sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha1sum" +CONVERSION_CMD_sha224sum = "sha224sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha224sum" +CONVERSION_CMD_sha256sum = "sha256sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha256sum" +CONVERSION_CMD_sha384sum = "sha384sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha384sum" +CONVERSION_CMD_sha512sum = "sha512sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha512sum" +CONVERSION_CMD_bmap = "bmaptool create ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.bmap" +CONVERSION_DEPENDS_lzma = "xz-native" +CONVERSION_DEPENDS_gz = "pigz-native" +CONVERSION_DEPENDS_bz2 = "pbzip2-native" +CONVERSION_DEPENDS_xz = "xz-native" +CONVERSION_DEPENDS_lz4 = "lz4-native" +CONVERSION_DEPENDS_lzo = "lzop-native" +CONVERSION_DEPENDS_zip = "zip-native" +CONVERSION_DEPENDS_sum = "mtd-utils-native" +CONVERSION_DEPENDS_bmap = "bmap-tools-native" RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4" RUNNABLE_MACHINE_PATTERNS ?= "qemu" @@ -246,9 +290,5 @@ DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" IMAGE_EXTENSION_live = "hddimg iso" # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES, -# images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hddimg, iso, etc. +# images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc. IMAGE_TYPES_MASKED ?= "" - -# The WICVARS variable is used to define list of bitbake variables used in wic code -# variables from this list is written to <image>.env file -WICVARS ?= "BBLAYERS DEPLOY_DIR_IMAGE HDDDIR IMAGE_BASENAME IMAGE_BOOT_FILES IMAGE_LINK_NAME IMAGE_ROOTFS INITRAMFS_FSTYPES INITRD ISODIR MACHINE_ARCH ROOTFS_SIZE STAGING_DATADIR STAGING_DIR_NATIVE STAGING_LIBDIR TARGET_SYS" diff --git a/meta/classes/image_types_uboot.bbclass b/meta/classes/image_types_uboot.bbclass index 081bca2cc9..5dfa39287d 100644 --- a/meta/classes/image_types_uboot.bbclass +++ b/meta/classes/image_types_uboot.bbclass @@ -2,25 +2,31 @@ inherit image_types kernel-arch oe_mkimage () { mkimage -A ${UBOOT_ARCH} -O linux -T ramdisk -C $2 -n ${IMAGE_NAME} \ - -d ${DEPLOY_DIR_IMAGE}/$1 ${DEPLOY_DIR_IMAGE}/$1.u-boot + -d ${IMGDEPLOYDIR}/$1 ${IMGDEPLOYDIR}/$1.u-boot if [ x$3 = x"clean" ]; then rm $1 fi } -COMPRESSIONTYPES += "gz.u-boot bz2.u-boot lzma.u-boot u-boot" +CONVERSIONTYPES += "gz.u-boot bz2.u-boot lz4.u-boot lzma.u-boot lzo.u-boot u-boot" -COMPRESS_DEPENDS_u-boot = "u-boot-mkimage-native" -COMPRESS_CMD_u-boot = "oe_mkimage ${IMAGE_NAME}.rootfs.${type} none" +CONVERSION_DEPENDS_u-boot = "u-boot-mkimage-native" +CONVERSION_CMD_u-boot = "oe_mkimage ${IMAGE_NAME}.rootfs.${type} none" -COMPRESS_DEPENDS_gz.u-boot = "u-boot-mkimage-native" -COMPRESS_CMD_gz.u-boot = "${COMPRESS_CMD_gz}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.gz gzip clean" +CONVERSION_DEPENDS_gz.u-boot = "u-boot-mkimage-native" +CONVERSION_CMD_gz.u-boot = "${CONVERSION_CMD_gz}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.gz gzip clean" -COMPRESS_DEPENDS_bz2.u-boot = "u-boot-mkimage-native" -COMPRESS_CMD_bz2.u-boot = "${COMPRESS_CMD_bz2}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.bz2 bzip2 clean" +CONVERSION_DEPENDS_bz2.u-boot = "u-boot-mkimage-native" +CONVERSION_CMD_bz2.u-boot = "${CONVERSION_CMD_bz2}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.bz2 bzip2 clean" -COMPRESS_DEPENDS_lzma.u-boot = "u-boot-mkimage-native" -COMPRESS_CMD_lzma.u-boot = "${COMPRESS_CMD_lzma}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.lzma lzma clean" +CONVERSION_DEPENDS_lz4.u-boot = "u-boot-mkimage-native" +CONVERSION_CMD_lz4.u-boot = "${CONVERSION_CMD_lz4_legacy}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.lz4 lz4 clean" -IMAGE_TYPES += "ext2.u-boot ext2.gz.u-boot ext2.bz2.u-boot ext2.lzma.u-boot ext3.gz.u-boot ext4.gz.u-boot" +CONVERSION_DEPENDS_lzma.u-boot = "u-boot-mkimage-native" +CONVERSION_CMD_lzma.u-boot = "${CONVERSION_CMD_lzma}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.lzma lzma clean" + +CONVERSION_DEPENDS_lzo.u-boot = "u-boot-mkimage-native" +CONVERSION_CMD_lzo.u-boot = "${CONVERSION_CMD_lzo}; oe_mkimage ${IMAGE_NAME}.rootfs.${type}.lzo lzo clean" + +IMAGE_TYPES += "ext2.u-boot ext2.gz.u-boot ext2.bz2.u-boot ext2.lzma.u-boot ext3.gz.u-boot ext4.gz.u-boot cpio.gz.u-boot" diff --git a/meta/classes/image_types_wic.bbclass b/meta/classes/image_types_wic.bbclass new file mode 100644 index 0000000000..4711c24593 --- /dev/null +++ b/meta/classes/image_types_wic.bbclass @@ -0,0 +1,117 @@ +# The WICVARS variable is used to define list of bitbake variables used in wic code +# variables from this list is written to <image>.env file +WICVARS ?= "\ + BBLAYERS IMGDEPLOYDIR DEPLOY_DIR_IMAGE FAKEROOTCMD IMAGE_BASENAME IMAGE_BOOT_FILES \ + IMAGE_LINK_NAME IMAGE_ROOTFS INITRAMFS_FSTYPES INITRD INITRD_LIVE ISODIR RECIPE_SYSROOT_NATIVE \ + ROOTFS_SIZE STAGING_DATADIR STAGING_DIR STAGING_LIBDIR TARGET_SYS TRANSLATED_TARGET_ARCH" + +WKS_FILE ??= "${IMAGE_BASENAME}.${MACHINE}.wks" +WKS_FILES ?= "${WKS_FILE} ${IMAGE_BASENAME}.wks" +WKS_SEARCH_PATH ?= "${THISDIR}:${@':'.join('%s/wic' % p for p in '${BBPATH}'.split(':'))}:${@':'.join('%s/scripts/lib/wic/canned-wks' % l for l in '${BBPATH}:${COREBASE}'.split(':'))}" +WKS_FULL_PATH = "${@wks_search(d.getVar('WKS_FILES').split(), d.getVar('WKS_SEARCH_PATH')) or ''}" + +def wks_search(files, search_path): + for f in files: + if os.path.isabs(f): + if os.path.exists(f): + return f + else: + searched = bb.utils.which(search_path, f) + if searched: + return searched + +WIC_CREATE_EXTRA_ARGS ?= "" + +IMAGE_CMD_wic () { + out="${IMGDEPLOYDIR}/${IMAGE_NAME}" + wks="${WKS_FULL_PATH}" + if [ -z "$wks" ]; then + bbfatal "No kickstart files from WKS_FILES were found: ${WKS_FILES}. Please set WKS_FILE or WKS_FILES appropriately." + fi + + BUILDDIR="${TOPDIR}" wic create "$wks" --vars "${STAGING_DIR}/${MACHINE}/imgdata/" -e "${IMAGE_BASENAME}" -o "$out/" ${WIC_CREATE_EXTRA_ARGS} + mv "$out/$(basename "${wks%.wks}")"*.direct "$out${IMAGE_NAME_SUFFIX}.wic" + rm -rf "$out/" +} +IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES" + +# Rebuild when the wks file or vars in WICVARS change +USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}" +WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}" +do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}" +do_image_wic[depends] += "wic-tools:do_populate_sysroot" +WKS_FILE_DEPENDS ??= '' +DEPENDS += "${@ '${WKS_FILE_DEPENDS}' if d.getVar('USING_WIC') else '' }" + +python do_write_wks_template () { + """Write out expanded template contents to WKS_FULL_PATH.""" + import re + + template_body = d.getVar('_WKS_TEMPLATE') + + # Remove any remnant variable references left behind by the expansion + # due to undefined variables + expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}") + while True: + new_body = re.sub(expand_var_regexp, '', template_body) + if new_body == template_body: + break + else: + template_body = new_body + + wks_file = d.getVar('WKS_FULL_PATH') + with open(wks_file, 'w') as f: + f.write(template_body) +} + +python () { + if d.getVar('USING_WIC'): + wks_file_u = d.getVar('WKS_FULL_PATH', False) + wks_file = d.expand(wks_file_u) + base, ext = os.path.splitext(wks_file) + if ext == '.in' and os.path.exists(wks_file): + wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base)) + d.setVar('WKS_FULL_PATH', wks_out_file) + d.setVar('WKS_TEMPLATE_PATH', wks_file_u) + d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True') + + # We need to re-parse each time the file changes, and bitbake + # needs to be told about that explicitly. + bb.parse.mark_dependency(d, wks_file) + + try: + with open(wks_file, 'r') as f: + body = f.read() + except (IOError, OSError) as exc: + pass + else: + # Previously, I used expandWithRefs to get the dependency list + # and add it to WICVARS, but there's no point re-parsing the + # file in process_wks_template as well, so just put it in + # a variable and let the metadata deal with the deps. + d.setVar('_WKS_TEMPLATE', body) + bb.build.addtask('do_write_wks_template', 'do_image_wic', None, d) +} + +# +# Write environment variables used by wic +# to tmp/sysroots/<machine>/imgdata/<image>.env +# +python do_rootfs_wicenv () { + wicvars = d.getVar('WICVARS') + if not wicvars: + return + + stdir = d.getVar('STAGING_DIR') + outdir = os.path.join(stdir, d.getVar('MACHINE'), 'imgdata') + bb.utils.mkdirhier(outdir) + basename = d.getVar('IMAGE_BASENAME') + with open(os.path.join(outdir, basename) + '.env', 'w') as envf: + for var in wicvars.split(): + value = d.getVar(var) + if value: + envf.write('%s="%s"\n' % (var, value.strip())) +} +addtask do_rootfs_wicenv after do_image before do_image_wic +do_rootfs_wicenv[vardeps] += "${WICVARS}" +do_rootfs_wicenv[prefuncs] = 'set_image_size' diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass index 61cd42bf34..dc1c2f8d41 100644 --- a/meta/classes/insane.bbclass +++ b/meta/classes/insane.bbclass @@ -30,19 +30,23 @@ QA_SANE = "True" WARN_QA ?= "ldflags useless-rpaths rpaths staticdev libdir xorg-driver-abi \ textrel already-stripped incompatible-license files-invalid \ installed-vs-shipped compile-host-path install-host-path \ - pn-overrides infodir build-deps file-rdeps \ + pn-overrides infodir build-deps \ unknown-configure-option symlink-to-sysroot multilib \ - invalid-pkgconfig \ + invalid-packageconfig host-user-contaminated \ " ERROR_QA ?= "dev-so debug-deps dev-deps debug-files arch pkgconfig la \ perms dep-cmp pkgvarcheck perm-config perm-line perm-link \ split-strip packages-list pkgv-undefined var-undefined \ version-going-backwards expanded-d invalid-chars \ + license-checksum dev-elf file-rdeps \ " +FAKEROOT_QA = "host-user-contaminated" +FAKEROOT_QA[doc] = "QA tests which need to run under fakeroot. If any \ +enabled tests are listed here, the do_package_qa task will run under fakeroot." ALL_QA = "${WARN_QA} ${ERROR_QA}" -UNKNOWN_CONFIGURE_WHITELIST ?= "--enable-nls --disable-nls --disable-silent-rules --disable-dependency-tracking --with-libtool-sysroot" +UNKNOWN_CONFIGURE_WHITELIST ?= "--enable-nls --disable-nls --disable-silent-rules --disable-dependency-tracking --with-libtool-sysroot --disable-static" # # dictionary for elf headers @@ -50,8 +54,8 @@ UNKNOWN_CONFIGURE_WHITELIST ?= "--enable-nls --disable-nls --disable-silent-rule # feel free to add and correct. # # TARGET_OS TARGET_ARCH MACHINE, OSABI, ABIVERSION, Little Endian, 32bit? -def package_qa_get_machine_dict(): - return { +def package_qa_get_machine_dict(d): + machdata = { "darwin9" : { "arm" : (40, 0, 0, True, 32), }, @@ -59,9 +63,13 @@ def package_qa_get_machine_dict(): "arm" : (40, 0, 0, True, 32), }, "elf" : { + "aarch64" : (183, 0, 0, True, 64), + "aarch64_be" :(183, 0, 0, False, 64), "i586" : (3, 0, 0, True, 32), "x86_64": (62, 0, 0, True, 64), "epiphany": (4643, 0, 0, True, 32), + "mips": ( 8, 0, 0, False, 32), + "mipsel": ( 8, 0, 0, True, 32), }, "linux" : { "aarch64" : (183, 0, 0, True, 64), @@ -83,6 +91,11 @@ def package_qa_get_machine_dict(): "mipsel": ( 8, 0, 0, True, 32), "mips64": ( 8, 0, 0, False, 64), "mips64el": ( 8, 0, 0, True, 64), + "mipsisa32r6": ( 8, 0, 0, False, 32), + "mipsisa32r6el": ( 8, 0, 0, True, 32), + "mipsisa64r6": ( 8, 0, 0, False, 64), + "mipsisa64r6el": ( 8, 0, 0, True, 64), + "nios2": (113, 0, 0, True, 32), "s390": (22, 0, 0, False, 32), "sh4": (42, 0, 0, True, 32), "sparc": ( 2, 0, 0, False, 32), @@ -122,6 +135,10 @@ def package_qa_get_machine_dict(): "mipsel": ( 8, 0, 0, True, 32), "mips64": ( 8, 0, 0, False, 64), "mips64el": ( 8, 0, 0, True, 64), + "microblaze": (189, 0, 0, False, 32), + "microblazeeb":(189, 0, 0, False, 32), + "microblazeel":(189, 0, 0, True, 32), + "sh4": ( 42, 0, 0, True, 32), }, "uclinux-uclibc" : { "bfin": ( 106, 0, 0, True, 32), @@ -157,44 +174,66 @@ def package_qa_get_machine_dict(): "linux-gnun32" : { "mips64": ( 8, 0, 0, False, 32), "mips64el": ( 8, 0, 0, True, 32), + "mipsisa64r6": ( 8, 0, 0, False, 32), + "mipsisa64r6el":( 8, 0, 0, True, 32), }, } + # Add in any extra user supplied data which may come from a BSP layer, removing the + # need to always change this class directly + extra_machdata = (d.getVar("PACKAGEQA_EXTRA_MACHDEFFUNCS") or "").split() + for m in extra_machdata: + call = m + "(machdata, d)" + locs = { "machdata" : machdata, "d" : d} + machdata = bb.utils.better_eval(call, locs) -def package_qa_clean_path(path,d): - """ Remove the common prefix from the path. In this case it is the TMPDIR""" - return path.replace(d.getVar('TMPDIR',True),"") + return machdata + + +def package_qa_clean_path(path, d, pkg=None): + """ + Remove redundant paths from the path for display. If pkg isn't set then + TMPDIR is stripped, otherwise PKGDEST/pkg is stripped. + """ + if pkg: + path = path.replace(os.path.join(d.getVar("PKGDEST"), pkg), "/") + return path.replace(d.getVar("TMPDIR"), "/").replace("//", "/") def package_qa_write_error(type, error, d): - logfile = d.getVar('QA_LOGFILE', True) + logfile = d.getVar('QA_LOGFILE') if logfile: - p = d.getVar('P', True) - f = file( logfile, "a+") - print >> f, "%s: %s [%s]" % (p, error, type) - f.close() + p = d.getVar('P') + with open(logfile, "a+") as f: + f.write("%s: %s [%s]\n" % (p, error, type)) def package_qa_handle_error(error_class, error_msg, d): package_qa_write_error(error_class, error_msg, d) - if error_class in (d.getVar("ERROR_QA", True) or "").split(): + if error_class in (d.getVar("ERROR_QA") or "").split(): bb.error("QA Issue: %s [%s]" % (error_msg, error_class)) d.setVar("QA_SANE", False) return False - elif error_class in (d.getVar("WARN_QA", True) or "").split(): + elif error_class in (d.getVar("WARN_QA") or "").split(): bb.warn("QA Issue: %s [%s]" % (error_msg, error_class)) else: bb.note("QA Issue: %s [%s]" % (error_msg, error_class)) return True +def package_qa_add_message(messages, section, new_msg): + if section not in messages: + messages[section] = new_msg + else: + messages[section] = messages[section] + "\n" + new_msg + QAPATHTEST[libexec] = "package_qa_check_libexec" def package_qa_check_libexec(path,name, d, elf, messages): # Skip the case where the default is explicitly /usr/libexec - libexec = d.getVar('libexecdir', True) + libexec = d.getVar('libexecdir') if libexec == "/usr/libexec": return True if 'libexec' in path.split(os.path.sep): - messages["libexec"] = "%s: %s is using libexec please relocate to %s" % (name, package_qa_clean_path(path, d), libexec) + package_qa_add_message(messages, "libexec", "%s: %s is using libexec please relocate to %s" % (name, package_qa_clean_path(path, d), libexec)) return False return True @@ -210,7 +249,7 @@ def package_qa_check_rpath(file,name, d, elf, messages): if os.path.islink(file): return - bad_dirs = [d.getVar('BASE_WORKDIR', True), d.getVar('STAGING_DIR_TARGET', True)] + bad_dirs = [d.getVar('BASE_WORKDIR'), d.getVar('STAGING_DIR_TARGET')] phdrs = elf.run_objdump("-p", d) @@ -222,7 +261,7 @@ def package_qa_check_rpath(file,name, d, elf, messages): rpath = m.group(1) for dir in bad_dirs: if dir in rpath: - messages["rpaths"] = "package %s contains bad RPATH %s in file %s" % (name, rpath, file) + package_qa_add_message(messages, "rpaths", "package %s contains bad RPATH %s in file %s" % (name, rpath, file)) QAPATHTEST[useless-rpaths] = "package_qa_check_useless_rpaths" def package_qa_check_useless_rpaths(file, name, d, elf, messages): @@ -238,8 +277,8 @@ def package_qa_check_useless_rpaths(file, name, d, elf, messages): if os.path.islink(file): return - libdir = d.getVar("libdir", True) - base_libdir = d.getVar("base_libdir", True) + libdir = d.getVar("libdir") + base_libdir = d.getVar("base_libdir") phdrs = elf.run_objdump("-p", d) @@ -252,7 +291,7 @@ def package_qa_check_useless_rpaths(file, name, d, elf, messages): if rpath_eq(rpath, libdir) or rpath_eq(rpath, base_libdir): # The dynamic linker searches both these places anyway. There is no point in # looking there again. - messages["useless-rpaths"] = "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d), rpath) + package_qa_add_message(messages, "useless-rpaths", "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d), rpath)) QAPATHTEST[dev-so] = "package_qa_check_dev" def package_qa_check_dev(path, name, d, elf, messages): @@ -261,8 +300,19 @@ def package_qa_check_dev(path, name, d, elf, messages): """ if not name.endswith("-dev") and not name.endswith("-dbg") and not name.endswith("-ptest") and not name.startswith("nativesdk-") and path.endswith(".so") and os.path.islink(path): - messages["dev-so"] = "non -dev/-dbg/nativesdk- package contains symlink .so: %s path '%s'" % \ - (name, package_qa_clean_path(path,d)) + package_qa_add_message(messages, "dev-so", "non -dev/-dbg/nativesdk- package contains symlink .so: %s path '%s'" % \ + (name, package_qa_clean_path(path,d))) + +QAPATHTEST[dev-elf] = "package_qa_check_dev_elf" +def package_qa_check_dev_elf(path, name, d, elf, messages): + """ + Check that -dev doesn't contain real shared libraries. The test has to + check that the file is not a link and is an ELF object as some recipes + install link-time .so files that are linker scripts. + """ + if name.endswith("-dev") and path.endswith(".so") and not os.path.islink(path) and elf: + package_qa_add_message(messages, "dev-elf", "-dev package contains non-symlink .so: %s path '%s'" % \ + (name, package_qa_clean_path(path,d))) QAPATHTEST[staticdev] = "package_qa_check_staticdev" def package_qa_check_staticdev(path, name, d, elf, messages): @@ -274,8 +324,8 @@ def package_qa_check_staticdev(path, name, d, elf, messages): """ if not name.endswith("-pic") and not name.endswith("-staticdev") and not name.endswith("-ptest") and path.endswith(".a") and not path.endswith("_nonshared.a"): - messages["staticdev"] = "non -staticdev package contains static .a library: %s path '%s'" % \ - (name, package_qa_clean_path(path,d)) + package_qa_add_message(messages, "staticdev", "non -staticdev package contains static .a library: %s path '%s'" % \ + (name, package_qa_clean_path(path,d))) def package_qa_check_libdir(d): """ @@ -285,13 +335,17 @@ def package_qa_check_libdir(d): """ import re - pkgdest = d.getVar('PKGDEST', True) - base_libdir = d.getVar("base_libdir",True) + os.sep - libdir = d.getVar("libdir", True) + os.sep - exec_prefix = d.getVar("exec_prefix", True) + os.sep + pkgdest = d.getVar('PKGDEST') + base_libdir = d.getVar("base_libdir") + os.sep + libdir = d.getVar("libdir") + os.sep + libexecdir = d.getVar("libexecdir") + os.sep + exec_prefix = d.getVar("exec_prefix") + os.sep messages = [] + # The re's are purposely fuzzy, as some there are some .so.x.y.z files + # that don't follow the standard naming convention. It checks later + # that they are actual ELF files lib_re = re.compile("^/lib.+\.so(\..+)?$") exec_re = re.compile("^%s.*/lib.+\.so(\..+)?$" % exec_prefix) @@ -300,9 +354,12 @@ def package_qa_check_libdir(d): # Skip subdirectories for any packages with libdir in INSANE_SKIP skippackages = [] for package in dirs: - if 'libdir' in (d.getVar('INSANE_SKIP_' + package, True) or "").split(): + if 'libdir' in (d.getVar('INSANE_SKIP_' + package) or "").split(): bb.note("Package %s skipping libdir QA test" % (package)) skippackages.append(package) + elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory' and package.endswith("-dbg"): + bb.note("Package %s skipping libdir QA test for PACKAGE_DEBUG_SPLIT_STYLE equals debug-file-directory" % (package)) + skippackages.append(package) for package in skippackages: dirs.remove(package) for file in files: @@ -313,10 +370,22 @@ def package_qa_check_libdir(d): rel_path = os.sep + rel_path if lib_re.match(rel_path): if base_libdir not in rel_path: - messages.append("%s: found library in wrong location: %s" % (package, rel_path)) + # make sure it's an actual ELF file + elf = oe.qa.ELFFile(full_path) + try: + elf.open() + messages.append("%s: found library in wrong location: %s" % (package, rel_path)) + except (oe.qa.NotELFFileError): + pass if exec_re.match(rel_path): - if libdir not in rel_path: - messages.append("%s: found library in wrong location: %s" % (package, rel_path)) + if libdir not in rel_path and libexecdir not in rel_path: + # make sure it's an actual ELF file + elf = oe.qa.ELFFile(full_path) + try: + elf.open() + messages.append("%s: found library in wrong location: %s" % (package, rel_path)) + except (oe.qa.NotELFFileError): + pass if messages: package_qa_handle_error("libdir", "\n".join(messages), d) @@ -329,8 +398,8 @@ def package_qa_check_dbg(path, name, d, elf, messages): if not "-dbg" in name and not "-ptest" in name: if '.debug' in path.split(os.path.sep): - messages["debug-files"] = "non debug package contains .debug directory: %s path %s" % \ - (name, package_qa_clean_path(path,d)) + package_qa_add_message(messages, "debug-files", "non debug package contains .debug directory: %s path %s" % \ + (name, package_qa_clean_path(path,d))) QAPATHTEST[perms] = "package_qa_check_perm" def package_qa_check_perm(path,name,d, elf, messages): @@ -339,48 +408,6 @@ def package_qa_check_perm(path,name,d, elf, messages): """ return -QAPATHTEST[unsafe-references-in-binaries] = "package_qa_check_unsafe_references_in_binaries" -def package_qa_check_unsafe_references_in_binaries(path, name, d, elf, messages): - """ - Ensure binaries in base_[bindir|sbindir|libdir] do not link to files under exec_prefix - """ - if unsafe_references_skippable(path, name, d): - return - - if elf: - import subprocess as sub - pn = d.getVar('PN', True) - - exec_prefix = d.getVar('exec_prefix', True) - sysroot_path = d.getVar('STAGING_DIR_TARGET', True) - sysroot_path_usr = sysroot_path + exec_prefix - - try: - ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read() - except bb.process.CmdError: - error_msg = pn + ": prelink-rtld aborted when processing %s" % path - package_qa_handle_error("unsafe-references-in-binaries", error_msg, d) - return False - - if sysroot_path_usr in ldd_output: - ldd_output = ldd_output.replace(sysroot_path, "") - - pkgdest = d.getVar('PKGDEST', True) - packages = d.getVar('PACKAGES', True) - - for package in packages.split(): - short_path = path.replace('%s/%s' % (pkgdest, package), "", 1) - if (short_path != path): - break - - base_err = pn + ": %s, installed in the base_prefix, requires a shared library under exec_prefix (%s)" % (short_path, exec_prefix) - for line in ldd_output.split('\n'): - if exec_prefix in line: - error_msg = "%s: %s" % (base_err, line.strip()) - package_qa_handle_error("unsafe-references-in-binaries", error_msg, d) - - return False - QAPATHTEST[unsafe-references-in-scripts] = "package_qa_check_unsafe_references_in_scripts" def package_qa_check_unsafe_references_in_scripts(path, name, d, elf, messages): """ @@ -392,14 +419,14 @@ def package_qa_check_unsafe_references_in_scripts(path, name, d, elf, messages): if not elf: import stat import subprocess - pn = d.getVar('PN', True) + pn = d.getVar('PN') # Ensure we're checking an executable script statinfo = os.stat(path) if bool(statinfo.st_mode & stat.S_IXUSR): # grep shell scripts for possible references to /exec_prefix/ - exec_prefix = d.getVar('exec_prefix', True) - statement = "grep -e '%s/' %s > /dev/null" % (exec_prefix, path) + exec_prefix = d.getVar('exec_prefix') + statement = "grep -e '%s/[^ :]\{1,\}/[^ :]\{1,\}' %s > /dev/null" % (exec_prefix, path) if subprocess.call(statement, shell=True) == 0: error_msg = pn + ": Found a reference to %s/ in %s" % (exec_prefix, path) package_qa_handle_error("unsafe-references-in-scripts", error_msg, d) @@ -422,19 +449,19 @@ def unsafe_references_skippable(path, name, d): return True # Skip unusual rootfs layouts which make these tests irrelevant - exec_prefix = d.getVar('exec_prefix', True) + exec_prefix = d.getVar('exec_prefix') if exec_prefix == "": return True - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') pkgdest = pkgdest + "/" + name pkgdest = os.path.abspath(pkgdest) - base_bindir = pkgdest + d.getVar('base_bindir', True) - base_sbindir = pkgdest + d.getVar('base_sbindir', True) - base_libdir = pkgdest + d.getVar('base_libdir', True) - bindir = pkgdest + d.getVar('bindir', True) - sbindir = pkgdest + d.getVar('sbindir', True) - libdir = pkgdest + d.getVar('libdir', True) + base_bindir = pkgdest + d.getVar('base_bindir') + base_sbindir = pkgdest + d.getVar('base_sbindir') + base_libdir = pkgdest + d.getVar('base_libdir') + bindir = pkgdest + d.getVar('bindir') + sbindir = pkgdest + d.getVar('sbindir') + libdir = pkgdest + d.getVar('libdir') if base_bindir == bindir and base_sbindir == sbindir and base_libdir == libdir: return True @@ -451,17 +478,19 @@ def package_qa_check_arch(path,name,d, elf, messages): """ Check if archs are compatible """ + import re + if not elf: return - target_os = d.getVar('TARGET_OS', True) - target_arch = d.getVar('TARGET_ARCH', True) - provides = d.getVar('PROVIDES', True) - bpn = d.getVar('BPN', True) + target_os = d.getVar('TARGET_OS') + target_arch = d.getVar('TARGET_ARCH') + provides = d.getVar('PROVIDES') + bpn = d.getVar('BPN') if target_arch == "allarch": - pn = d.getVar('PN', True) - messages["arch"] = pn + ": Recipe inherits the allarch class, but has packaged architecture-specific binaries" + pn = d.getVar('PN') + package_qa_add_message(messages, "arch", pn + ": Recipe inherits the allarch class, but has packaged architecture-specific binaries") return # FIXME: Cross package confuse this check, so just skip them @@ -476,20 +505,20 @@ def package_qa_check_arch(path,name,d, elf, messages): #if this will throw an exception, then fix the dict above (machine, osabi, abiversion, littleendian, bits) \ - = package_qa_get_machine_dict()[target_os][target_arch] + = package_qa_get_machine_dict(d)[target_os][target_arch] # Check the architecture and endiannes of the binary - if not ((machine == elf.machine()) or \ - ((("virtual/kernel" in provides) or bb.data.inherits_class("module", d) ) and (target_os == "linux-gnux32" or target_os == "linux-gnun32"))): - messages["arch"] = "Architecture did not match (%d to %d) on %s" % \ - (machine, elf.machine(), package_qa_clean_path(path,d)) - elif not ((bits == elf.abiSize()) or \ - ((("virtual/kernel" in provides) or bb.data.inherits_class("module", d) ) and (target_os == "linux-gnux32" or target_os == "linux-gnun32"))): - messages["arch"] = "Bit size did not match (%d to %d) %s on %s" % \ - (bits, elf.abiSize(), bpn, package_qa_clean_path(path,d)) + is_32 = (("virtual/kernel" in provides) or bb.data.inherits_class("module", d)) and \ + (target_os == "linux-gnux32" or re.match('mips64.*32', d.getVar('DEFAULTTUNE'))) + if not ((machine == elf.machine()) or is_32): + package_qa_add_message(messages, "arch", "Architecture did not match (%s, expected %s) on %s" % \ + (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path,d))) + elif not ((bits == elf.abiSize()) or is_32): + package_qa_add_message(messages, "arch", "Bit size did not match (%d to %d) %s on %s" % \ + (bits, elf.abiSize(), bpn, package_qa_clean_path(path,d))) elif not littleendian == elf.isLittleEndian(): - messages["arch"] = "Endiannes did not match (%d to %d) on %s" % \ - (littleendian, elf.isLittleEndian(), package_qa_clean_path(path,d)) + package_qa_add_message(messages, "arch", "Endiannes did not match (%d to %d) on %s" % \ + (littleendian, elf.isLittleEndian(), package_qa_clean_path(path,d))) QAPATHTEST[desktop] = "package_qa_check_desktop" def package_qa_check_desktop(path, name, d, elf, messages): @@ -497,11 +526,11 @@ def package_qa_check_desktop(path, name, d, elf, messages): Run all desktop files through desktop-file-validate. """ if path.endswith(".desktop"): - desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE',True),'desktop-file-validate') + desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE'),'desktop-file-validate') output = os.popen("%s %s" % (desktop_file_validate, path)) # This only produces output on errors for l in output: - messages["desktop"] = "Desktop file issue: " + l.strip() + package_qa_add_message(messages, "desktop", "Desktop file issue: " + l.strip()) QAPATHTEST[textrel] = "package_qa_textrel" def package_qa_textrel(path, name, d, elf, messages): @@ -525,7 +554,7 @@ def package_qa_textrel(path, name, d, elf, messages): sane = False if not sane: - messages["textrel"] = "ELF binary '%s' has relocations in .text" % path + package_qa_add_message(messages, "textrel", "ELF binary '%s' has relocations in .text" % path) QAPATHTEST[ldflags] = "package_qa_hash_style" def package_qa_hash_style(path, name, d, elf, messages): @@ -539,9 +568,9 @@ def package_qa_hash_style(path, name, d, elf, messages): if os.path.islink(path): return - gnu_hash = "--hash-style=gnu" in d.getVar('LDFLAGS', True) + gnu_hash = "--hash-style=gnu" in d.getVar('LDFLAGS') if not gnu_hash: - gnu_hash = "--hash-style=both" in d.getVar('LDFLAGS', True) + gnu_hash = "--hash-style=both" in d.getVar('LDFLAGS') if not gnu_hash: return @@ -560,7 +589,7 @@ def package_qa_hash_style(path, name, d, elf, messages): sane = True if has_syms and not sane: - messages["ldflags"] = "No GNU_HASH in the elf binary: '%s'" % path + package_qa_add_message(messages, "ldflags", "No GNU_HASH in the elf binary: '%s'" % path) QAPATHTEST[buildpaths] = "package_qa_check_buildpaths" @@ -576,11 +605,15 @@ def package_qa_check_buildpaths(path, name, d, elf, messages): if os.path.islink(path): return - tmpdir = d.getVar('TMPDIR', True) - with open(path) as f: - file_content = f.read() + # Ignore ipk and deb's CONTROL dir + if path.find(name + "/CONTROL/") != -1 or path.find(name + "/DEBIAN/") != -1: + return + + tmpdir = d.getVar('TMPDIR') + with open(path, 'rb') as f: + file_content = f.read().decode('utf-8', errors='ignore') if tmpdir in file_content: - messages["buildpaths"] = "File %s in package contained reference to tmpdir" % package_qa_clean_path(path,d) + package_qa_add_message(messages, "buildpaths", "File %s in package contained reference to tmpdir" % package_qa_clean_path(path,d)) QAPATHTEST[xorg-driver-abi] = "package_qa_check_xorg_driver_abi" @@ -595,11 +628,11 @@ def package_qa_check_xorg_driver_abi(path, name, d, elf, messages): driverdir = d.expand("${libdir}/xorg/modules/drivers/") if driverdir in path and path.endswith(".so"): - mlprefix = d.getVar('MLPREFIX', True) or '' - for rdep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + name, True) or ""): + mlprefix = d.getVar('MLPREFIX') or '' + for rdep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + name) or ""): if rdep.startswith("%sxorg-abi-" % mlprefix): return - messages["xorg-driver-abi"] = "Package %s contains Xorg driver (%s) but no xorg-abi- dependencies" % (name, os.path.basename(path)) + package_qa_add_message(messages, "xorg-driver-abi", "Package %s contains Xorg driver (%s) but no xorg-abi- dependencies" % (name, os.path.basename(path))) QAPATHTEST[infodir] = "package_qa_check_infodir" def package_qa_check_infodir(path, name, d, elf, messages): @@ -609,7 +642,7 @@ def package_qa_check_infodir(path, name, d, elf, messages): infodir = d.expand("${infodir}/dir") if infodir in path: - messages["infodir"] = "The /usr/share/info/dir file is not meant to be shipped in a particular package." + package_qa_add_message(messages, "infodir", "The /usr/share/info/dir file is not meant to be shipped in a particular package.") QAPATHTEST[symlink-to-sysroot] = "package_qa_check_symlink_to_sysroot" def package_qa_check_symlink_to_sysroot(path, name, d, elf, messages): @@ -619,38 +652,42 @@ def package_qa_check_symlink_to_sysroot(path, name, d, elf, messages): if os.path.islink(path): target = os.readlink(path) if os.path.isabs(target): - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') if target.startswith(tmpdir): - trimmed = path.replace(os.path.join (d.getVar("PKGDEST", True), name), "") - messages["symlink-to-sysroot"] = "Symlink %s in %s points to TMPDIR" % (trimmed, name) -def package_qa_check_license(workdir, d): + trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "") + package_qa_add_message(messages, "symlink-to-sysroot", "Symlink %s in %s points to TMPDIR" % (trimmed, name)) + +# Check license variables +do_populate_lic[postfuncs] += "populate_lic_qa_checksum" +python populate_lic_qa_checksum() { """ - Check for changes in the license files + Check for changes in the license files. """ import tempfile sane = True - lic_files = d.getVar('LIC_FILES_CHKSUM', True) - lic = d.getVar('LICENSE', True) - pn = d.getVar('PN', True) + lic_files = d.getVar('LIC_FILES_CHKSUM') or '' + lic = d.getVar('LICENSE') + pn = d.getVar('PN') if lic == "CLOSED": - return True + return - if not lic_files: - bb.error(pn + ": Recipe file does not have license file information (LIC_FILES_CHKSUM)") - return False + if not lic_files and d.getVar('SRC_URI'): + sane = package_qa_handle_error("license-checksum", pn + ": Recipe file fetches files and does not have license file information (LIC_FILES_CHKSUM)", d) - srcdir = d.getVar('S', True) + srcdir = d.getVar('S') for url in lic_files.split(): try: (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) except bb.fetch.MalformedUrl: - raise bb.build.FuncFailed( pn + ": LIC_FILES_CHKSUM contains an invalid URL: " + url) + sane = package_qa_handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM contains an invalid URL: " + url, d) + continue srclicfile = os.path.join(srcdir, path) if not os.path.isfile(srclicfile): - raise bb.build.FuncFailed( pn + ": LIC_FILES_CHKSUM points to an invalid file: " + srclicfile) + package_qa_handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM points to an invalid file: " + srclicfile, d) + continue recipemd5 = parm.get('md5', '') beginline, endline = 0, 0 @@ -661,17 +698,21 @@ def package_qa_check_license(workdir, d): if (not beginline) and (not endline): md5chksum = bb.utils.md5_file(srclicfile) + with open(srclicfile, 'rb') as f: + license = f.read() else: fi = open(srclicfile, 'rb') fo = tempfile.NamedTemporaryFile(mode='wb', prefix='poky.', suffix='.tmp', delete=False) tmplicfile = fo.name; lineno = 0 linesout = 0 + license = [] for line in fi: lineno += 1 - if (lineno >= beginline): + if (lineno >= beginline): if ((lineno <= endline) or not endline): fo.write(line) + license.append(line) linesout += 1 else: break @@ -679,14 +720,39 @@ def package_qa_check_license(workdir, d): fo.close() fi.close() md5chksum = bb.utils.md5_file(tmplicfile) + license = b''.join(license) os.unlink(tmplicfile) if recipemd5 == md5chksum: bb.note (pn + ": md5 checksum matched for ", url) else: if recipemd5: - bb.error(pn + ": md5 data is not matching for ", url) - bb.error(pn + ": The new md5 checksum is ", md5chksum) + msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url + msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum + try: + license_lines = license.decode('utf-8').split('\n') + except: + # License text might not be valid UTF-8, in which + # case we don't know how to include it in our output + # and have to skip it. + pass + else: + max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20) + if not license_lines or license_lines[-1] != '': + # Ensure that our license text ends with a line break + # (will be added with join() below). + license_lines.append('') + remove = len(license_lines) - max_lines + if remove > 0: + start = max_lines // 2 + end = start + remove - 1 + del license_lines[start:end] + license_lines.insert(start, '...') + msg = msg + "\n" + pn + ": Here is the selected license text:" + \ + "\n" + \ + "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \ + "\n" + "\n".join(license_lines) + \ + "{:^^70}".format(" endline=%d " % endline if endline else "") if beginline: if endline: srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline) @@ -696,29 +762,31 @@ def package_qa_check_license(workdir, d): srcfiledesc = "%s (ending on line %d)" % (srclicfile, endline) else: srcfiledesc = srclicfile - bb.error(pn + ": Check if the license information has changed in %s to verify that the LICENSE value \"%s\" remains valid" % (srcfiledesc, lic)) + msg = msg + "\n" + pn + ": Check if the license information has changed in %s to verify that the LICENSE value \"%s\" remains valid" % (srcfiledesc, lic) + else: - bb.error(pn + ": md5 checksum is not specified for ", url) - bb.error(pn + ": The md5 checksum is ", md5chksum) - sane = False + msg = pn + ": LIC_FILES_CHKSUM is not specified for " + url + msg = msg + "\n" + pn + ": The md5 checksum is " + md5chksum + sane = package_qa_handle_error("license-checksum", msg, d) - return sane + if not sane: + bb.fatal("Fatal QA errors found, failing task.") +} def package_qa_check_staged(path,d): """ - Check staged la and pc files for sanity - -e.g. installed being false + Check staged la and pc files for common problems like references to the work + directory. - As this is run after every stage we should be able - to find the one responsible for the errors easily even - if we look at every .pc and .la file + As this is run after every stage we should be able to find the one + responsible for the errors easily even if we look at every .pc and .la file. """ sane = True - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') workdir = os.path.join(tmpdir, "work") + recipesysroot = d.getVar("RECIPE_SYSROOT") - installed = "installed=yes" if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): pkgconfigcheck = workdir else: @@ -733,25 +801,44 @@ def package_qa_check_staged(path,d): if file.endswith(".la"): with open(path) as f: file_content = f.read() + file_content = file_content.replace(recipesysroot, "") if workdir in file_content: error_msg = "%s failed sanity test (workdir) in path %s" % (file,root) sane = package_qa_handle_error("la", error_msg, d) elif file.endswith(".pc"): with open(path) as f: file_content = f.read() + file_content = file_content.replace(recipesysroot, "") if pkgconfigcheck in file_content: error_msg = "%s failed sanity test (tmpdir) in path %s" % (file,root) sane = package_qa_handle_error("pkgconfig", error_msg, d) return sane +# Run all package-wide warnfuncs and errorfuncs +def package_qa_package(warnfuncs, errorfuncs, skip, package, d): + warnings = {} + errors = {} + + for func in warnfuncs: + func(package, d, warnings) + for func in errorfuncs: + func(package, d, errors) + + for w in warnings: + package_qa_handle_error(w, warnings[w], d) + for e in errors: + package_qa_handle_error(e, errors[e], d) + + return len(errors) == 0 + # Walk over all files in a directory and call func -def package_qa_walk(path, warnfuncs, errorfuncs, skip, package, d): +def package_qa_walk(warnfuncs, errorfuncs, skip, package, d): import oe.qa #if this will throw an exception, then fix the dict above - target_os = d.getVar('TARGET_OS', True) - target_arch = d.getVar('TARGET_ARCH', True) + target_os = d.getVar('TARGET_OS') + target_arch = d.getVar('TARGET_ARCH') warnings = {} errors = {} @@ -759,7 +846,8 @@ def package_qa_walk(path, warnfuncs, errorfuncs, skip, package, d): elf = oe.qa.ELFFile(path) try: elf.open() - except: + except (IOError, oe.qa.NotELFFileError): + # IOError can happen if the packaging control files disappear, elf = None for func in warnfuncs: func(path, package, d, elf, warnings) @@ -771,38 +859,34 @@ def package_qa_walk(path, warnfuncs, errorfuncs, skip, package, d): for e in errors: package_qa_handle_error(e, errors[e], d) - return len(errors) == 0 - def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d): # Don't do this check for kernel/module recipes, there aren't too many debug/development # packages and you can get false positives e.g. on kernel-module-lirc-dev if bb.data.inherits_class("kernel", d) or bb.data.inherits_class("module-base", d): - return True + return - sane = True if not "-dbg" in pkg and not "packagegroup-" in pkg and not "-image" in pkg: localdata = bb.data.createCopy(d) localdata.setVar('OVERRIDES', pkg) - bb.data.update_data(localdata) # Now check the RDEPENDS - rdepends = bb.utils.explode_deps(localdata.getVar('RDEPENDS', True) or "") + rdepends = bb.utils.explode_deps(localdata.getVar('RDEPENDS') or "") # Now do the sanity check!!! if "build-deps" not in skip: for rdepend in rdepends: if "-dbg" in rdepend and "debug-deps" not in skip: error_msg = "%s rdepends on %s" % (pkg,rdepend) - sane = package_qa_handle_error("debug-deps", error_msg, d) + package_qa_handle_error("debug-deps", error_msg, d) if (not "-dev" in pkg and not "-staticdev" in pkg) and rdepend.endswith("-dev") and "dev-deps" not in skip: error_msg = "%s rdepends on %s" % (pkg, rdepend) - sane = package_qa_handle_error("dev-deps", error_msg, d) + package_qa_handle_error("dev-deps", error_msg, d) if rdepend not in packages: rdep_data = oe.packagedata.read_subpkgdata(rdepend, d) if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: continue if not rdep_data or not 'PN' in rdep_data: - pkgdata_dir = d.getVar("PKGDATA_DIR", True) + pkgdata_dir = d.getVar("PKGDATA_DIR") try: possibles = os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdepend)) except OSError: @@ -813,8 +897,11 @@ def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d): break if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: continue - error_msg = "%s rdepends on %s, but it isn't a build dependency?" % (pkg, rdepend) - sane = package_qa_handle_error("build-deps", error_msg, d) + if rdep_data and 'PN' in rdep_data: + error_msg = "%s rdepends on %s, but it isn't a build dependency, missing %s in DEPENDS or PACKAGECONFIG?" % (pkg, rdepend, rdep_data['PN']) + else: + error_msg = "%s rdepends on %s, but it isn't a build dependency?" % (pkg, rdepend) + package_qa_handle_error("build-deps", error_msg, d) if "file-rdeps" not in skip: ignored_file_rdeps = set(['/bin/sh', '/usr/bin/env', 'rtld(GNU_HASH)']) @@ -858,7 +945,7 @@ def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d): # The python is not a package, but python-core provides it, so # skip checking /usr/bin/python if python is in the rdeps, in # case there is a RDEPENDS_pkg = "python" in the recipe. - for py in [ d.getVar('MLPREFIX', True) + "python", "python" ]: + for py in [ d.getVar('MLPREFIX') + "python", "python" ]: if py in done: filerdepends.pop("/usr/bin/python",None) done.remove(py) @@ -879,82 +966,60 @@ def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d): break if filerdepends: for key in filerdepends: - error_msg = "%s contained in package %s requires %s, but no providers found in its RDEPENDS" % \ - (filerdepends[key],pkg, key) - sane = package_qa_handle_error("file-rdeps", error_msg, d) - - return sane + error_msg = "%s contained in package %s requires %s, but no providers found in RDEPENDS_%s?" % \ + (filerdepends[key].replace("_%s" % pkg, "").replace("@underscore@", "_"), pkg, key, pkg) + package_qa_handle_error("file-rdeps", error_msg, d) def package_qa_check_deps(pkg, pkgdest, skip, d): - sane = True localdata = bb.data.createCopy(d) localdata.setVar('OVERRIDES', pkg) - bb.data.update_data(localdata) def check_valid_deps(var): - sane = True try: - rvar = bb.utils.explode_dep_versions2(localdata.getVar(var, True) or "") + rvar = bb.utils.explode_dep_versions2(localdata.getVar(var) or "") except ValueError as e: bb.fatal("%s_%s: %s" % (var, pkg, e)) for dep in rvar: for v in rvar[dep]: if v and not v.startswith(('< ', '= ', '> ', '<= ', '>=')): error_msg = "%s_%s is invalid: %s (%s) only comparisons <, =, >, <=, and >= are allowed" % (var, pkg, dep, v) - sane = package_qa_handle_error("dep-cmp", error_msg, d) - return sane - - sane = True - if not check_valid_deps('RDEPENDS'): - sane = False - if not check_valid_deps('RRECOMMENDS'): - sane = False - if not check_valid_deps('RSUGGESTS'): - sane = False - if not check_valid_deps('RPROVIDES'): - sane = False - if not check_valid_deps('RREPLACES'): - sane = False - if not check_valid_deps('RCONFLICTS'): - sane = False + package_qa_handle_error("dep-cmp", error_msg, d) - return sane + check_valid_deps('RDEPENDS') + check_valid_deps('RRECOMMENDS') + check_valid_deps('RSUGGESTS') + check_valid_deps('RPROVIDES') + check_valid_deps('RREPLACES') + check_valid_deps('RCONFLICTS') -QAPATHTEST[expanded-d] = "package_qa_check_expanded_d" -def package_qa_check_expanded_d(path,name,d,elf,messages): +QAPKGTEST[expanded-d] = "package_qa_check_expanded_d" +def package_qa_check_expanded_d(package, d, messages): """ Check for the expanded D (${D}) value in pkg_* and FILES variables, warn the user to use it correctly. """ - sane = True - expanded_d = d.getVar('D',True) - - # Get packages for current recipe and iterate - packages = d.getVar('PACKAGES', True).split(" ") - for pak in packages: - # Go through all variables and check if expanded D is found, warn the user accordingly - for var in 'FILES','pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm': - bbvar = d.getVar(var + "_" + pak, False) - if bbvar: - # Bitbake expands ${D} within bbvar during the previous step, so we check for its expanded value - if expanded_d in bbvar: - if var == 'FILES': - messages["expanded-d"] = "FILES in %s recipe should not contain the ${D} variable as it references the local build directory not the target filesystem, best solution is to remove the ${D} reference" % pak - sane = False - else: - messages["expanded-d"] = "%s in %s recipe contains ${D}, it should be replaced by $D instead" % (var, pak) - sane = False + expanded_d = d.getVar('D') + + for var in 'FILES','pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm': + bbvar = d.getVar(var + "_" + package) or "" + if expanded_d in bbvar: + if var == 'FILES': + package_qa_add_message(messages, "expanded-d", "FILES in %s recipe should not contain the ${D} variable as it references the local build directory not the target filesystem, best solution is to remove the ${D} reference" % package) + sane = False + else: + package_qa_add_message(messages, "expanded-d", "%s in %s recipe contains ${D}, it should be replaced by $D instead" % (var, package)) + sane = False return sane def package_qa_check_encoding(keys, encode, d): - def check_encoding(key,enc): + def check_encoding(key, enc): sane = True - value = d.getVar(key, True) + value = d.getVar(key) if value: try: - s = unicode(value, enc) + s = value.encode(enc) except UnicodeDecodeError as e: error_msg = "%s has non %s characters" % (key,enc) sane = False @@ -966,6 +1031,41 @@ def package_qa_check_encoding(keys, encode, d): if not sane: break +HOST_USER_UID := "${@os.getuid()}" +HOST_USER_GID := "${@os.getgid()}" + +QAPATHTEST[host-user-contaminated] = "package_qa_check_host_user" +def package_qa_check_host_user(path, name, d, elf, messages): + """Check for paths outside of /home which are owned by the user running bitbake.""" + + if not os.path.lexists(path): + return + + dest = d.getVar('PKGDEST') + pn = d.getVar('PN') + home = os.path.join(dest, 'home') + if path == home or path.startswith(home + os.sep): + return + + try: + stat = os.lstat(path) + except OSError as exc: + import errno + if exc.errno != errno.ENOENT: + raise + else: + rootfs_path = path[len(dest):] + check_uid = int(d.getVar('HOST_USER_UID')) + if stat.st_uid == check_uid: + package_qa_add_message(messages, "host-user-contaminated", "%s: %s is owned by uid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, rootfs_path, check_uid)) + return False + + check_gid = int(d.getVar('HOST_USER_GID')) + if stat.st_gid == check_gid: + package_qa_add_message(messages, "host-user-contaminated", "%s: %s is owned by gid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, rootfs_path, check_gid)) + return False + return True + # The PACKAGE FUNC to scan each package python do_package_qa () { import subprocess @@ -978,8 +1078,8 @@ python do_package_qa () { # Check non UTF-8 characters on recipe's metadata package_qa_check_encoding(['DESCRIPTION', 'SUMMARY', 'LICENSE', 'SECTION'], 'utf-8', d) - logdir = d.getVar('T', True) - pkg = d.getVar('PN', True) + logdir = d.getVar('T') + pkg = d.getVar('PN') # Check the compile log for host contamination compilelog = os.path.join(logdir,"log.do_compile") @@ -1002,8 +1102,8 @@ python do_package_qa () { package_qa_handle_error("install-host-path", msg, d) # Scan the packages... - pkgdest = d.getVar('PKGDEST', True) - packages = set((d.getVar('PACKAGES', True) or '').split()) + pkgdest = d.getVar('PKGDEST') + packages = set((d.getVar('PACKAGES') or '').split()) cpath = oe.cachedpath.CachedPath() global pkgfiles @@ -1018,7 +1118,6 @@ python do_package_qa () { if not packages: return - testmatrix = d.getVarFlags("QAPATHTEST") import re # The package name matches the [a-z0-9.+-]+ regular expression pkgname_pattern = re.compile("^[a-z0-9.+-]+$") @@ -1028,51 +1127,62 @@ python do_package_qa () { for dep in taskdepdata: taskdeps.add(taskdepdata[dep][0]) - g = globals() - walk_sane = True - rdepends_sane = True - deps_sane = True for package in packages: - skip = (d.getVar('INSANE_SKIP_' + package, True) or "").split() + def parse_test_matrix(matrix_name): + testmatrix = d.getVarFlags(matrix_name) or {} + g = globals() + warnchecks = [] + for w in (d.getVar("WARN_QA") or "").split(): + if w in skip: + continue + if w in testmatrix and testmatrix[w] in g: + warnchecks.append(g[testmatrix[w]]) + if w == 'unsafe-references-in-binaries': + oe.utils.write_ld_so_conf(d) + + errorchecks = [] + for e in (d.getVar("ERROR_QA") or "").split(): + if e in skip: + continue + if e in testmatrix and testmatrix[e] in g: + errorchecks.append(g[testmatrix[e]]) + if e == 'unsafe-references-in-binaries': + oe.utils.write_ld_so_conf(d) + return warnchecks, errorchecks + + skip = (d.getVar('INSANE_SKIP_' + package) or "").split() if skip: bb.note("Package %s skipping QA tests: %s" % (package, str(skip))) - warnchecks = [] - for w in (d.getVar("WARN_QA", True) or "").split(): - if w in skip: - continue - if w in testmatrix and testmatrix[w] in g: - warnchecks.append(g[testmatrix[w]]) - errorchecks = [] - for e in (d.getVar("ERROR_QA", True) or "").split(): - if e in skip: - continue - if e in testmatrix and testmatrix[e] in g: - errorchecks.append(g[testmatrix[e]]) + bb.note("Checking Package: %s" % package) # Check package name if not pkgname_pattern.match(package): package_qa_handle_error("pkgname", - "%s doesn't match the [a-z0-9.+-]+ regex\n" % package, d) + "%s doesn't match the [a-z0-9.+-]+ regex" % package, d) + + warn_checks, error_checks = parse_test_matrix("QAPATHTEST") + package_qa_walk(warn_checks, error_checks, skip, package, d) - path = "%s/%s" % (pkgdest, package) - if not package_qa_walk(path, warnchecks, errorchecks, skip, package, d): - walk_sane = False - if not package_qa_check_rdepends(package, pkgdest, skip, taskdeps, packages, d): - rdepends_sane = False - if not package_qa_check_deps(package, pkgdest, skip, d): - deps_sane = False + warn_checks, error_checks = parse_test_matrix("QAPKGTEST") + package_qa_package(warn_checks, error_checks, skip, package, d) + package_qa_check_rdepends(package, pkgdest, skip, taskdeps, packages, d) + package_qa_check_deps(package, pkgdest, skip, d) - if 'libdir' in d.getVar("ALL_QA", True).split(): + if 'libdir' in d.getVar("ALL_QA").split(): package_qa_check_libdir(d) - qa_sane = d.getVar("QA_SANE", True) - if not walk_sane or not rdepends_sane or not deps_sane or not qa_sane: + qa_sane = d.getVar("QA_SANE") + if not qa_sane: bb.fatal("QA run found fatal errors. Please consider fixing them.") bb.note("DONE with PACKAGE QA") } +# binutils is used for most checks, so need to set as dependency +# POPULATESYSROOTDEPS is defined in staging class. +do_package_qa[depends] += "${POPULATESYSROOTDEPS}" +do_package_qa[vardepsexclude] = "BB_TASKDEPDATA" do_package_qa[rdeptask] = "do_packagedata" addtask do_package_qa after do_packagedata do_package before do_build @@ -1087,7 +1197,7 @@ addtask do_package_qa_setscene python do_qa_staging() { bb.note("QA checking staging") - if not package_qa_check_staged(d.expand('${SYSROOT_DESTDIR}${STAGING_LIBDIR}'), d): + if not package_qa_check_staged(d.expand('${SYSROOT_DESTDIR}${libdir}'), d): bb.fatal("QA staging was broken by the package built above") } @@ -1099,35 +1209,37 @@ python do_qa_configure() { ########################################################################### configs = [] - workdir = d.getVar('WORKDIR', True) - bb.note("Checking autotools environment for common misconfiguration") - for root, dirs, files in os.walk(workdir): - statement = "grep -e 'CROSS COMPILE Badness:' -e 'is unsafe for cross-compilation' %s > /dev/null" % \ - os.path.join(root,"config.log") - if "config.log" in files: - if subprocess.call(statement, shell=True) == 0: - bb.fatal("""This autoconf log indicates errors, it looked at host include and/or library paths while determining system capabilities. -Rerun configure task after fixing this. The path was '%s'""" % root) - - if "configure.ac" in files: - configs.append(os.path.join(root,"configure.ac")) - if "configure.in" in files: - configs.append(os.path.join(root, "configure.in")) + workdir = d.getVar('WORKDIR') + + if bb.data.inherits_class('autotools', d): + bb.note("Checking autotools environment for common misconfiguration") + for root, dirs, files in os.walk(workdir): + statement = "grep -q -F -e 'CROSS COMPILE Badness:' -e 'is unsafe for cross-compilation' %s" % \ + os.path.join(root,"config.log") + if "config.log" in files: + if subprocess.call(statement, shell=True) == 0: + bb.fatal("""This autoconf log indicates errors, it looked at host include and/or library paths while determining system capabilities. +Rerun configure task after fixing this.""") + + if "configure.ac" in files: + configs.append(os.path.join(root,"configure.ac")) + if "configure.in" in files: + configs.append(os.path.join(root, "configure.in")) ########################################################################### # Check gettext configuration and dependencies are correct ########################################################################### - cnf = d.getVar('EXTRA_OECONF', True) or "" - if "gettext" not in d.getVar('P', True) and "gcc-runtime" not in d.getVar('P', True) and "--disable-nls" not in cnf: - ml = d.getVar("MLPREFIX", True) or "" + cnf = d.getVar('EXTRA_OECONF') or "" + if "gettext" not in d.getVar('P') and "gcc-runtime" not in d.getVar('P') and "--disable-nls" not in cnf: + ml = d.getVar("MLPREFIX") or "" if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('nativesdk', d): gt = "gettext-native" elif bb.data.inherits_class('cross-canadian', d): gt = "nativesdk-gettext" else: gt = "virtual/" + ml + "gettext" - deps = bb.utils.explode_deps(d.getVar('DEPENDS', True) or "") + deps = bb.utils.explode_deps(d.getVar('DEPENDS') or "") if gt not in deps: for config in configs: gnu = "grep \"^[[:space:]]*AM_GNU_GETTEXT\" %s >/dev/null" % config @@ -1136,58 +1248,54 @@ Rerun configure task after fixing this. The path was '%s'""" % root) Missing inherit gettext?""" % (gt, config)) ########################################################################### - # Check license variables - ########################################################################### - - if not package_qa_check_license(workdir, d): - bb.fatal("Licensing Error: LIC_FILES_CHKSUM does not match, please fix") - - ########################################################################### # Check unrecognised configure options (with a white list) ########################################################################### if bb.data.inherits_class("autotools", d): bb.note("Checking configure output for unrecognised options") try: flag = "WARNING: unrecognized options:" - log = os.path.join(d.getVar('B', True), 'config.log') - output = subprocess.check_output(['grep', '-F', flag, log]).replace(', ', ' ') + log = os.path.join(d.getVar('B'), 'config.log') + output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ') options = set() for line in output.splitlines(): options |= set(line.partition(flag)[2].split()) - whitelist = set(d.getVar("UNKNOWN_CONFIGURE_WHITELIST", True).split()) + whitelist = set(d.getVar("UNKNOWN_CONFIGURE_WHITELIST").split()) options -= whitelist if options: - pn = d.getVar('PN', True) + pn = d.getVar('PN') error_msg = pn + ": configure was passed unrecognised options: " + " ".join(options) package_qa_handle_error("unknown-configure-option", error_msg, d) except subprocess.CalledProcessError: pass # Check invalid PACKAGECONFIG - pkgconfig = (d.getVar("PACKAGECONFIG", True) or "").split() + pkgconfig = (d.getVar("PACKAGECONFIG") or "").split() if pkgconfig: pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {} for pconfig in pkgconfig: if pconfig not in pkgconfigflags: - pn = d.getVar('PN', True) + pn = d.getVar('PN') error_msg = "%s: invalid PACKAGECONFIG: %s" % (pn, pconfig) - package_qa_handle_error("invalid-pkgconfig", error_msg, d) + package_qa_handle_error("invalid-packageconfig", error_msg, d) + + qa_sane = d.getVar("QA_SANE") + if not qa_sane: + bb.fatal("Fatal QA errors found, failing task.") } python do_qa_unpack() { - bb.note("Checking has ${S} been created") - - s_dir = d.getVar('S', True) - if not os.path.exists(s_dir): - bb.warn('%s: the directory %s (%s) pointed to by the S variable doesn\'t exist - please set S within the recipe to point to where the source has been unpacked to' % (d.getVar('PN', True), d.getVar('S', False), s_dir)) + src_uri = d.getVar('SRC_URI') + s_dir = d.getVar('S') + if src_uri and not os.path.exists(s_dir): + bb.warn('%s: the directory %s (%s) pointed to by the S variable doesn\'t exist - please set S within the recipe to point to where the source has been unpacked to' % (d.getVar('PN'), d.getVar('S', False), s_dir)) } # The Staging Func, to check all staging #addtask qa_staging after do_populate_sysroot before do_build do_populate_sysroot[postfuncs] += "do_qa_staging " -# Check broken config.log files, for packages requiring Gettext which don't -# have it in DEPENDS and for correct LIC_FILES_CHKSUM +# Check broken config.log files, for packages requiring Gettext which +# don't have it in DEPENDS. #addtask qa_configure after do_configure before do_compile do_configure[postfuncs] += "do_qa_configure " @@ -1195,7 +1303,7 @@ do_configure[postfuncs] += "do_qa_configure " do_unpack[postfuncs] += "do_qa_unpack" python () { - tests = d.getVar('ALL_QA', True).split() + tests = d.getVar('ALL_QA').split() if "desktop" in tests: d.appendVar("PACKAGE_DEPENDS", " desktop-file-utils-native") @@ -1204,7 +1312,7 @@ python () { ########################################################################### # Checking ${FILESEXTRAPATHS} - extrapaths = (d.getVar("FILESEXTRAPATHS", True) or "") + extrapaths = (d.getVar("FILESEXTRAPATHS") or "") if '__default' not in extrapaths.split(":"): msg = "FILESEXTRAPATHS-variable, must always use _prepend (or _append)\n" msg += "type of assignment, and don't forget the colon.\n" @@ -1216,24 +1324,29 @@ python () { msg += "%s\n" % extrapaths bb.warn(msg) - if d.getVar('do_stage', True) is not None: - bb.fatal("Legacy staging found for %s as it has a do_stage function. This will need conversion to a do_install or often simply removal to work with OE-core" % d.getVar("FILE", True)) - - overrides = d.getVar('OVERRIDES', True).split(':') - pn = d.getVar('PN', True) + overrides = d.getVar('OVERRIDES').split(':') + pn = d.getVar('PN') if pn in overrides: - msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE", True), pn) + msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE"), pn) package_qa_handle_error("pn-overrides", msg, d) issues = [] - if (d.getVar('PACKAGES', True) or "").split(): - for dep in (d.getVar('QADEPENDS', True) or "").split(): + if (d.getVar('PACKAGES') or "").split(): + for dep in (d.getVar('QADEPENDS') or "").split(): d.appendVarFlag('do_package_qa', 'depends', " %s:do_populate_sysroot" % dep) for var in 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RCONFLICTS', 'RPROVIDES', 'RREPLACES', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm', 'ALLOW_EMPTY': if d.getVar(var, False): issues.append(var) + + fakeroot_tests = d.getVar('FAKEROOT_QA').split() + if set(tests) & set(fakeroot_tests): + d.setVarFlag('do_package_qa', 'fakeroot', '1') + d.appendVarFlag('do_package_qa', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') else: d.setVarFlag('do_package_qa', 'rdeptask', '') for i in issues: - package_qa_handle_error("pkgvarcheck", "%s: Variable %s is set as not being package specific, please fix this." % (d.getVar("FILE", True), i), d) + package_qa_handle_error("pkgvarcheck", "%s: Variable %s is set as not being package specific, please fix this." % (d.getVar("FILE"), i), d) + qa_sane = d.getVar("QA_SANE") + if not qa_sane: + bb.fatal("Fatal QA errors found, failing task.") } diff --git a/meta/classes/kernel-arch.bbclass b/meta/classes/kernel-arch.bbclass index 211b72bee6..d036fcf20c 100644 --- a/meta/classes/kernel-arch.bbclass +++ b/meta/classes/kernel-arch.bbclass @@ -13,27 +13,30 @@ valid_archs = "alpha cris ia64 \ sh sh64 um h8300 \ parisc s390 v850 \ avr32 blackfin \ - microblaze" + microblaze \ + nios2 arc xtensa" def map_kernel_arch(a, d): import re - valid_archs = d.getVar('valid_archs', True).split() + valid_archs = d.getVar('valid_archs').split() if re.match('(i.86|athlon|x86.64)$', a): return 'x86' elif re.match('armeb$', a): return 'arm' elif re.match('aarch64$', a): return 'arm64' elif re.match('aarch64_be$', a): return 'arm64' - elif re.match('mips(el|64|64el)$', a): return 'mips' + elif re.match('mips(isa|)(32|64|)(r6|)(el|)$', a): return 'mips' elif re.match('p(pc|owerpc)(|64)', a): return 'powerpc' elif re.match('sh(3|4)$', a): return 'sh' elif re.match('bfin', a): return 'blackfin' elif re.match('microblazee[bl]', a): return 'microblaze' elif a in valid_archs: return a else: + if not d.getVar("TARGET_OS").startswith("linux"): + return a bb.error("cannot map '%s' to a linux kernel architecture" % a) -export ARCH = "${@map_kernel_arch(d.getVar('TARGET_ARCH', True), d)}" +export ARCH = "${@map_kernel_arch(d.getVar('TARGET_ARCH'), d)}" def map_uboot_arch(a, d): import re @@ -42,7 +45,7 @@ def map_uboot_arch(a, d): elif re.match('i.86$', a): return 'x86' return a -export UBOOT_ARCH = "${@map_uboot_arch(d.getVar('ARCH', True), d)}" +export UBOOT_ARCH = "${@map_uboot_arch(d.getVar('ARCH'), d)}" # Set TARGET_??_KERNEL_ARCH in the machine .conf to set architecture # specific options necessary for building the kernel and modules. @@ -56,4 +59,5 @@ HOST_AR_KERNEL_ARCH ?= "${TARGET_AR_KERNEL_ARCH}" KERNEL_CC = "${CCACHE}${HOST_PREFIX}gcc ${HOST_CC_KERNEL_ARCH} -fuse-ld=bfd" KERNEL_LD = "${CCACHE}${HOST_PREFIX}ld.bfd ${HOST_LD_KERNEL_ARCH}" KERNEL_AR = "${CCACHE}${HOST_PREFIX}ar ${HOST_AR_KERNEL_ARCH}" +TOOLCHAIN = "gcc" diff --git a/meta/classes/kernel-fitimage.bbclass b/meta/classes/kernel-fitimage.bbclass index 2a56a54516..2630b47316 100644 --- a/meta/classes/kernel-fitimage.bbclass +++ b/meta/classes/kernel-fitimage.bbclass @@ -1,27 +1,45 @@ -inherit kernel-uboot +inherit kernel-uboot uboot-sign python __anonymous () { - kerneltype = d.getVar('KERNEL_IMAGETYPE', True) - if kerneltype == 'fitImage': - depends = d.getVar("DEPENDS", True) + kerneltypes = d.getVar('KERNEL_IMAGETYPES') or "" + if 'fitImage' in kerneltypes.split(): + depends = d.getVar("DEPENDS") depends = "%s u-boot-mkimage-native dtc-native" % depends d.setVar("DEPENDS", depends) + if d.getVar("UBOOT_ARCH") == "x86": + replacementtype = "bzImage" + else: + replacementtype = "zImage" + # Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal # to kernel.bbclass . We have to override it, since we pack zImage # (at least for now) into the fitImage . - d.setVar("KERNEL_IMAGETYPE_FOR_MAKE", "zImage") + typeformake = d.getVar("KERNEL_IMAGETYPE_FOR_MAKE") or "" + if 'fitImage' in typeformake.split(): + d.setVar('KERNEL_IMAGETYPE_FOR_MAKE', typeformake.replace('fitImage', replacementtype)) - image = d.getVar('INITRAMFS_IMAGE', True) + image = d.getVar('INITRAMFS_IMAGE') if image: - d.appendVarFlag('do_assemble_fitimage', 'depends', ' ${INITRAMFS_IMAGE}:do_rootfs') + d.appendVarFlag('do_assemble_fitimage_initramfs', 'depends', ' ${INITRAMFS_IMAGE}:do_image_complete') + + # Verified boot will sign the fitImage and append the public key to + # U-Boot dtb. We ensure the U-Boot dtb is deployed before assembling + # the fitImage: + if d.getVar('UBOOT_SIGN_ENABLE') == "1": + uboot_pn = d.getVar('PREFERRED_PROVIDER_u-boot') or 'u-boot' + d.appendVarFlag('do_assemble_fitimage', 'depends', ' %s:do_deploy' % uboot_pn) } +# Options for the device tree compiler passed to mkimage '-D' feature: +UBOOT_MKIMAGE_DTCOPTS ??= "" + # # Emit the fitImage ITS header # +# $1 ... .its filename fitimage_emit_fit_header() { - cat << EOF >> fit-image.its + cat << EOF >> ${1} /dts-v1/; / { @@ -33,32 +51,33 @@ EOF # # Emit the fitImage section bits # -# $1 ... Section bit type: imagestart - image section start +# $1 ... .its filename +# $2 ... Section bit type: imagestart - image section start # confstart - configuration section start # sectend - section end # fitend - fitimage end # fitimage_emit_section_maint() { - case $1 in + case $2 in imagestart) - cat << EOF >> fit-image.its + cat << EOF >> ${1} images { EOF ;; confstart) - cat << EOF >> fit-image.its + cat << EOF >> ${1} configurations { EOF ;; sectend) - cat << EOF >> fit-image.its + cat << EOF >> ${1} }; EOF ;; fitend) - cat << EOF >> fit-image.its + cat << EOF >> ${1} }; EOF ;; @@ -68,27 +87,28 @@ EOF # # Emit the fitImage ITS kernel section # -# $1 ... Image counter -# $2 ... Path to kernel image -# $3 ... Compression type +# $1 ... .its filename +# $2 ... Image counter +# $3 ... Path to kernel image +# $4 ... Compression type fitimage_emit_section_kernel() { kernel_csum="sha1" ENTRYPOINT=${UBOOT_ENTRYPOINT} - if test -n "${UBOOT_ENTRYSYMBOL}"; then + if [ -n "${UBOOT_ENTRYSYMBOL}" ]; then ENTRYPOINT=`${HOST_PREFIX}nm ${S}/vmlinux | \ - awk '$3=="${UBOOT_ENTRYSYMBOL}" {print $1}'` + awk '$4=="${UBOOT_ENTRYSYMBOL}" {print $2}'` fi - cat << EOF >> fit-image.its - kernel@${1} { + cat << EOF >> ${1} + kernel@${2} { description = "Linux kernel"; - data = /incbin/("${2}"); + data = /incbin/("${3}"); type = "kernel"; arch = "${UBOOT_ARCH}"; os = "linux"; - compression = "${3}"; + compression = "${4}"; load = <${UBOOT_LOADADDRESS}>; entry = <${ENTRYPOINT}>; hash@1 { @@ -101,16 +121,17 @@ EOF # # Emit the fitImage ITS DTB section # -# $1 ... Image counter -# $2 ... Path to DTB image +# $1 ... .its filename +# $2 ... Image counter +# $3 ... Path to DTB image fitimage_emit_section_dtb() { dtb_csum="sha1" - cat << EOF >> fit-image.its - fdt@${1} { + cat << EOF >> ${1} + fdt@${2} { description = "Flattened Device Tree blob"; - data = /incbin/("${2}"); + data = /incbin/("${3}"); type = "flat_dt"; arch = "${UBOOT_ARCH}"; compression = "none"; @@ -122,113 +143,337 @@ EOF } # +# Emit the fitImage ITS setup section +# +# $1 ... .its filename +# $2 ... Image counter +# $3 ... Path to setup image +fitimage_emit_section_setup() { + + setup_csum="sha1" + + cat << EOF >> ${1} + setup@${2} { + description = "Linux setup.bin"; + data = /incbin/("${3}"); + type = "x86_setup"; + arch = "${UBOOT_ARCH}"; + os = "linux"; + compression = "none"; + load = <0x00090000>; + entry = <0x00090000>; + hash@1 { + algo = "${setup_csum}"; + }; + }; +EOF +} + +# +# Emit the fitImage ITS ramdisk section +# +# $1 ... .its filename +# $2 ... Image counter +# $3 ... Path to ramdisk image +fitimage_emit_section_ramdisk() { + + ramdisk_csum="sha1" + ramdisk_ctype="none" + ramdisk_loadline="" + ramdisk_entryline="" + + if [ -n "${UBOOT_RD_LOADADDRESS}" ]; then + ramdisk_loadline="load = <${UBOOT_RD_LOADADDRESS}>;" + fi + if [ -n "${UBOOT_RD_ENTRYPOINT}" ]; then + ramdisk_entryline="entry = <${UBOOT_RD_ENTRYPOINT}>;" + fi + + case $3 in + *.gz) + ramdisk_ctype="gzip" + ;; + *.bz2) + ramdisk_ctype="bzip2" + ;; + *.lzma) + ramdisk_ctype="lzma" + ;; + *.lzo) + ramdisk_ctype="lzo" + ;; + *.lz4) + ramdisk_ctype="lz4" + ;; + esac + + cat << EOF >> ${1} + ramdisk@${2} { + description = "${INITRAMFS_IMAGE}"; + data = /incbin/("${3}"); + type = "ramdisk"; + arch = "${UBOOT_ARCH}"; + os = "linux"; + compression = "${ramdisk_ctype}"; + ${ramdisk_loadline} + ${ramdisk_entryline} + hash@1 { + algo = "${ramdisk_csum}"; + }; + }; +EOF +} + +# # Emit the fitImage ITS configuration section # -# $1 ... Linux kernel ID -# $2 ... DTB image ID +# $1 ... .its filename +# $2 ... Linux kernel ID +# $3 ... DTB image name +# $4 ... ramdisk ID +# $5 ... config ID +# $6 ... default flag fitimage_emit_section_config() { conf_csum="sha1" + if [ -n "${UBOOT_SIGN_ENABLE}" ] ; then + conf_sign_keyname="${UBOOT_SIGN_KEYNAME}" + fi # Test if we have any DTBs at all - if [ -z "${2}" ] ; then - conf_desc="Boot Linux kernel" - fdt_line="" - else - conf_desc="Boot Linux kernel with FDT blob" - fdt_line="fdt = \"fdt@${2}\";" - fi - kernel_line="kernel = \"kernel@${1}\";" - - cat << EOF >> fit-image.its - default = "conf@1"; - conf@1 { - description = "${conf_desc}"; + conf_desc="Linux kernel" + kernel_line="kernel = \"kernel@${2}\";" + fdt_line="" + ramdisk_line="" + setup_line="" + default_line="" + + if [ -n "${3}" ]; then + conf_desc="${conf_desc}, FDT blob" + fdt_line="fdt = \"fdt@${3}\";" + fi + + if [ -n "${4}" ]; then + conf_desc="${conf_desc}, ramdisk" + ramdisk_line="ramdisk = \"ramdisk@${4}\";" + fi + + if [ -n "${5}" ]; then + conf_desc="${conf_desc}, setup" + setup_line="setup = \"setup@${5}\";" + fi + + if [ "${6}" = "1" ]; then + default_line="default = \"conf@${3}\";" + fi + + cat << EOF >> ${1} + ${default_line} + conf@${3} { + description = "${6} ${conf_desc}"; ${kernel_line} ${fdt_line} + ${ramdisk_line} + ${setup_line} hash@1 { algo = "${conf_csum}"; }; - }; EOF -} -do_assemble_fitimage() { - if test "x${KERNEL_IMAGETYPE}" = "xfitImage" ; then - kernelcount=1 - dtbcount="" - rm -f fit-image.its - - fitimage_emit_fit_header - - # - # Step 1: Prepare a kernel image section. - # - fitimage_emit_section_maint imagestart - - uboot_prep_kimage - fitimage_emit_section_kernel "${kernelcount}" linux.bin "${linux_comp}" - - # - # Step 2: Prepare a DTB image section - # - if test -n "${KERNEL_DEVICETREE}"; then - dtbcount=1 - for DTB in ${KERNEL_DEVICETREE}; do - if echo ${DTB} | grep -q '/dts/'; then - bbwarn "${DTB} contains the full path to the the dts file, but only the dtb name should be used." - DTB=`basename ${DTB} | sed 's,\.dts$,.dtb,g'` - fi - DTB_PATH="arch/${ARCH}/boot/dts/${DTB}" - if [ ! -e "${DTB_PATH}" ]; then - DTB_PATH="arch/${ARCH}/boot/${DTB}" - fi - - fitimage_emit_section_dtb ${dtbcount} ${DTB_PATH} - dtbcount=`expr ${dtbcount} + 1` - done + if [ ! -z "${conf_sign_keyname}" ] ; then + + sign_line="sign-images = \"kernel\"" + + if [ -n "${3}" ]; then + sign_line="${sign_line}, \"fdt\"" + fi + + if [ -n "${4}" ]; then + sign_line="${sign_line}, \"ramdisk\"" fi - fitimage_emit_section_maint sectend + if [ -n "${5}" ]; then + sign_line="${sign_line}, \"setup\"" + fi + + sign_line="${sign_line};" + + cat << EOF >> ${1} + signature@1 { + algo = "${conf_csum},rsa2048"; + key-name-hint = "${conf_sign_keyname}"; + ${sign_line} + }; +EOF + fi + + cat << EOF >> ${1} + }; +EOF +} - # Force the first Kernel and DTB in the default config - kernelcount=1 +# +# Assemble fitImage +# +# $1 ... .its filename +# $2 ... fitImage name +# $3 ... include ramdisk +fitimage_assemble() { + kernelcount=1 + dtbcount="" + DTBS="" + ramdiskcount=${3} + setupcount="" + rm -f ${1} arch/${ARCH}/boot/${2} + + fitimage_emit_fit_header ${1} + + # + # Step 1: Prepare a kernel image section. + # + fitimage_emit_section_maint ${1} imagestart + + uboot_prep_kimage + fitimage_emit_section_kernel ${1} "${kernelcount}" linux.bin "${linux_comp}" + + # + # Step 2: Prepare a DTB image section + # + if [ -n "${KERNEL_DEVICETREE}" ]; then dtbcount=1 + for DTB in ${KERNEL_DEVICETREE}; do + if echo ${DTB} | grep -q '/dts/'; then + bbwarn "${DTB} contains the full path to the the dts file, but only the dtb name should be used." + DTB=`basename ${DTB} | sed 's,\.dts$,.dtb,g'` + fi + DTB_PATH="arch/${ARCH}/boot/dts/${DTB}" + if [ ! -e "${DTB_PATH}" ]; then + DTB_PATH="arch/${ARCH}/boot/${DTB}" + fi + + DTBS="${DTBS} ${DTB}" + fitimage_emit_section_dtb ${1} ${DTB} ${DTB_PATH} + done + fi - # - # Step 3: Prepare a configurations section - # - fitimage_emit_section_maint confstart + # + # Step 3: Prepare a setup section. (For x86) + # + if [ -e arch/${ARCH}/boot/setup.bin ]; then + setupcount=1 + fitimage_emit_section_setup ${1} "${setupcount}" arch/${ARCH}/boot/setup.bin + fi - fitimage_emit_section_config ${kernelcount} ${dtbcount} + # + # Step 4: Prepare a ramdisk section. + # + if [ "x${ramdiskcount}" = "x1" ] ; then + # Find and use the first initramfs image archive type we find + for img in cpio.lz4 cpio.lzo cpio.lzma cpio.xz cpio.gz cpio; do + initramfs_path="${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE_NAME}.${img}" + echo "Using $initramfs_path" + if [ -e "${initramfs_path}" ]; then + fitimage_emit_section_ramdisk ${1} "${ramdiskcount}" "${initramfs_path}" + break + fi + done + fi - fitimage_emit_section_maint sectend + fitimage_emit_section_maint ${1} sectend - fitimage_emit_section_maint fitend + # Force the first Kernel and DTB in the default config + kernelcount=1 + if [ -n "${dtbcount}" ]; then + dtbcount=1 + fi + + # + # Step 5: Prepare a configurations section + # + fitimage_emit_section_maint ${1} confstart + + if [ -n "${DTBS}" ]; then + i=1 + for DTB in ${DTBS}; do + fitimage_emit_section_config ${1} "${kernelcount}" "${DTB}" "${ramdiskcount}" "${setupcount}" "`expr ${i} = ${dtbcount}`" + i=`expr ${i} + 1` + done + fi + + fitimage_emit_section_maint ${1} sectend + + fitimage_emit_section_maint ${1} fitend + + # + # Step 6: Assemble the image + # + uboot-mkimage \ + ${@'-D "${UBOOT_MKIMAGE_DTCOPTS}"' if len('${UBOOT_MKIMAGE_DTCOPTS}') else ''} \ + -f ${1} \ + arch/${ARCH}/boot/${2} + + # + # Step 7: Sign the image and add public key to U-Boot dtb + # + if [ "x${UBOOT_SIGN_ENABLE}" = "x1" ] ; then + uboot-mkimage \ + ${@'-D "${UBOOT_MKIMAGE_DTCOPTS}"' if len('${UBOOT_MKIMAGE_DTCOPTS}') else ''} \ + -F -k "${UBOOT_SIGN_KEYDIR}" \ + -K "${DEPLOY_DIR_IMAGE}/${UBOOT_DTB_BINARY}" \ + -r arch/${ARCH}/boot/${2} + fi +} - # - # Step 4: Assemble the image - # - uboot-mkimage -f fit-image.its arch/${ARCH}/boot/fitImage +do_assemble_fitimage() { + if echo ${KERNEL_IMAGETYPES} | grep -wq "fitImage"; then + cd ${B} + fitimage_assemble fit-image.its fitImage fi } addtask assemble_fitimage before do_install after do_compile +do_assemble_fitimage_initramfs() { + if echo ${KERNEL_IMAGETYPES} | grep -wq "fitImage" && \ + test -n "${INITRAMFS_IMAGE}" ; then + cd ${B} + fitimage_assemble fit-image-${INITRAMFS_IMAGE}.its fitImage-${INITRAMFS_IMAGE} 1 + fi +} + +addtask assemble_fitimage_initramfs before do_deploy after do_install + + +kernel_do_deploy[vardepsexclude] = "DATETIME" kernel_do_deploy_append() { # Update deploy directory - if test "x${KERNEL_IMAGETYPE}" = "xfitImage" ; then + if echo ${KERNEL_IMAGETYPES} | grep -wq "fitImage"; then cd ${B} echo "Copying fit-image.its source file..." - its_base_name="${KERNEL_IMAGETYPE}-its-${PV}-${PR}-${MACHINE}-${DATETIME}" - its_symlink_name=${KERNEL_IMAGETYPE}-its-${MACHINE} + its_base_name="fitImage-its-${PV}-${PR}-${MACHINE}-${DATETIME}" + its_symlink_name=fitImage-its-${MACHINE} install -m 0644 fit-image.its ${DEPLOYDIR}/${its_base_name}.its - linux_bin_base_name="${KERNEL_IMAGETYPE}-linux.bin-${PV}-${PR}-${MACHINE}-${DATETIME}" - linux_bin_symlink_name=${KERNEL_IMAGETYPE}-linux.bin-${MACHINE} + linux_bin_base_name="fitImage-linux.bin-${PV}-${PR}-${MACHINE}-${DATETIME}" + linux_bin_symlink_name=fitImage-linux.bin-${MACHINE} install -m 0644 linux.bin ${DEPLOYDIR}/${linux_bin_base_name}.bin + if [ -n "${INITRAMFS_IMAGE}" ]; then + echo "Copying fit-image-${INITRAMFS_IMAGE}.its source file..." + its_initramfs_base_name="fitImage-its-${INITRAMFS_IMAGE_NAME}-${PV}-${PR}-${DATETIME}" + its_initramfs_symlink_name=fitImage-its-${INITRAMFS_IMAGE_NAME} + install -m 0644 fit-image-${INITRAMFS_IMAGE}.its ${DEPLOYDIR}/${its_initramfs_base_name}.its + fit_initramfs_base_name="fitImage-${INITRAMFS_IMAGE_NAME}-${PV}-${PR}-${DATETIME}" + fit_initramfs_symlink_name=fitImage-${INITRAMFS_IMAGE_NAME} + install -m 0644 arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} ${DEPLOYDIR}/${fit_initramfs_base_name}.bin + fi + cd ${DEPLOYDIR} ln -sf ${its_base_name}.its ${its_symlink_name}.its ln -sf ${linux_bin_base_name}.bin ${linux_bin_symlink_name}.bin + + if [ -n "${INITRAMFS_IMAGE}" ]; then + ln -sf ${its_initramfs_base_name}.its ${its_initramfs_symlink_name}.its + ln -sf ${fit_initramfs_base_name}.bin ${fit_initramfs_symlink_name}.bin + fi fi } diff --git a/meta/classes/kernel-grub.bbclass b/meta/classes/kernel-grub.bbclass index a63f482a91..5d92f3b636 100644 --- a/meta/classes/kernel-grub.bbclass +++ b/meta/classes/kernel-grub.bbclass @@ -10,41 +10,44 @@ # updates the new kernel as the boot priority. # -pkg_preinst_kernel-image_append () { +python __anonymous () { + import re + + preinst = ''' # Parsing confliction [ -f "$D/boot/grub/menu.list" ] && grubcfg="$D/boot/grub/menu.list" [ -f "$D/boot/grub/grub.cfg" ] && grubcfg="$D/boot/grub/grub.cfg" if [ -n "$grubcfg" ]; then # Dereference symlink to avoid confliction with new kernel name. - if grep -q "/${KERNEL_IMAGETYPE} \+root=" $grubcfg; then - if [ -L "$D/boot/${KERNEL_IMAGETYPE}" ]; then - kimage=`realpath $D/boot/${KERNEL_IMAGETYPE} 2>/dev/null` + if grep -q "/KERNEL_IMAGETYPE \+root=" $grubcfg; then + if [ -L "$D/boot/KERNEL_IMAGETYPE" ]; then + kimage=`realpath $D/boot/KERNEL_IMAGETYPE 2>/dev/null` if [ -f "$D$kimage" ]; then - sed -i "s:${KERNEL_IMAGETYPE} \+root=:${kimage##*/} root=:" $grubcfg + sed -i "s:KERNEL_IMAGETYPE \+root=:${kimage##*/} root=:" $grubcfg fi fi fi # Rename old kernel if it conflicts with new kernel name. - if grep -q "/${KERNEL_IMAGETYPE}-${KERNEL_VERSION} \+root=" $grubcfg; then - if [ -f "$D/boot/${KERNEL_IMAGETYPE}-${KERNEL_VERSION}" ]; then + if grep -q "/KERNEL_IMAGETYPE-${KERNEL_VERSION} \+root=" $grubcfg; then + if [ -f "$D/boot/KERNEL_IMAGETYPE-${KERNEL_VERSION}" ]; then timestamp=`date +%s` - kimage="$D/boot/${KERNEL_IMAGETYPE}-${KERNEL_VERSION}-$timestamp-back" - sed -i "s:${KERNEL_IMAGETYPE}-${KERNEL_VERSION} \+root=:${kimage##*/} root=:" $grubcfg - mv "$D/boot/${KERNEL_IMAGETYPE}-${KERNEL_VERSION}" "$kimage" + kimage="$D/boot/KERNEL_IMAGETYPE-${KERNEL_VERSION}-$timestamp-back" + sed -i "s:KERNEL_IMAGETYPE-${KERNEL_VERSION} \+root=:${kimage##*/} root=:" $grubcfg + mv "$D/boot/KERNEL_IMAGETYPE-${KERNEL_VERSION}" "$kimage" fi fi fi -} +''' -pkg_postinst_kernel-image_prepend () { + postinst = ''' get_new_grub_cfg() { grubcfg="$1" old_image="$2" - title="Update ${KERNEL_IMAGETYPE}-${KERNEL_VERSION}-${PV}" + title="Update KERNEL_IMAGETYPE-${KERNEL_VERSION}-${PV}" if [ "${grubcfg##*/}" = "grub.cfg" ]; then rootfs=`grep " *linux \+[^ ]\+ \+root=" $grubcfg -m 1 | \ - sed "s#${old_image}#${old_image%/*}/${KERNEL_IMAGETYPE}-${KERNEL_VERSION}#"` + sed "s#${old_image}#${old_image%/*}/KERNEL_IMAGETYPE-${KERNEL_VERSION}#"` echo "menuentry \"$title\" {" echo " set root=(hd0,1)" @@ -52,7 +55,7 @@ pkg_postinst_kernel-image_prepend () { echo "}" elif [ "${grubcfg##*/}" = "menu.list" ]; then rootfs=`grep "kernel \+[^ ]\+ \+root=" $grubcfg -m 1 | \ - sed "s#${old_image}#${old_image%/*}/${KERNEL_IMAGETYPE}-${KERNEL_VERSION}#"` + sed "s#${old_image}#${old_image%/*}/KERNEL_IMAGETYPE-${KERNEL_VERSION}#"` echo "default 0" echo "timeout 30" @@ -87,5 +90,16 @@ pkg_postinst_kernel-image_prepend () { mv $grubcfgtmp $grubcfg echo "Caution! Update kernel may affect kernel-module!" fi +''' + + imagetypes = d.getVar('KERNEL_IMAGETYPES') + imagetypes = re.sub(r'\.gz$', '', imagetypes) + + for type in imagetypes.split(): + typelower = type.lower() + preinst_append = preinst.replace('KERNEL_IMAGETYPE', type) + postinst_prepend = postinst.replace('KERNEL_IMAGETYPE', type) + d.setVar('pkg_preinst_kernel-image-' + typelower + '_append', preinst_append) + d.setVar('pkg_postinst_kernel-image-' + typelower + '_prepend', postinst_prepend) } diff --git a/meta/classes/kernel-module-split.bbclass b/meta/classes/kernel-module-split.bbclass index e1a70e6215..5e10dcf735 100644 --- a/meta/classes/kernel-module-split.bbclass +++ b/meta/classes/kernel-module-split.bbclass @@ -22,6 +22,8 @@ if [ x"$D" = "x" ]; then fi } +PACKAGE_WRITE_DEPS += "kmod-native depmodwrapper-cross" + do_install_append() { install -d ${D}${sysconfdir}/modules-load.d/ ${D}${sysconfdir}/modprobe.d/ } @@ -30,23 +32,21 @@ PACKAGESPLITFUNCS_prepend = "split_kernel_module_packages " KERNEL_MODULES_META_PACKAGE ?= "kernel-modules" +KERNEL_MODULE_PACKAGE_PREFIX ?= "" +KERNEL_MODULE_PACKAGE_SUFFIX ?= "-${KERNEL_VERSION}" +KERNEL_MODULE_PROVIDE_VIRTUAL ?= "1" + python split_kernel_module_packages () { import re modinfoexp = re.compile("([^=]+)=(.*)") - kerverrexp = re.compile('^(.*-hh.*)[\.\+].*$') - depmodpat0 = re.compile("^(.*\.k?o):..*$") - depmodpat1 = re.compile("^(.*\.k?o):\s*(.*\.k?o)\s*$") - depmodpat2 = re.compile("^(.*\.k?o):\s*(.*\.k?o)\s*\\\$") - depmodpat3 = re.compile("^\t(.*\.k?o)\s*\\\$") - depmodpat4 = re.compile("^\t(.*\.k?o)\s*$") def extract_modinfo(file): import tempfile, subprocess - tempfile.tempdir = d.getVar("WORKDIR", True) + tempfile.tempdir = d.getVar("WORKDIR") tf = tempfile.mkstemp() tmpfile = tf[1] - cmd = "%sobjcopy -j .modinfo -O binary %s %s" % (d.getVar("HOST_PREFIX", True) or "", file, tmpfile) + cmd = "%sobjcopy -j .modinfo -O binary %s %s" % (d.getVar("HOST_PREFIX") or "", file, tmpfile) subprocess.call(cmd, shell=True) f = open(tmpfile) l = f.read().split("\000") @@ -61,77 +61,15 @@ python split_kernel_module_packages () { vals[m.group(1)] = m.group(2) return vals - def parse_depmod(): - - dvar = d.getVar('PKGD', True) - - kernelver = d.getVar('KERNEL_VERSION', True) - kernelver_stripped = kernelver - m = kerverrexp.match(kernelver) - if m: - kernelver_stripped = m.group(1) - staging_kernel_dir = d.getVar("STAGING_KERNEL_BUILDDIR", True) - system_map_file = "%s/boot/System.map-%s" % (dvar, kernelver) - if not os.path.exists(system_map_file): - system_map_file = "%s/System.map-%s" % (staging_kernel_dir, kernelver) - if not os.path.exists(system_map_file): - bb.fatal("System.map-%s does not exist in '%s/boot' nor STAGING_KERNEL_BUILDDIR '%s'" % (kernelver, dvar, staging_kernel_dir)) - - cmd = "depmod -n -a -b %s -F %s %s" % (dvar, system_map_file, kernelver_stripped) - f = os.popen(cmd, 'r') - - deps = {} - line = f.readline() - while line: - if not depmodpat0.match(line): - line = f.readline() - continue - m1 = depmodpat1.match(line) - if m1: - deps[m1.group(1)] = m1.group(2).split() - else: - m2 = depmodpat2.match(line) - if m2: - deps[m2.group(1)] = m2.group(2).split() - line = f.readline() - m3 = depmodpat3.match(line) - while m3: - deps[m2.group(1)].extend(m3.group(1).split()) - line = f.readline() - m3 = depmodpat3.match(line) - m4 = depmodpat4.match(line) - deps[m2.group(1)].extend(m4.group(1).split()) - line = f.readline() - f.close() - return deps - - def get_dependencies(file, pattern, format): - # file no longer includes PKGD - file = file.replace(d.getVar('PKGD', True) or '', '', 1) - # instead is prefixed with /lib/modules/${KERNEL_VERSION} - file = file.replace("/lib/modules/%s/" % d.getVar('KERNEL_VERSION', True) or '', '', 1) - - if file in module_deps: - dependencies = [] - for i in module_deps[file]: - m = re.match(pattern, os.path.basename(i)) - if not m: - continue - on = legitimize_package_name(m.group(1)) - dependency_pkg = format % on - dependencies.append(dependency_pkg) - return dependencies - return [] - def frob_metadata(file, pkg, pattern, format, basename): vals = extract_modinfo(file) - dvar = d.getVar('PKGD', True) + dvar = d.getVar('PKGD') # If autoloading is requested, output /etc/modules-load.d/<name>.conf and append # appropriate modprobe commands to the postinst - autoloadlist = (d.getVar("KERNEL_MODULE_AUTOLOAD", True) or "").split() - autoload = d.getVar('module_autoload_%s' % basename, True) + autoloadlist = (d.getVar("KERNEL_MODULE_AUTOLOAD") or "").split() + autoload = d.getVar('module_autoload_%s' % basename) if autoload and autoload == basename: bb.warn("module_autoload_%s was replaced by KERNEL_MODULE_AUTOLOAD for cases where basename == module name, please drop it" % basename) if autoload and basename not in autoloadlist: @@ -145,15 +83,15 @@ python split_kernel_module_packages () { else: f.write('%s\n' % basename) f.close() - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: bb.fatal("pkg_postinst_%s not defined" % pkg) - postinst += d.getVar('autoload_postinst_fragment', True) % (autoload or basename) + postinst += d.getVar('autoload_postinst_fragment') % (autoload or basename) d.setVar('pkg_postinst_%s' % pkg, postinst) # Write out any modconf fragment - modconflist = (d.getVar("KERNEL_MODULE_PROBECONF", True) or "").split() - modconf = d.getVar('module_conf_%s' % basename, True) + modconflist = (d.getVar("KERNEL_MODULE_PROBECONF") or "").split() + modconf = d.getVar('module_conf_%s' % basename) if modconf and basename in modconflist: name = '%s/etc/modprobe.d/%s.conf' % (dvar, basename) f = open(name, 'w') @@ -162,16 +100,22 @@ python split_kernel_module_packages () { elif modconf: bb.error("Please ensure module %s is listed in KERNEL_MODULE_PROBECONF since module_conf_%s is set" % (basename, basename)) - files = d.getVar('FILES_%s' % pkg, True) + files = d.getVar('FILES_%s' % pkg) files = "%s /etc/modules-load.d/%s.conf /etc/modprobe.d/%s.conf" % (files, basename, basename) d.setVar('FILES_%s' % pkg, files) if "description" in vals: - old_desc = d.getVar('DESCRIPTION_' + pkg, True) or "" + old_desc = d.getVar('DESCRIPTION_' + pkg) or "" d.setVar('DESCRIPTION_' + pkg, old_desc + "; " + vals["description"]) - rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or "") - for dep in get_dependencies(file, pattern, format): + rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg) or "") + modinfo_deps = [] + if "depends" in vals and vals["depends"] != "": + for dep in vals["depends"].split(","): + on = legitimize_package_name(dep) + dependency_pkg = format % on + modinfo_deps.append(dependency_pkg) + for dep in modinfo_deps: if not dep in rdepends: rdepends[dep] = [] d.setVar('RDEPENDS_' + pkg, bb.utils.join_deps(rdepends, commasep=False)) @@ -179,25 +123,33 @@ python split_kernel_module_packages () { # Avoid automatic -dev recommendations for modules ending with -dev. d.setVarFlag('RRECOMMENDS_' + pkg, 'nodeprrecs', 1) - module_deps = parse_depmod() + # Provide virtual package without postfix + providevirt = d.getVar('KERNEL_MODULE_PROVIDE_VIRTUAL') + if providevirt == "1": + postfix = format.split('%s')[1] + d.setVar('RPROVIDES_' + pkg, pkg.replace(postfix, '')) + module_regex = '^(.*)\.k?o$' - module_pattern = 'kernel-module-%s' - postinst = d.getVar('pkg_postinst_modules', True) - postrm = d.getVar('pkg_postrm_modules', True) + module_pattern_prefix = d.getVar('KERNEL_MODULE_PACKAGE_PREFIX') + module_pattern_suffix = d.getVar('KERNEL_MODULE_PACKAGE_SUFFIX') + module_pattern = module_pattern_prefix + 'kernel-module-%s' + module_pattern_suffix + + postinst = d.getVar('pkg_postinst_modules') + postrm = d.getVar('pkg_postrm_modules') - modules = do_split_packages(d, root='/lib/modules', file_regex=module_regex, output_pattern=module_pattern, description='%s kernel module', postinst=postinst, postrm=postrm, recursive=True, hook=frob_metadata, extra_depends='kernel-%s' % (d.getVar("KERNEL_VERSION", True))) + modules = do_split_packages(d, root='${nonarch_base_libdir}/modules', file_regex=module_regex, output_pattern=module_pattern, description='%s kernel module', postinst=postinst, postrm=postrm, recursive=True, hook=frob_metadata, extra_depends='kernel-%s' % (d.getVar("KERNEL_VERSION"))) if modules: - metapkg = d.getVar('KERNEL_MODULES_META_PACKAGE', True) + metapkg = d.getVar('KERNEL_MODULES_META_PACKAGE') d.appendVar('RDEPENDS_' + metapkg, ' '+' '.join(modules)) # If modules-load.d and modprobe.d are empty at this point, remove them to # avoid warnings. removedirs only raises an OSError if an empty # directory cannot be removed. - dvar = d.getVar('PKGD', True) + dvar = d.getVar('PKGD') for dir in ["%s/etc/modprobe.d" % (dvar), "%s/etc/modules-load.d" % (dvar), "%s/etc" % (dvar)]: if len(os.listdir(dir)) == 0: os.rmdir(dir) } -do_package[vardeps] += '${@" ".join(map(lambda s: "module_conf_" + s, (d.getVar("KERNEL_MODULE_PROBECONF", True) or "").split()))}' +do_package[vardeps] += '${@" ".join(map(lambda s: "module_conf_" + s, (d.getVar("KERNEL_MODULE_PROBECONF") or "").split()))}' diff --git a/meta/classes/kernel-uboot.bbclass b/meta/classes/kernel-uboot.bbclass index 345e7f5f3b..868e97d7a7 100644 --- a/meta/classes/kernel-uboot.bbclass +++ b/meta/classes/kernel-uboot.bbclass @@ -1,5 +1,5 @@ uboot_prep_kimage() { - if test -e arch/${ARCH}/boot/compressed/vmlinux ; then + if [ -e arch/${ARCH}/boot/compressed/vmlinux ]; then vmlinux_path="arch/${ARCH}/boot/compressed/vmlinux" linux_suffix="" linux_comp="none" diff --git a/meta/classes/kernel-uimage.bbclass b/meta/classes/kernel-uimage.bbclass index f73965b13f..19c6ade507 100644 --- a/meta/classes/kernel-uimage.bbclass +++ b/meta/classes/kernel-uimage.bbclass @@ -1,33 +1,34 @@ inherit kernel-uboot python __anonymous () { - kerneltype = d.getVar('KERNEL_IMAGETYPE', True) - if kerneltype == 'uImage': - depends = d.getVar("DEPENDS", True) + if "uImage" in (d.getVar('KERNEL_IMAGETYPES') or "").split(): + depends = d.getVar("DEPENDS") depends = "%s u-boot-mkimage-native" % depends d.setVar("DEPENDS", depends) - # Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal - # to kernel.bbclass . We override the variable here, since we need - # to build uImage using the kernel build system if and only if - # KEEPUIMAGE == yes. Otherwise, we pack compressed vmlinux into - # the uImage . - if d.getVar("KEEPUIMAGE", True) != 'yes': - d.setVar("KERNEL_IMAGETYPE_FOR_MAKE", "zImage") + # Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal + # to kernel.bbclass . We override the variable here, since we need + # to build uImage using the kernel build system if and only if + # KEEPUIMAGE == yes. Otherwise, we pack compressed vmlinux into + # the uImage . + if d.getVar("KEEPUIMAGE") != 'yes': + typeformake = d.getVar("KERNEL_IMAGETYPE_FOR_MAKE") or "" + if "uImage" in typeformake.split(): + d.setVar('KERNEL_IMAGETYPE_FOR_MAKE', typeformake.replace('uImage', 'vmlinux')) } do_uboot_mkimage() { - if test "x${KERNEL_IMAGETYPE}" = "xuImage" ; then + if echo "${KERNEL_IMAGETYPES}" | grep -wq "uImage"; then if test "x${KEEPUIMAGE}" != "xyes" ; then uboot_prep_kimage ENTRYPOINT=${UBOOT_ENTRYPOINT} - if test -n "${UBOOT_ENTRYSYMBOL}"; then + if [ -n "${UBOOT_ENTRYSYMBOL}" ]; then ENTRYPOINT=`${HOST_PREFIX}nm ${S}/vmlinux | \ awk '$3=="${UBOOT_ENTRYSYMBOL}" {print $1}'` fi - uboot-mkimage -A ${UBOOT_ARCH} -O linux -T kernel -C "${linux_comp}" -a ${UBOOT_LOADADDRESS} -e $ENTRYPOINT -n "${DISTRO_NAME}/${PV}/${MACHINE}" -d linux.bin arch/${ARCH}/boot/uImage + uboot-mkimage -A ${UBOOT_ARCH} -O linux -T kernel -C "${linux_comp}" -a ${UBOOT_LOADADDRESS} -e $ENTRYPOINT -n "${DISTRO_NAME}/${PV}/${MACHINE}" -d linux.bin ${B}/arch/${ARCH}/boot/uImage rm -f linux.bin fi fi diff --git a/meta/classes/kernel-yocto.bbclass b/meta/classes/kernel-yocto.bbclass index 231e08deba..50226f6d49 100644 --- a/meta/classes/kernel-yocto.bbclass +++ b/meta/classes/kernel-yocto.bbclass @@ -1,5 +1,7 @@ # remove tasks that modify the source tree in case externalsrc is inherited -SRCTREECOVEREDTASKS += "do_kernel_link_vmlinux do_kernel_configme do_validate_branches do_kernel_configcheck do_kernel_checkout do_shared_workdir do_fetch do_unpack do_patch" +SRCTREECOVEREDTASKS += "do_kernel_configme do_validate_branches do_kernel_configcheck do_kernel_checkout do_fetch do_unpack do_patch" +PATCH_GIT_USER_EMAIL ?= "kernel-yocto@oe" +PATCH_GIT_USER_NAME ?= "OpenEmbedded" # returns local (absolute) path names for all valid patches in the # src_uri @@ -52,7 +54,9 @@ def get_machine_branch(d, default): parm = urldata.parm if "branch" in parm: branches = urldata.parm.get("branch").split(',') - return branches[0] + btype = urldata.parm.get("type") + if btype != "kmeta": + return branches[0] return default @@ -117,62 +121,76 @@ do_kernel_metadata() { patches="${@" ".join(find_patches(d))}" feat_dirs="${@" ".join(find_kernel_feature_dirs(d))}" - # add any explicitly referenced features onto the end of the feature - # list that is passed to the kernel build scripts. - if [ -n "${KERNEL_FEATURES}" ]; then - for feat in ${KERNEL_FEATURES}; do - addon_features="$addon_features --feature $feat" - done - fi - # check for feature directories/repos/branches that were part of the # SRC_URI. If they were supplied, we convert them into include directives # for the update part of the process - if [ -n "${feat_dirs}" ]; then - for f in ${feat_dirs}; do + for f in ${feat_dirs}; do if [ -d "${WORKDIR}/$f/meta" ]; then - includes="$includes -I${WORKDIR}/$f/meta" - elif [ -d "${WORKDIR}/$f" ]; then - includes="$includes -I${WORKDIR}/$f" + includes="$includes -I${WORKDIR}/$f/kernel-meta" + elif [ -d "${WORKDIR}/$f" ]; then + includes="$includes -I${WORKDIR}/$f" + fi + done + for s in ${sccs} ${patches}; do + sdir=$(dirname $s) + includes="$includes -I${sdir}" + # if a SRC_URI passed patch or .scc has a subdir of "kernel-meta", + # then we add it to the search path + if [ -d "${sdir}/kernel-meta" ]; then + includes="$includes -I${sdir}/kernel-meta" + fi + done + + # expand kernel features into their full path equivalents + bsp_definition=$(spp ${includes} --find -DKMACHINE=${KMACHINE} -DKTYPE=${LINUX_KERNEL_TYPE}) + meta_dir=$(kgit --meta) + + # run1: pull all the configuration fragments, no matter where they come from + elements="`echo -n ${bsp_definition} ${sccs} ${patches} ${KERNEL_FEATURES}`" + if [ -n "${elements}" ]; then + echo "${bsp_definition}" > ${S}/${meta_dir}/bsp_definition + scc --force -o ${S}/${meta_dir}:cfg,merge,meta ${includes} ${bsp_definition} ${sccs} ${patches} ${KERNEL_FEATURES} + if [ $? -ne 0 ]; then + bbfatal_log "Could not generate configuration queue for ${KMACHINE}." fi - done fi - # updates or generates the target description - updateme ${updateme_flags} -DKDESC=${KMACHINE}:${LINUX_KERNEL_TYPE} \ - ${includes} ${addon_features} ${ARCH} ${KMACHINE} ${sccs} ${patches} - if [ $? -ne 0 ]; then - bbfatal_log "Could not update ${machine_branch}" + # run2: only generate patches for elements that have been passed on the SRC_URI + elements="`echo -n ${sccs} ${patches} ${KERNEL_FEATURES}`" + if [ -n "${elements}" ]; then + scc --force -o ${S}/${meta_dir}:patch --cmds patch ${includes} ${sccs} ${patches} ${KERNEL_FEATURES} + if [ $? -ne 0 ]; then + bbfatal_log "Could not generate configuration queue for ${KMACHINE}." + fi fi } do_patch() { + set +e cd ${S} - # executes and modifies the source tree as required - patchme ${KMACHINE} - if [ $? -ne 0 ]; then - bberror "Could not apply patches for ${KMACHINE}." - bbfatal_log "Patch failures can be resolved in the linux source directory ${S})" - fi - - # check to see if the specified SRCREV is reachable from the final branch. - # if it wasn't something wrong has happened, and we should error. - machine_srcrev="${SRCREV_machine}" - if [ -z "${machine_srcrev}" ]; then - # fallback to SRCREV if a non machine_meta tree is being built - machine_srcrev="${SRCREV}" - # if SRCREV cannot be reached something is wrong. - if [ -z "${machine_srcrev}" ]; then - bbfatal "Neither SRCREV_machine or SRCREV was specified!" + check_git_config + meta_dir=$(kgit --meta) + (cd ${meta_dir}; ln -sf patch.queue series) + if [ -f "${meta_dir}/series" ]; then + kgit-s2q --gen -v --patches .kernel-meta/ + if [ $? -ne 0 ]; then + bberror "Could not apply patches for ${KMACHINE}." + bbfatal_log "Patch failures can be resolved in the linux source directory ${S})" fi fi - if [ "${machine_srcrev}" != "AUTOINC" ]; then - if ! [ "$(git rev-parse --verify ${machine_srcrev}~0)" = "$(git merge-base ${machine_srcrev} HEAD)" ]; then - bberror "SRCREV ${machine_srcrev} was specified, but is not reachable" - bbfatal "Check the BSP description for incorrect branch selection, or other errors." - fi + if [ -f "${meta_dir}/merge.queue" ]; then + # we need to merge all these branches + for b in $(cat ${meta_dir}/merge.queue); do + git show-ref --verify --quiet refs/heads/${b} + if [ $? -eq 0 ]; then + bbnote "Merging branch ${b}" + git merge -q --no-ff -m "Merge branch ${b}" ${b} + else + bbfatal "branch ${b} does not exist, cannot merge" + fi + done fi } @@ -182,11 +200,18 @@ do_kernel_checkout() { source_dir=`echo ${S} | sed 's%/$%%'` source_workdir="${WORKDIR}/git" if [ -d "${WORKDIR}/git/" ]; then - # case: git repository (bare or non-bare) + # case: git repository # if S is WORKDIR/git, then we shouldn't be moving or deleting the tree. if [ "${source_dir}" != "${source_workdir}" ]; then - rm -rf ${S} - mv ${WORKDIR}/git ${S} + if [ -d "${source_workdir}/.git" ]; then + # regular git repository with .git + rm -rf ${S} + mv ${WORKDIR}/git ${S} + else + # create source for bare cloned git repository + git clone ${WORKDIR}/git ${S} + rm -rf ${WORKDIR}/git + fi fi cd ${S} else @@ -228,31 +253,43 @@ do_kernel_checkout() { do_kernel_checkout[dirs] = "${S}" addtask kernel_checkout before do_kernel_metadata after do_unpack -addtask kernel_metadata after do_validate_branches before do_patch +addtask kernel_metadata after do_validate_branches do_unpack before do_patch do_kernel_metadata[depends] = "kern-tools-native:do_populate_sysroot" +do_validate_branches[depends] = "kern-tools-native:do_populate_sysroot" do_kernel_configme[dirs] += "${S} ${B}" do_kernel_configme() { - bbnote "kernel configme" - export KMETA=${KMETA} + set +e - if [ -n "${KCONFIG_MODE}" ]; then - configmeflags=${KCONFIG_MODE} - else - # If a defconfig was passed, use =n as the baseline, which is achieved - # via --allnoconfig + # translate the kconfig_mode into something that merge_config.sh + # understands + case ${KCONFIG_MODE} in + *allnoconfig) + config_flags="-n" + ;; + *alldefconfig) + config_flags="" + ;; + *) if [ -f ${WORKDIR}/defconfig ]; then - configmeflags="--allnoconfig" + config_flags="-n" fi - fi + ;; + esac cd ${S} - PATH=${PATH}:${S}/scripts/util - configme ${configmeflags} --reconfig --output ${B} ${LINUX_KERNEL_TYPE} ${KMACHINE} + + meta_dir=$(kgit --meta) + configs="$(scc --configs -o ${meta_dir})" + if [ -z "${configs}" ]; then + bbfatal_log "Could not find configuration queue (${meta_dir}/config.queue)" + fi + + CFLAGS="${CFLAGS} ${TOOLCHAIN_OPTIONS}" ARCH=${ARCH} merge_config.sh -O ${B} ${config_flags} ${configs} > ${meta_dir}/cfg/merge_config_build.log 2>&1 if [ $? -ne 0 ]; then bbfatal_log "Could not configure ${KMACHINE}-${LINUX_KERNEL_TYPE}" fi - + echo "# Global settings from linux recipe" >> ${B}/.config echo "CONFIG_LOCALVERSION="\"${LINUX_VERSION_EXTENSION}\" >> ${B}/.config } @@ -265,41 +302,28 @@ python do_kernel_configcheck() { # if KMETA isn't set globally by a recipe using this routine, we need to # set the default to 'meta'. Otherwise, kconf_check is not passed a valid # meta-series for processing - kmeta = d.getVar( "KMETA", True ) or "meta" + kmeta = d.getVar("KMETA") or "meta" if not os.path.exists(kmeta): kmeta = "." + kmeta - pathprefix = "export PATH=%s:%s; " % (d.getVar('PATH', True), "${S}/scripts/util/") - cmd = d.expand("cd ${S}; kconf_check -config %s/meta-series ${S} ${B}" % kmeta) + pathprefix = "export PATH=%s:%s; " % (d.getVar('PATH'), "${S}/scripts/util/") + + cmd = d.expand("scc --configs -o ${S}/.kernel-meta") + ret, configs = oe.utils.getstatusoutput("%s%s" % (pathprefix, cmd)) + + cmd = d.expand("cd ${S}; kconf_check --report -o ${S}/%s/cfg/ ${B}/.config ${S} %s" % (kmeta,configs)) ret, result = oe.utils.getstatusoutput("%s%s" % (pathprefix, cmd)) - config_check_visibility = int(d.getVar( "KCONF_AUDIT_LEVEL", True ) or 0) - bsp_check_visibility = int(d.getVar( "KCONF_BSP_AUDIT_LEVEL", True ) or 0) + config_check_visibility = int(d.getVar("KCONF_AUDIT_LEVEL") or 0) + bsp_check_visibility = int(d.getVar("KCONF_BSP_AUDIT_LEVEL") or 0) # if config check visibility is non-zero, report dropped configuration values - mismatch_file = "${S}/" + kmeta + "/" + "mismatch.cfg" + mismatch_file = d.expand("${S}/%s/cfg/mismatch.txt" % kmeta) if os.path.exists(mismatch_file): if config_check_visibility: with open (mismatch_file, "r") as myfile: results = myfile.read() bb.warn( "[kernel config]: specified values did not make it into the kernel's final configuration:\n\n%s" % results) - - # if config check visibility is level 2 or higher, report non-hardware options - nonhw_file = "${S}/" + kmeta + "/" + "nonhw_report.cfg" - if os.path.exists(nonhw_file): - if config_check_visibility > 1: - with open (nonhw_file, "r") as myfile: - results = myfile.read() - bb.warn( "[kernel config]: BSP specified non-hw configuration:\n\n%s" % results) - - bsp_desc = "${S}/" + kmeta + "/" + "top_tgt" - if os.path.exists(bsp_desc) and bsp_check_visibility > 1: - with open (bsp_desc, "r") as myfile: - bsp_tgt = myfile.read() - m = re.match("^(.*)scratch.obj(.*)$", bsp_tgt) - if not m is None: - bb.warn( "[kernel]: An auto generated BSP description was used, this normally indicates a misconfiguration.\n" + - "Check that your machine (%s) has an associated kernel description." % "${MACHINE}" ) } # Ensure that the branches (BSP and meta) are on the locations specified by @@ -342,22 +366,14 @@ do_validate_branches() { current_branch=`git rev-parse --abbrev-ref HEAD` git branch "$current_branch-orig" git reset --hard ${force_srcrev} + # We've checked out HEAD, make sure we cleanup kgit-s2q fence post check + # so the patches are applied as expected otherwise no patching + # would be done in some corner cases. + kgit-s2q --clean fi fi } -# Many scripts want to look in arch/$arch/boot for the bootable -# image. This poses a problem for vmlinux based booting. This -# task arranges to have vmlinux appear in the normalized directory -# location. -do_kernel_link_vmlinux() { - if [ ! -d "${B}/arch/${ARCH}/boot" ]; then - mkdir ${B}/arch/${ARCH}/boot - fi - cd ${B}/arch/${ARCH}/boot - ln -sf ../../../vmlinux -} - OE_TERMINAL_EXPORTS += "KBUILD_OUTPUT" KBUILD_OUTPUT = "${B}" diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass index 919293e43d..8954b28b2c 100644 --- a/meta/classes/kernel.bbclass +++ b/meta/classes/kernel.bbclass @@ -1,7 +1,10 @@ inherit linux-kernel-base kernel-module-split PROVIDES += "virtual/kernel" -DEPENDS += "virtual/${TARGET_PREFIX}binutils virtual/${TARGET_PREFIX}gcc kmod-native depmodwrapper-cross bc-native" +DEPENDS += "virtual/${TARGET_PREFIX}binutils virtual/${TARGET_PREFIX}gcc kmod-native bc-native lzop-native" +PACKAGE_WRITE_DEPS += "depmodwrapper-cross virtual/update-alternatives-native" + +do_deploy[depends] += "depmodwrapper-cross:do_populate_sysroot" S = "${STAGING_KERNEL_DIR}" B = "${WORKDIR}/build" @@ -13,25 +16,67 @@ INHIBIT_DEFAULT_DEPS = "1" KERNEL_IMAGETYPE ?= "zImage" INITRAMFS_IMAGE ?= "" +INITRAMFS_IMAGE_NAME ?= "${@['${INITRAMFS_IMAGE}-${MACHINE}', ''][d.getVar('INITRAMFS_IMAGE') == '']}" INITRAMFS_TASK ?= "" INITRAMFS_IMAGE_BUNDLE ?= "" +# KERNEL_VERSION is extracted from source code. It is evaluated as +# None for the first parsing, since the code has not been fetched. +# After the code is fetched, it will be evaluated as real version +# number and cause kernel to be rebuilt. To avoid this, make +# KERNEL_VERSION_NAME and KERNEL_VERSION_PKG_NAME depend on +# LINUX_VERSION which is a constant. +KERNEL_VERSION_NAME = "${@d.getVar('KERNEL_VERSION') or ""}" +KERNEL_VERSION_NAME[vardepvalue] = "${LINUX_VERSION}" +KERNEL_VERSION_PKG_NAME = "${@legitimize_package_name(d.getVar('KERNEL_VERSION'))}" +KERNEL_VERSION_PKG_NAME[vardepvalue] = "${LINUX_VERSION}" + python __anonymous () { import re - kerneltype = d.getVar('KERNEL_IMAGETYPE', True) + # Merge KERNEL_IMAGETYPE and KERNEL_ALT_IMAGETYPE into KERNEL_IMAGETYPES + type = d.getVar('KERNEL_IMAGETYPE') or "" + alttype = d.getVar('KERNEL_ALT_IMAGETYPE') or "" + types = d.getVar('KERNEL_IMAGETYPES') or "" + if type not in types.split(): + types = (type + ' ' + types).strip() + if alttype not in types.split(): + types = (alttype + ' ' + types).strip() + d.setVar('KERNEL_IMAGETYPES', types) + + typeformake = re.sub(r'\.gz', '', types) + d.setVar('KERNEL_IMAGETYPE_FOR_MAKE', typeformake) + + for type in types.split(): + typelower = type.lower() + imagedest = d.getVar('KERNEL_IMAGEDEST') + + d.appendVar('PACKAGES', ' ' + 'kernel-image-' + typelower) + + d.setVar('FILES_kernel-image-' + typelower, '/' + imagedest + '/' + type + '-${KERNEL_VERSION_NAME}') + + d.appendVar('RDEPENDS_kernel-image', ' ' + 'kernel-image-' + typelower) - d.setVar("KERNEL_IMAGETYPE_FOR_MAKE", re.sub(r'\.gz$', '', kerneltype)) + d.setVar('PKG_kernel-image-' + typelower, 'kernel-image-' + typelower + '-${KERNEL_VERSION_PKG_NAME}') - image = d.getVar('INITRAMFS_IMAGE', True) + d.setVar('ALLOW_EMPTY_kernel-image-' + typelower, '1') + + priority = d.getVar('KERNEL_PRIORITY') + postinst = '#!/bin/sh\n' + 'update-alternatives --install /' + imagedest + '/' + type + ' ' + type + ' ' + type + '-${KERNEL_VERSION_NAME} ' + priority + ' || true' + '\n' + d.setVar('pkg_postinst_kernel-image-' + typelower, postinst) + + postrm = '#!/bin/sh\n' + 'update-alternatives --remove' + ' ' + type + ' ' + type + '-${KERNEL_VERSION_NAME} || true' + '\n' + d.setVar('pkg_postrm_kernel-image-' + typelower, postrm) + + image = d.getVar('INITRAMFS_IMAGE') if image: - d.appendVarFlag('do_bundle_initramfs', 'depends', ' ${INITRAMFS_IMAGE}:do_rootfs') + d.appendVarFlag('do_bundle_initramfs', 'depends', ' ${INITRAMFS_IMAGE}:do_image_complete') # NOTE: setting INITRAMFS_TASK is for backward compatibility # The preferred method is to set INITRAMFS_IMAGE, because # this INITRAMFS_TASK has circular dependency problems # if the initramfs requires kernel modules - image_task = d.getVar('INITRAMFS_TASK', True) + image_task = d.getVar('INITRAMFS_TASK') if image_task: d.appendVarFlag('do_configure', 'depends', ' ${INITRAMFS_TASK}') } @@ -60,17 +105,21 @@ inherit ${KERNEL_CLASSES} do_unpack[cleandirs] += " ${S} ${STAGING_KERNEL_DIR} ${B} ${STAGING_KERNEL_BUILDDIR}" do_clean[cleandirs] += " ${S} ${STAGING_KERNEL_DIR} ${B} ${STAGING_KERNEL_BUILDDIR}" base_do_unpack_append () { - s = d.getVar("S", True) + s = d.getVar("S") if s[-1] == '/': # drop trailing slash, so that os.symlink(kernsrc, s) doesn't use s as directory name and fail s=s[:-1] - kernsrc = d.getVar("STAGING_KERNEL_DIR", True) + kernsrc = d.getVar("STAGING_KERNEL_DIR") if s != kernsrc: bb.utils.mkdirhier(kernsrc) bb.utils.remove(kernsrc, recurse=True) - import subprocess - subprocess.call(d.expand("mv ${S} ${STAGING_KERNEL_DIR}"), shell=True) - os.symlink(kernsrc, s) + if d.getVar("EXTERNALSRC"): + # With EXTERNALSRC S will not be wiped so we can symlink to it + os.symlink(s, kernsrc) + else: + import shutil + shutil.move(s, kernsrc) + os.symlink(kernsrc, s) } inherit kernel-arch deploy @@ -82,20 +131,20 @@ PACKAGES_DYNAMIC += "^kernel-firmware-.*" export OS = "${TARGET_OS}" export CROSS_COMPILE = "${TARGET_PREFIX}" -KERNEL_PRIORITY ?= "${@int(d.getVar('PV',1).split('-')[0].split('+')[0].split('.')[0]) * 10000 + \ - int(d.getVar('PV',1).split('-')[0].split('+')[0].split('.')[1]) * 100 + \ - int(d.getVar('PV',1).split('-')[0].split('+')[0].split('.')[-1])}" +KERNEL_PRIORITY ?= "${@int(d.getVar('PV').split('-')[0].split('+')[0].split('.')[0]) * 10000 + \ + int(d.getVar('PV').split('-')[0].split('+')[0].split('.')[1]) * 100 + \ + int(d.getVar('PV').split('-')[0].split('+')[0].split('.')[-1])}" KERNEL_RELEASE ?= "${KERNEL_VERSION}" -# Where built kernel lies in the kernel tree -KERNEL_OUTPUT ?= "arch/${ARCH}/boot/${KERNEL_IMAGETYPE}" +# The directory where built kernel lies in the kernel tree +KERNEL_OUTPUT_DIR ?= "arch/${ARCH}/boot" KERNEL_IMAGEDEST = "boot" # # configuration # -export CMDLINE_CONSOLE = "console=${@d.getVar("KERNEL_CONSOLE",1) or "ttyS0"}" +export CMDLINE_CONSOLE = "console=${@d.getVar("KERNEL_CONSOLE") or "ttyS0"}" KERNEL_VERSION = "${@get_kernelversion_headers('${B}')}" @@ -111,49 +160,42 @@ UBOOT_LOADADDRESS ?= "${UBOOT_ENTRYPOINT}" # Some Linux kernel configurations need additional parameters on the command line KERNEL_EXTRA_ARGS ?= "" -# For the kernel, we don't want the '-e MAKEFLAGS=' in EXTRA_OEMAKE. -# We don't want to override kernel Makefile variables from the environment -EXTRA_OEMAKE = "" - +EXTRA_OEMAKE = " HOSTCC="${BUILD_CC}" HOSTCPP="${BUILD_CPP}"" KERNEL_ALT_IMAGETYPE ??= "" -# Define where the kernel headers are installed on the target as well as where -# they are staged. -KERNEL_SRC_PATH = "/usr/src/kernel" - copy_initramfs() { echo "Copying initramfs into ./usr ..." # In case the directory is not created yet from the first pass compile: mkdir -p ${B}/usr # Find and use the first initramfs image archive type we find - rm -f ${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.cpio - for img in cpio.gz cpio.lz4 cpio.lzo cpio.lzma cpio.xz; do - if [ -e "${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE}-${MACHINE}.$img" ]; then - cp ${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE}-${MACHINE}.$img ${B}/usr/. + rm -f ${B}/usr/${INITRAMFS_IMAGE_NAME}.cpio + for img in cpio cpio.gz cpio.lz4 cpio.lzo cpio.lzma cpio.xz; do + if [ -e "${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE_NAME}.$img" ]; then + cp ${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE_NAME}.$img ${B}/usr/. case $img in *gz) echo "gzip decompressing image" - gunzip -f ${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.$img + gunzip -f ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img break ;; *lz4) echo "lz4 decompressing image" - lz4 -df ${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.$img + lz4 -df ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img break ;; *lzo) echo "lzo decompressing image" - lzop -df ${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.$img + lzop -df ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img break ;; *lzma) echo "lzma decompressing image" - lzma -df ${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.$img + lzma -df ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img break ;; *xz) echo "xz decompressing image" - xz -df ${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.$img + xz -df ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img break ;; esac @@ -162,25 +204,44 @@ copy_initramfs() { echo "Finished copy of initramfs into ./usr" } -INITRAMFS_BASE_NAME = "${KERNEL_IMAGETYPE}-initramfs-${PV}-${PR}-${MACHINE}-${DATETIME}" +INITRAMFS_BASE_NAME ?= "initramfs-${PV}-${PR}-${MACHINE}-${DATETIME}" INITRAMFS_BASE_NAME[vardepsexclude] = "DATETIME" do_bundle_initramfs () { if [ ! -z "${INITRAMFS_IMAGE}" -a x"${INITRAMFS_IMAGE_BUNDLE}" = x1 ]; then echo "Creating a kernel image with a bundled initramfs..." copy_initramfs - if [ -e ${KERNEL_OUTPUT} ] ; then - mv -f ${KERNEL_OUTPUT} ${KERNEL_OUTPUT}.bak - fi - use_alternate_initrd=CONFIG_INITRAMFS_SOURCE=${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.cpio + # Backing up kernel image relies on its type(regular file or symbolic link) + tmp_path="" + for type in ${KERNEL_IMAGETYPES} ; do + if [ -h ${KERNEL_OUTPUT_DIR}/$type ] ; then + linkpath=`readlink -n ${KERNEL_OUTPUT_DIR}/$type` + realpath=`readlink -fn ${KERNEL_OUTPUT_DIR}/$type` + mv -f $realpath $realpath.bak + tmp_path=$tmp_path" "$type"#"$linkpath"#"$realpath + elif [ -f ${KERNEL_OUTPUT_DIR}/$type ]; then + mv -f ${KERNEL_OUTPUT_DIR}/$type ${KERNEL_OUTPUT_DIR}/$type.bak + tmp_path=$tmp_path" "$type"##" + fi + done + use_alternate_initrd=CONFIG_INITRAMFS_SOURCE=${B}/usr/${INITRAMFS_IMAGE_NAME}.cpio kernel_do_compile - mv -f ${KERNEL_OUTPUT} ${KERNEL_OUTPUT}.initramfs - mv -f ${KERNEL_OUTPUT}.bak ${KERNEL_OUTPUT} - # Update install area - echo "There is kernel image bundled with initramfs: ${B}/${KERNEL_OUTPUT}.initramfs" - install -m 0644 ${B}/${KERNEL_OUTPUT}.initramfs ${D}/boot/${KERNEL_IMAGETYPE}-initramfs-${MACHINE}.bin - echo "${B}/${KERNEL_OUTPUT}.initramfs" + # Restoring kernel image + for tp in $tmp_path ; do + type=`echo $tp|cut -d "#" -f 1` + linkpath=`echo $tp|cut -d "#" -f 2` + realpath=`echo $tp|cut -d "#" -f 3` + if [ -n "$realpath" ]; then + mv -f $realpath $realpath.initramfs + mv -f $realpath.bak $realpath + ln -sf $linkpath.initramfs ${B}/${KERNEL_OUTPUT_DIR}/$type.initramfs + else + mv -f ${KERNEL_OUTPUT_DIR}/$type ${KERNEL_OUTPUT_DIR}/$type.initramfs + mv -f ${KERNEL_OUTPUT_DIR}/$type.bak ${KERNEL_OUTPUT_DIR}/$type + fi + done fi } +do_bundle_initramfs[dirs] = "${B}" python do_devshell_prepend () { os.environ["LDFLAGS"] = '' @@ -201,18 +262,32 @@ kernel_do_compile() { # The old style way of copying an prebuilt image and building it # is turned on via INTIRAMFS_TASK != "" copy_initramfs - use_alternate_initrd=CONFIG_INITRAMFS_SOURCE=${B}/usr/${INITRAMFS_IMAGE}-${MACHINE}.cpio - fi - oe_runmake ${KERNEL_IMAGETYPE_FOR_MAKE} ${KERNEL_ALT_IMAGETYPE} CC="${KERNEL_CC}" LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS} $use_alternate_initrd - if test "${KERNEL_IMAGETYPE_FOR_MAKE}.gz" = "${KERNEL_IMAGETYPE}"; then - gzip -9c < "${KERNEL_IMAGETYPE_FOR_MAKE}" > "${KERNEL_OUTPUT}" + use_alternate_initrd=CONFIG_INITRAMFS_SOURCE=${B}/usr/${INITRAMFS_IMAGE_NAME}.cpio fi + for typeformake in ${KERNEL_IMAGETYPE_FOR_MAKE} ; do + oe_runmake ${typeformake} CC="${KERNEL_CC}" LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS} $use_alternate_initrd + for type in ${KERNEL_IMAGETYPES} ; do + if test "${typeformake}.gz" = "${type}"; then + mkdir -p "${KERNEL_OUTPUT_DIR}" + gzip -9c < "${typeformake}" > "${KERNEL_OUTPUT_DIR}/${type}" + break; + fi + done + done } do_compile_kernelmodules() { unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS MACHINE if (grep -q -i -e '^CONFIG_MODULES=y$' ${B}/.config); then oe_runmake -C ${B} ${PARALLEL_MAKE} modules CC="${KERNEL_CC}" LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS} + + # Module.symvers gets updated during the + # building of the kernel modules. We need to + # update this in the shared workdir since some + # external kernel modules has a dependency on + # other kernel modules and will look at this + # file to do symbol lookups + cp ${B}/Module.symvers ${STAGING_KERNEL_BUILDDIR}/ else bbnote "no modules to compile" fi @@ -225,11 +300,11 @@ kernel_do_install() { # unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS MACHINE if (grep -q -i -e '^CONFIG_MODULES=y$' .config); then - oe_runmake DEPMOD=echo INSTALL_MOD_PATH="${D}" modules_install - rm "${D}/lib/modules/${KERNEL_VERSION}/build" - rm "${D}/lib/modules/${KERNEL_VERSION}/source" + oe_runmake DEPMOD=echo MODLIB=${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION} INSTALL_FW_PATH=${D}${nonarch_base_libdir}/firmware modules_install + rm "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/build" + rm "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/source" # If the kernel/ directory is empty remove it to prevent QA issues - rmdir --ignore-fail-on-non-empty "${D}/lib/modules/${KERNEL_VERSION}/kernel" + rmdir --ignore-fail-on-non-empty "${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}/kernel" else bbnote "no modules to install" fi @@ -239,7 +314,9 @@ kernel_do_install() { # install -d ${D}/${KERNEL_IMAGEDEST} install -d ${D}/boot - install -m 0644 ${KERNEL_OUTPUT} ${D}/${KERNEL_IMAGEDEST}/${KERNEL_IMAGETYPE}-${KERNEL_VERSION} + for type in ${KERNEL_IMAGETYPES} ; do + install -m 0644 ${KERNEL_OUTPUT_DIR}/${type} ${D}/${KERNEL_IMAGEDEST}/${type}-${KERNEL_VERSION} + done install -m 0644 System.map ${D}/boot/System.map-${KERNEL_VERSION} install -m 0644 .config ${D}/boot/config-${KERNEL_VERSION} install -m 0644 vmlinux ${D}/boot/vmlinux-${KERNEL_VERSION} @@ -249,6 +326,40 @@ kernel_do_install() { } do_install[prefuncs] += "package_get_auto_pr" +# Must be ran no earlier than after do_kernel_checkout or else Makefile won't be in ${S}/Makefile +do_kernel_version_sanity_check() { + if [ "x${KERNEL_VERSION_SANITY_SKIP}" = "x1" ]; then + exit 0 + fi + + # The Makefile determines the kernel version shown at runtime + # Don't use KERNEL_VERSION because the headers it grabs the version from aren't generated until do_compile + VERSION=$(grep "^VERSION =" ${S}/Makefile | sed s/.*=\ *//) + PATCHLEVEL=$(grep "^PATCHLEVEL =" ${S}/Makefile | sed s/.*=\ *//) + SUBLEVEL=$(grep "^SUBLEVEL =" ${S}/Makefile | sed s/.*=\ *//) + EXTRAVERSION=$(grep "^EXTRAVERSION =" ${S}/Makefile | sed s/.*=\ *//) + + # Build a string for regex and a plain version string + reg="^${VERSION}\.${PATCHLEVEL}" + vers="${VERSION}.${PATCHLEVEL}" + if [ -n "${SUBLEVEL}" ]; then + # Ignoring a SUBLEVEL of zero is fine + if [ "${SUBLEVEL}" = "0" ]; then + reg="${reg}(\.${SUBLEVEL})?" + else + reg="${reg}\.${SUBLEVEL}" + vers="${vers}.${SUBLEVEL}" + fi + fi + vers="${vers}${EXTRAVERSION}" + reg="${reg}${EXTRAVERSION}" + + if [ -z `echo ${PV} | grep -E "${reg}"` ]; then + bbfatal "Package Version (${PV}) does not match of kernel being built (${vers}). Please update the PV variable to match the kernel source or set KERNEL_VERSION_SANITY_SKIP=\"1\" in your recipe." + fi + exit 0 +} + addtask shared_workdir after do_compile before do_compile_kernelmodules addtask shared_workdir_setscene @@ -283,6 +394,14 @@ do_shared_workdir () { cp .config $kerneldir/ mkdir -p $kerneldir/include/config cp include/config/kernel.release $kerneldir/include/config/kernel.release + if [ -e certs/signing_key.pem ]; then + # The signing_key.* files are stored in the certs/ dir in + # newer Linux kernels + mkdir -p $kerneldir/certs + cp certs/signing_key.* $kerneldir/certs/ + elif [ -e signing_key.priv ]; then + cp signing_key.* $kerneldir/ + fi # We can also copy over all the generated files and avoid special cases # like version.h, but we've opted to keep this small until file creep starts @@ -318,6 +437,20 @@ sysroot_stage_all () { KERNEL_CONFIG_COMMAND ?= "oe_runmake_call -C ${S} O=${B} oldnoconfig || yes '' | oe_runmake -C ${S} O=${B} oldconfig" +python check_oldest_kernel() { + oldest_kernel = d.getVar('OLDEST_KERNEL') + kernel_version = d.getVar('KERNEL_VERSION') + tclibc = d.getVar('TCLIBC') + if tclibc == 'glibc': + kernel_version = kernel_version.split('-', 1)[0] + if oldest_kernel and kernel_version: + if bb.utils.vercmp_string(kernel_version, oldest_kernel) < 0: + bb.warn('%s: OLDEST_KERNEL is "%s" but the version of the kernel you are building is "%s" - therefore %s as built may not be compatible with this kernel. Either set OLDEST_KERNEL to an older version, or build a newer kernel.' % (d.getVar('PN'), oldest_kernel, kernel_version, tclibc)) +} + +check_oldest_kernel[vardepsexclude] += "OLDEST_KERNEL KERNEL_VERSION" +do_configure[prefuncs] += "check_oldest_kernel" + kernel_do_configure() { # fixes extra + in /lib/modules/2.6.37+ # $ scripts/setlocalversion . => + @@ -334,10 +467,12 @@ kernel_do_configure() { if [ -f "${WORKDIR}/defconfig" ] && [ ! -f "${B}/.config" ]; then cp "${WORKDIR}/defconfig" "${B}/.config" fi - eval ${KERNEL_CONFIG_COMMAND} + + ${KERNEL_CONFIG_COMMAND} } do_savedefconfig() { + bbplain "Saving defconfig to:\n${B}/defconfig" oe_runmake -C ${B} savedefconfig } do_savedefconfig[nostamp] = "1" @@ -348,16 +483,16 @@ inherit cml1 EXPORT_FUNCTIONS do_compile do_install do_configure # kernel-base becomes kernel-${KERNEL_VERSION} -# kernel-image becomes kernel-image-${KERNEL_VERISON} +# kernel-image becomes kernel-image-${KERNEL_VERSION} PACKAGES = "kernel kernel-base kernel-vmlinux kernel-image kernel-dev kernel-modules" FILES_${PN} = "" -FILES_kernel-base = "/lib/modules/${KERNEL_VERSION}/modules.order /lib/modules/${KERNEL_VERSION}/modules.builtin" -FILES_kernel-image = "/boot/${KERNEL_IMAGETYPE}*" -FILES_kernel-dev = "/boot/System.map* /boot/Module.symvers* /boot/config* ${KERNEL_SRC_PATH} /lib/modules/${KERNEL_VERSION}/build" -FILES_kernel-vmlinux = "/boot/vmlinux*" +FILES_kernel-base = "${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.order ${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.builtin" +FILES_kernel-image = "" +FILES_kernel-dev = "/boot/System.map* /boot/Module.symvers* /boot/config* ${KERNEL_SRC_PATH} ${nonarch_base_libdir}/modules/${KERNEL_VERSION}/build" +FILES_kernel-vmlinux = "/boot/vmlinux-${KERNEL_VERSION_NAME}" FILES_kernel-modules = "" RDEPENDS_kernel = "kernel-base" -# Allow machines to override this dependency if kernel image files are +# Allow machines to override this dependency if kernel image files are # not wanted in images as standard RDEPENDS_kernel-base ?= "kernel-image" PKG_kernel-image = "kernel-image-${@legitimize_package_name('${KERNEL_VERSION}')}" @@ -381,39 +516,50 @@ pkg_postinst_kernel-base () { fi } -pkg_postinst_kernel-image () { - update-alternatives --install /${KERNEL_IMAGEDEST}/${KERNEL_IMAGETYPE} ${KERNEL_IMAGETYPE} /${KERNEL_IMAGEDEST}/${KERNEL_IMAGETYPE}-${KERNEL_VERSION} ${KERNEL_PRIORITY} || true -} - -pkg_postrm_kernel-image () { - update-alternatives --remove ${KERNEL_IMAGETYPE} ${KERNEL_IMAGETYPE}-${KERNEL_VERSION} || true -} - PACKAGESPLITFUNCS_prepend = "split_kernel_packages " python split_kernel_packages () { - do_split_packages(d, root='/lib/firmware', file_regex='^(.*)\.(bin|fw|cis|dsp)$', output_pattern='kernel-firmware-%s', description='Firmware for %s', recursive=True, extra_depends='') + do_split_packages(d, root='${nonarch_base_libdir}/firmware', file_regex='^(.*)\.(bin|fw|cis|csp|dsp)$', output_pattern='kernel-firmware-%s', description='Firmware for %s', recursive=True, extra_depends='') } +# Many scripts want to look in arch/$arch/boot for the bootable +# image. This poses a problem for vmlinux and vmlinuz based +# booting. This task arranges to have vmlinux and vmlinuz appear +# in the normalized directory location. +do_kernel_link_images() { + if [ ! -d "${B}/arch/${ARCH}/boot" ]; then + mkdir ${B}/arch/${ARCH}/boot + fi + cd ${B}/arch/${ARCH}/boot + ln -sf ../../../vmlinux + if [ -f ../../../vmlinuz ]; then + ln -sf ../../../vmlinuz + fi + if [ -f ../../../vmlinuz.bin ]; then + ln -sf ../../../vmlinuz.bin + fi +} +addtask kernel_link_images after do_compile before do_strip + do_strip() { if [ -n "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}" ]; then - if [ "${KERNEL_IMAGETYPE}" != "vmlinux" ]; then - bbwarn "image type will not be stripped (not supported): ${KERNEL_IMAGETYPE}" + if ! (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux"); then + bbwarn "image type(s) will not be stripped (not supported): ${KERNEL_IMAGETYPES}" return fi cd ${B} - headers=`"$CROSS_COMPILE"readelf -S ${KERNEL_OUTPUT} | \ + headers=`"$CROSS_COMPILE"readelf -S ${KERNEL_OUTPUT_DIR}/vmlinux | \ grep "^ \{1,\}\[[0-9 ]\{1,\}\] [^ ]" | \ sed "s/^ \{1,\}\[[0-9 ]\{1,\}\] //" | \ gawk '{print $1}'` for str in ${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}; do { - if [ "$headers" != *"$str"* ]; then + if ! (echo "$headers" | grep -q "^$str$"); then bbwarn "Section not found: $str"; fi - "$CROSS_COMPILE"strip -s -R $str ${KERNEL_OUTPUT} + "$CROSS_COMPILE"strip -s -R $str ${KERNEL_OUTPUT_DIR}/vmlinux }; done bbnote "KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping sections:" \ @@ -422,7 +568,7 @@ do_strip() { } do_strip[dirs] = "${B}" -addtask do_strip before do_sizecheck after do_kernel_link_vmlinux +addtask strip before do_sizecheck after do_kernel_link_images # Support checking the kernel size since some kernels need to reside in partitions # with a fixed length or there is a limit in transferring the kernel to memory @@ -432,20 +578,22 @@ do_sizecheck() { if [ -n "$invalid" ]; then die "Invalid KERNEL_IMAGE_MAXSIZE: ${KERNEL_IMAGE_MAXSIZE}, should be an integerx (The unit is Kbytes)" fi - size=`du -ks ${B}/${KERNEL_OUTPUT} | awk '{ print $1}'` - if [ $size -ge ${KERNEL_IMAGE_MAXSIZE} ]; then - die "This kernel (size=$size(K) > ${KERNEL_IMAGE_MAXSIZE}(K)) is too big for your device. Please reduce the size of the kernel by making more of it modular." - fi + for type in ${KERNEL_IMAGETYPES} ; do + size=`du -ks ${B}/${KERNEL_OUTPUT_DIR}/$type | awk '{print $1}'` + if [ $size -ge ${KERNEL_IMAGE_MAXSIZE} ]; then + warn "This kernel $type (size=$size(K) > ${KERNEL_IMAGE_MAXSIZE}(K)) is too big for your device. Please reduce the size of the kernel by making more of it modular." + fi + done fi } do_sizecheck[dirs] = "${B}" addtask sizecheck before do_install after do_strip -KERNEL_IMAGE_BASE_NAME ?= "${KERNEL_IMAGETYPE}-${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}" +KERNEL_IMAGE_BASE_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}" # Don't include the DATETIME variable in the sstate package signatures KERNEL_IMAGE_BASE_NAME[vardepsexclude] = "DATETIME" -KERNEL_IMAGE_SYMLINK_NAME ?= "${KERNEL_IMAGETYPE}-${MACHINE}" +KERNEL_IMAGE_SYMLINK_NAME ?= "${MACHINE}" MODULE_IMAGE_BASE_NAME ?= "modules-${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}" MODULE_IMAGE_BASE_NAME[vardepsexclude] = "DATETIME" MODULE_TARBALL_BASE_NAME ?= "${MODULE_IMAGE_BASE_NAME}.tgz" @@ -454,33 +602,39 @@ MODULE_TARBALL_SYMLINK_NAME ?= "modules-${MACHINE}.tgz" MODULE_TARBALL_DEPLOY ?= "1" kernel_do_deploy() { - install -m 0644 ${KERNEL_OUTPUT} ${DEPLOYDIR}/${KERNEL_IMAGE_BASE_NAME}.bin + for type in ${KERNEL_IMAGETYPES} ; do + base_name=${type}-${KERNEL_IMAGE_BASE_NAME} + install -m 0644 ${KERNEL_OUTPUT_DIR}/${type} ${DEPLOYDIR}/${base_name}.bin + done if [ ${MODULE_TARBALL_DEPLOY} = "1" ] && (grep -q -i -e '^CONFIG_MODULES=y$' .config); then mkdir -p ${D}/lib tar -cvzf ${DEPLOYDIR}/${MODULE_TARBALL_BASE_NAME} -C ${D} lib ln -sf ${MODULE_TARBALL_BASE_NAME} ${DEPLOYDIR}/${MODULE_TARBALL_SYMLINK_NAME} fi - ln -sf ${KERNEL_IMAGE_BASE_NAME}.bin ${DEPLOYDIR}/${KERNEL_IMAGE_SYMLINK_NAME}.bin - ln -sf ${KERNEL_IMAGE_BASE_NAME}.bin ${DEPLOYDIR}/${KERNEL_IMAGETYPE} - - cp ${COREBASE}/meta/files/deploydir_readme.txt ${DEPLOYDIR}/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt + for type in ${KERNEL_IMAGETYPES} ; do + base_name=${type}-${KERNEL_IMAGE_BASE_NAME} + symlink_name=${type}-${KERNEL_IMAGE_SYMLINK_NAME} + ln -sf ${base_name}.bin ${DEPLOYDIR}/${symlink_name}.bin + ln -sf ${base_name}.bin ${DEPLOYDIR}/${type} + done cd ${B} # Update deploy directory - if [ -e "${KERNEL_OUTPUT}.initramfs" ]; then - echo "Copying deploy kernel-initramfs image and setting up links..." - initramfs_base_name=${INITRAMFS_BASE_NAME} - initramfs_symlink_name=${KERNEL_IMAGETYPE}-initramfs-${MACHINE} - install -m 0644 ${KERNEL_OUTPUT}.initramfs ${DEPLOYDIR}/${initramfs_base_name}.bin - cd ${DEPLOYDIR} - ln -sf ${initramfs_base_name}.bin ${initramfs_symlink_name}.bin - fi + for type in ${KERNEL_IMAGETYPES} ; do + if [ -e "${KERNEL_OUTPUT_DIR}/${type}.initramfs" ]; then + echo "Copying deploy ${type} kernel-initramfs image and setting up links..." + initramfs_base_name=${type}-${INITRAMFS_BASE_NAME} + initramfs_symlink_name=${type}-initramfs-${MACHINE} + install -m 0644 ${KERNEL_OUTPUT_DIR}/${type}.initramfs ${DEPLOYDIR}/${initramfs_base_name}.bin + ln -sf ${initramfs_base_name}.bin ${DEPLOYDIR}/${initramfs_symlink_name}.bin + fi + done } +do_deploy[cleandirs] = "${DEPLOYDIR}" do_deploy[dirs] = "${DEPLOYDIR} ${B}" do_deploy[prefuncs] += "package_get_auto_pr" -addtask deploy after do_populate_sysroot +addtask deploy after do_populate_sysroot do_packagedata EXPORT_FUNCTIONS do_deploy - diff --git a/meta/classes/kernelsrc.bbclass b/meta/classes/kernelsrc.bbclass index 9efd46a92d..675d40ec9a 100644 --- a/meta/classes/kernelsrc.bbclass +++ b/meta/classes/kernelsrc.bbclass @@ -1,7 +1,7 @@ S = "${STAGING_KERNEL_DIR}" -do_fetch[noexec] = "1" -do_unpack[depends] += "virtual/kernel:do_patch" -do_unpack[noexec] = "1" +deltask do_fetch +deltask do_unpack +do_patch[depends] += "virtual/kernel:do_patch" do_patch[noexec] = "1" do_package[depends] += "virtual/kernel:do_populate_sysroot" KERNEL_VERSION = "${@get_kernelversion_file("${STAGING_KERNEL_BUILDDIR}")}" diff --git a/meta/classes/libc-common.bbclass b/meta/classes/libc-common.bbclass index bbc80167dd..9ea2c03749 100644 --- a/meta/classes/libc-common.bbclass +++ b/meta/classes/libc-common.bbclass @@ -4,14 +4,12 @@ do_install() { h=`echo $r|sed -e's,\.x$,.h,'` install -m 0644 ${S}/sunrpc/rpcsvc/$h ${D}/${includedir}/rpcsvc/ done - install -d ${D}/${sysconfdir}/ - install -m 0644 ${WORKDIR}/etc/ld.so.conf ${D}/${sysconfdir}/ + install -Dm 0644 ${WORKDIR}/etc/ld.so.conf ${D}/${sysconfdir}/ld.so.conf install -d ${D}${localedir} make -f ${WORKDIR}/generate-supported.mk IN="${S}/localedata/SUPPORTED" OUT="${WORKDIR}/SUPPORTED" # get rid of some broken files... for i in ${GLIBC_BROKEN_LOCALES}; do - grep -v $i ${WORKDIR}/SUPPORTED > ${WORKDIR}/SUPPORTED.tmp - mv ${WORKDIR}/SUPPORTED.tmp ${WORKDIR}/SUPPORTED + sed -i "/$i/d" ${WORKDIR}/SUPPORTED done rm -f ${D}${sysconfdir}/rpc rm -rf ${D}${datadir}/zoneinfo @@ -19,15 +17,15 @@ do_install() { } def get_libc_fpu_setting(bb, d): - if d.getVar('TARGET_FPU', True) in [ 'soft', 'ppc-efd' ]: + if d.getVar('TARGET_FPU') in [ 'soft', 'ppc-efd' ]: return "--without-fp" return "" python populate_packages_prepend () { - if d.getVar('DEBIAN_NAMES', True): - pkgs = d.getVar('PACKAGES', True).split() - bpn = d.getVar('BPN', True) - prefix = d.getVar('MLPREFIX', True) or "" + if d.getVar('DEBIAN_NAMES'): + pkgs = d.getVar('PACKAGES').split() + bpn = d.getVar('BPN') + prefix = d.getVar('MLPREFIX') or "" # Set the base package... d.setVar('PKG_' + prefix + bpn, prefix + 'libc6') libcprefix = prefix + bpn + '-' diff --git a/meta/classes/libc-package.bbclass b/meta/classes/libc-package.bbclass index 47be691e2d..739adce694 100644 --- a/meta/classes/libc-package.bbclass +++ b/meta/classes/libc-package.bbclass @@ -9,25 +9,27 @@ GLIBC_INTERNAL_USE_BINARY_LOCALE ?= "ondevice" +GLIBC_SPLIT_LC_PACKAGES ?= "0" + python __anonymous () { - enabled = d.getVar("ENABLE_BINARY_LOCALE_GENERATION", True) + enabled = d.getVar("ENABLE_BINARY_LOCALE_GENERATION") - pn = d.getVar("PN", True) + pn = d.getVar("PN") if pn.endswith("-initial"): enabled = False if enabled and int(enabled): import re - target_arch = d.getVar("TARGET_ARCH", True) - binary_arches = d.getVar("BINARY_LOCALE_ARCHES", True) or "" - use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF", True) or "" + target_arch = d.getVar("TARGET_ARCH") + binary_arches = d.getVar("BINARY_LOCALE_ARCHES") or "" + use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF") or "" for regexp in binary_arches.split(" "): r = re.compile(regexp) if r.match(target_arch): - depends = d.getVar("DEPENDS", True) + depends = d.getVar("DEPENDS") if use_cross_localedef == "1" : depends = "%s cross-localedef-native" % depends else: @@ -47,15 +49,6 @@ python __anonymous () { OVERRIDES_append = ":${TARGET_ARCH}-${TARGET_OS}" -do_configure_prepend() { - if [ -e ${S}/elf/ldd.bash.in ]; then - sed -e "s#@BASH@#/bin/sh#" -i ${S}/elf/ldd.bash.in - fi -} - - - -# indentation removed on purpose locale_base_postinst() { #!/bin/sh @@ -63,33 +56,14 @@ if [ "x$D" != "x" ]; then exit 1 fi -rm -rf ${TMP_LOCALE} -mkdir -p ${TMP_LOCALE} -if [ -f ${localedir}/locale-archive ]; then - cp ${localedir}/locale-archive ${TMP_LOCALE}/ -fi -localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s --prefix=/tmp/locale %s -mkdir -p ${localedir}/ -mv ${TMP_LOCALE}/locale-archive ${localedir}/ -rm -rf ${TMP_LOCALE} +localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s %s } -# indentation removed on purpose locale_base_postrm() { #!/bin/sh - -rm -rf ${TMP_LOCALE} -mkdir -p ${TMP_LOCALE} -if [ -f ${localedir}/locale-archive ]; then - cp ${localedir}/locale-archive ${TMP_LOCALE}/ -fi -localedef --delete-from-archive --inputfile=${datadir}/locales/%s --charmap=%s --prefix=/tmp/locale %s -mv ${TMP_LOCALE}/locale-archive ${localedir}/ -rm -rf ${TMP_LOCALE} +localedef --delete-from-archive --inputfile=${datadir}/locales/%s --charmap=%s %s } - -TMP_LOCALE="/tmp/locale${localedir}" LOCALETREESRC ?= "${PKGD}" do_prep_locale_tree() { @@ -120,21 +94,21 @@ inherit qemu python package_do_split_gconvs () { import re - if (d.getVar('PACKAGE_NO_GCONV', True) == '1'): + if (d.getVar('PACKAGE_NO_GCONV') == '1'): bb.note("package requested not splitting gconvs") return - if not d.getVar('PACKAGES', True): + if not d.getVar('PACKAGES'): return - mlprefix = d.getVar("MLPREFIX", True) or "" + mlprefix = d.getVar("MLPREFIX") or "" - bpn = d.getVar('BPN', True) - libdir = d.getVar('libdir', True) + bpn = d.getVar('BPN') + libdir = d.getVar('libdir') if not libdir: bb.error("libdir not defined") return - datadir = d.getVar('datadir', True) + datadir = d.getVar('datadir') if not datadir: bb.error("datadir not defined") return @@ -142,7 +116,7 @@ python package_do_split_gconvs () { gconv_libdir = base_path_join(libdir, "gconv") charmap_dir = base_path_join(datadir, "i18n", "charmaps") locales_dir = base_path_join(datadir, "i18n", "locales") - binary_locales_dir = d.getVar('localedir', True) + binary_locales_dir = d.getVar('localedir') def calc_gconv_deps(fn, pkg, file_regex, output_pattern, group): deps = [] @@ -150,6 +124,7 @@ python package_do_split_gconvs () { c_re = re.compile('^copy "(.*)"') i_re = re.compile('^include "(\w+)".*') for l in f.readlines(): + l = l.decode("latin-1") m = c_re.match(l) or i_re.match(l) if m: dp = legitimize_package_name('%s%s-gconv-%s' % (mlprefix, bpn, m.group(1))) @@ -171,6 +146,7 @@ python package_do_split_gconvs () { c_re = re.compile('^copy "(.*)"') i_re = re.compile('^include "(\w+)".*') for l in f.readlines(): + l = l.decode("latin-1") m = c_re.match(l) or i_re.match(l) if m: dp = legitimize_package_name('%s%s-charmap-%s' % (mlprefix, bpn, m.group(1))) @@ -191,6 +167,7 @@ python package_do_split_gconvs () { c_re = re.compile('^copy "(.*)"') i_re = re.compile('^include "(\w+)".*') for l in f.readlines(): + l = l.decode("latin-1") m = c_re.match(l) or i_re.match(l) if m: dp = legitimize_package_name(mlprefix+bpn+'-localedata-%s' % m.group(1)) @@ -206,13 +183,13 @@ python package_do_split_gconvs () { description='locale definition for %s', hook=calc_locale_deps, extra_depends='') d.setVar('PACKAGES', d.getVar('PACKAGES', False) + ' ' + d.getVar('MLPREFIX', False) + bpn + '-gconv') - use_bin = d.getVar("GLIBC_INTERNAL_USE_BINARY_LOCALE", True) + use_bin = d.getVar("GLIBC_INTERNAL_USE_BINARY_LOCALE") dot_re = re.compile("(.*)\.(.*)") # Read in supported locales and associated encodings supported = {} - with open(base_path_join(d.getVar('WORKDIR', True), "SUPPORTED")) as f: + with open(base_path_join(d.getVar('WORKDIR'), "SUPPORTED")) as f: for line in f.readlines(): try: locale, charset = line.rstrip().split() @@ -221,9 +198,9 @@ python package_do_split_gconvs () { supported[locale] = charset # GLIBC_GENERATE_LOCALES var specifies which locales to be generated. empty or "all" means all locales - to_generate = d.getVar('GLIBC_GENERATE_LOCALES', True) + to_generate = d.getVar('GLIBC_GENERATE_LOCALES') if not to_generate or to_generate == 'all': - to_generate = supported.keys() + to_generate = sorted(supported.keys()) else: to_generate = to_generate.split() for locale in to_generate: @@ -236,35 +213,34 @@ python package_do_split_gconvs () { supported[locale] = charset def output_locale_source(name, pkgname, locale, encoding): - d.setVar('RDEPENDS_%s' % pkgname, 'localedef %s-localedata-%s %s-charmap-%s' % \ - (mlprefix+bpn, legitimize_package_name(locale), mlprefix+bpn, legitimize_package_name(encoding))) - d.setVar('pkg_postinst_%s' % pkgname, d.getVar('locale_base_postinst', True) \ + d.setVar('RDEPENDS_%s' % pkgname, '%slocaledef %s-localedata-%s %s-charmap-%s' % \ + (mlprefix, mlprefix+bpn, legitimize_package_name(locale), mlprefix+bpn, legitimize_package_name(encoding))) + d.setVar('pkg_postinst_%s' % pkgname, d.getVar('locale_base_postinst') \ % (locale, encoding, locale)) - d.setVar('pkg_postrm_%s' % pkgname, d.getVar('locale_base_postrm', True) % \ + d.setVar('pkg_postrm_%s' % pkgname, d.getVar('locale_base_postrm') % \ (locale, encoding, locale)) def output_locale_binary_rdepends(name, pkgname, locale, encoding): - m = re.match("(.*)\.(.*)", name) - if m: - libc_name = "%s.%s" % (m.group(1), m.group(2).lower()) - else: - libc_name = name - d.setVar('RDEPENDS_%s' % pkgname, legitimize_package_name('%s-binary-localedata-%s' \ - % (mlprefix+bpn, libc_name))) + dep = legitimize_package_name('%s-binary-localedata-%s' % (bpn, name)) + lcsplit = d.getVar('GLIBC_SPLIT_LC_PACKAGES') + if lcsplit and int(lcsplit): + d.appendVar('PACKAGES', ' ' + dep) + d.setVar('ALLOW_EMPTY_%s' % dep, '1') + d.setVar('RDEPENDS_%s' % pkgname, mlprefix + dep) commands = {} def output_locale_binary(name, pkgname, locale, encoding): - treedir = base_path_join(d.getVar("WORKDIR", True), "locale-tree") - ldlibdir = base_path_join(treedir, d.getVar("base_libdir", True)) - path = d.getVar("PATH", True) + treedir = base_path_join(d.getVar("WORKDIR"), "locale-tree") + ldlibdir = base_path_join(treedir, d.getVar("base_libdir")) + path = d.getVar("PATH") i18npath = base_path_join(treedir, datadir, "i18n") gconvpath = base_path_join(treedir, "iconvdata") outputpath = base_path_join(treedir, binary_locales_dir) - use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF", True) or "0" + use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF") or "0" if use_cross_localedef == "1": - target_arch = d.getVar('TARGET_ARCH', True) + target_arch = d.getVar('TARGET_ARCH') locale_arch_options = { \ "arm": " --uint32-align=4 --little-endian ", \ "armeb": " --uint32-align=4 --big-endian ", \ @@ -274,9 +250,13 @@ python package_do_split_gconvs () { "powerpc": " --uint32-align=4 --big-endian ", \ "powerpc64": " --uint32-align=4 --big-endian ", \ "mips": " --uint32-align=4 --big-endian ", \ + "mipsisa32r6": " --uint32-align=4 --big-endian ", \ "mips64": " --uint32-align=4 --big-endian ", \ + "mipsisa64r6": " --uint32-align=4 --big-endian ", \ "mipsel": " --uint32-align=4 --little-endian ", \ + "mipsisa32r6el": " --uint32-align=4 --little-endian ", \ "mips64el":" --uint32-align=4 --little-endian ", \ + "mipsisa64r6el":" --uint32-align=4 --little-endian ", \ "i586": " --uint32-align=4 --little-endian ", \ "i686": " --uint32-align=4 --little-endian ", \ "x86_64": " --uint32-align=4 --little-endian " } @@ -285,9 +265,9 @@ python package_do_split_gconvs () { localedef_opts = locale_arch_options[target_arch] else: bb.error("locale_arch_options not found for target_arch=" + target_arch) - raise bb.build.FuncFailed("unknown arch:" + target_arch + " for locale_arch_options") + bb.fatal("unknown arch:" + target_arch + " for locale_arch_options") - localedef_opts += " --force --old-style --no-archive --prefix=%s \ + localedef_opts += " --force --no-archive --prefix=%s \ --inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/%s" \ % (treedir, treedir, datadir, locale, encoding, outputpath, name) @@ -295,11 +275,11 @@ python package_do_split_gconvs () { (path, i18npath, gconvpath, localedef_opts) else: # earlier slower qemu way qemu = qemu_target_binary(d) - localedef_opts = "--force --old-style --no-archive --prefix=%s \ + localedef_opts = "--force --no-archive --prefix=%s \ --inputfile=%s/i18n/locales/%s --charmap=%s %s" \ % (treedir, datadir, locale, encoding, name) - qemu_options = d.getVar('QEMU_OPTIONS', True) + qemu_options = d.getVar('QEMU_OPTIONS') cmd = "PSEUDO_RELOADED=YES PATH=\"%s\" I18NPATH=\"%s\" %s -L %s \ -E LD_LIBRARY_PATH=%s %s %s/bin/localedef %s" % \ @@ -312,7 +292,7 @@ python package_do_split_gconvs () { def output_locale(name, locale, encoding): pkgname = d.getVar('MLPREFIX', False) + 'locale-base-' + legitimize_package_name(name) d.setVar('ALLOW_EMPTY_%s' % pkgname, '1') - d.setVar('PACKAGES', '%s %s' % (pkgname, d.getVar('PACKAGES', True))) + d.setVar('PACKAGES', '%s %s' % (pkgname, d.getVar('PACKAGES'))) rprovides = ' %svirtual-locale-%s' % (mlprefix, legitimize_package_name(name)) m = re.match("(.*)_(.*)", name) if m: @@ -331,7 +311,9 @@ python package_do_split_gconvs () { bb.note("preparing tree for binary locale generation") bb.build.exec_func("do_prep_locale_tree", d) - utf8_only = int(d.getVar('LOCALE_UTF8_ONLY', True) or 0) + utf8_only = int(d.getVar('LOCALE_UTF8_ONLY') or 0) + utf8_is_default = int(d.getVar('LOCALE_UTF8_IS_DEFAULT') or 0) + encodings = {} for locale in to_generate: charset = supported[locale] @@ -344,10 +326,11 @@ python package_do_split_gconvs () { else: base = locale - # Precompiled locales are kept as is, obeying SUPPORTED, while - # others are adjusted, ensuring that the non-suffixed locales - # are utf-8, while the suffixed are not. - if use_bin == "precompiled": + # Non-precompiled locales may be renamed so that the default + # (non-suffixed) encoding is always UTF-8, i.e., instead of en_US and + # en_US.UTF-8, we have en_US and en_US.ISO-8859-1. This implicitly + # contradicts SUPPORTED. + if use_bin == "precompiled" or not utf8_is_default: output_locale(locale, base, charset) else: if charset == 'UTF-8': @@ -355,8 +338,13 @@ python package_do_split_gconvs () { else: output_locale('%s.%s' % (base, charset), base, charset) + def metapkg_hook(file, pkg, pattern, format, basename): + name = basename.split('/', 1)[0] + metapkg = legitimize_package_name('%s-binary-localedata-%s' % (mlprefix+bpn, name)) + d.appendVar('RDEPENDS_%s' % metapkg, ' ' + pkg) + if use_bin == "compile": - makefile = base_path_join(d.getVar("WORKDIR", True), "locale-tree", "Makefile") + makefile = base_path_join(d.getVar("WORKDIR"), "locale-tree", "Makefile") m = open(makefile, "w") m.write("all: %s\n\n" % " ".join(commands.keys())) for cmd in commands: @@ -368,13 +356,18 @@ python package_do_split_gconvs () { bb.build.exec_func("oe_runmake", d) bb.note("collecting binary locales from locale tree") bb.build.exec_func("do_collect_bins_from_locale_tree", d) - do_split_packages(d, binary_locales_dir, file_regex='(.*)', \ - output_pattern=bpn+'-binary-localedata-%s', \ - description='binary locale definition for %s', extra_depends='', allow_dirs=True) - elif use_bin == "precompiled": - do_split_packages(d, binary_locales_dir, file_regex='(.*)', \ - output_pattern=bpn+'-binary-localedata-%s', \ - description='binary locale definition for %s', extra_depends='', allow_dirs=True) + + if use_bin in ('compile', 'precompiled'): + lcsplit = d.getVar('GLIBC_SPLIT_LC_PACKAGES') + if lcsplit and int(lcsplit): + do_split_packages(d, binary_locales_dir, file_regex='^(.*/LC_\w+)', \ + output_pattern=bpn+'-binary-localedata-%s', \ + description='binary locale definition for %s', recursive=True, + hook=metapkg_hook, extra_depends='', allow_dirs=True, match_path=True) + else: + do_split_packages(d, binary_locales_dir, file_regex='(.*)', \ + output_pattern=bpn+'-binary-localedata-%s', \ + description='binary locale definition for %s', extra_depends='', allow_dirs=True) else: bb.note("generation of binary locales disabled. this may break i18n!") @@ -385,4 +378,3 @@ python package_do_split_gconvs () { python populate_packages_prepend () { bb.build.exec_func('package_do_split_gconvs', d) } - diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass index c616a20121..d4be478166 100644 --- a/meta/classes/license.bbclass +++ b/meta/classes/license.bbclass @@ -21,26 +21,29 @@ python write_package_manifest() { license_image_dir = d.expand('${LICENSE_DIRECTORY}/${IMAGE_NAME}') bb.utils.mkdirhier(license_image_dir) from oe.rootfs import image_list_installed_packages + from oe.utils import format_pkg_list + + pkgs = image_list_installed_packages(d) + output = format_pkg_list(pkgs) open(os.path.join(license_image_dir, 'package.manifest'), - 'w+').write(image_list_installed_packages(d)) + 'w+').write(output) +} + +python write_deploy_manifest() { + license_deployed_manifest(d) } python license_create_manifest() { - import re import oe.packagedata from oe.rootfs import image_list_installed_packages - bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split() - bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses) - bad_licenses = expand_wildcard_licenses(d, bad_licenses) - - build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True) + build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS') if build_images_from_feeds == "1": return 0 pkg_dic = {} - for pkg in image_list_installed_packages(d).splitlines(): - pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True), + for pkg in sorted(image_list_installed_packages(d)): + pkg_info = os.path.join(d.getVar('PKGDATA_DIR'), 'runtime-reverse', pkg) pkg_name = os.path.basename(os.readlink(pkg_info)) @@ -49,8 +52,18 @@ python license_create_manifest() { pkg_lic_name = "LICENSE_" + pkg_name pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name] - license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True), - d.getVar('IMAGE_NAME', True), 'license.manifest') + rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'), + d.getVar('IMAGE_NAME'), 'license.manifest') + write_license_files(d, rootfs_license_manifest, pkg_dic) +} + +def write_license_files(d, license_manifest, pkg_dic): + import re + + bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split() + bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses) + bad_licenses = expand_wildcard_licenses(d, bad_licenses) + with open(license_manifest, "w") as license_file: for pkg in sorted(pkg_dic): if bad_licenses: @@ -59,25 +72,33 @@ python license_create_manifest() { oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"], bad_licenses, canonical_license, d) except oe.license.LicenseError as exc: - bb.fatal('%s: %s' % (d.getVar('P', True), exc)) + bb.fatal('%s: %s' % (d.getVar('P'), exc)) else: - pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"]) + pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', ' ', pkg_dic[pkg]["LICENSE"]) pkg_dic[pkg]["LICENSES"] = re.sub(' *', ' ', pkg_dic[pkg]["LICENSES"]) pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split() - license_file.write("PACKAGE NAME: %s\n" % pkg) - license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"]) - license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"]) - license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"]) - - # If the package doesn't contain any file, that is, its size is 0, the license - # isn't relevant as far as the final image is concerned. So doing license check - # doesn't make much sense, skip it. - if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0": - continue + if not "IMAGE_MANIFEST" in pkg_dic[pkg]: + # Rootfs manifest + license_file.write("PACKAGE NAME: %s\n" % pkg) + license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"]) + license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"]) + license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"]) + + # If the package doesn't contain any file, that is, its size is 0, the license + # isn't relevant as far as the final image is concerned. So doing license check + # doesn't make much sense, skip it. + if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0": + continue + else: + # Image manifest + license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"]) + license_file.write("VERSION: %s\n" % pkg_dic[pkg]["PV"]) + license_file.write("LICENSE: %s\n" % pkg_dic[pkg]["LICENSE"]) + license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"]) for lic in pkg_dic[pkg]["LICENSES"]: - lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True), + lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'), pkg_dic[pkg]["PN"], "generic_%s" % re.sub('\+', '', lic)) # add explicity avoid of CLOSED license because isn't generic @@ -93,22 +114,27 @@ python license_create_manifest() { # - Just copy the manifest # - Copy the manifest and the license directories # With both options set we see a .5 M increase in core-image-minimal - copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True) - copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True) + copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST') + copy_lic_dirs = d.getVar('COPY_LIC_DIRS') if copy_lic_manifest == "1": - rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'), + rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS'), 'usr', 'share', 'common-licenses') - os.makedirs(rootfs_license_dir) + bb.utils.mkdirhier(rootfs_license_dir) rootfs_license_manifest = os.path.join(rootfs_license_dir, - 'license.manifest') - os.link(license_manifest, rootfs_license_manifest) + os.path.split(license_manifest)[1]) + if not os.path.exists(rootfs_license_manifest): + os.link(license_manifest, rootfs_license_manifest) if copy_lic_dirs == "1": for pkg in sorted(pkg_dic): pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg) - os.makedirs(pkg_rootfs_license_dir) - pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True), + bb.utils.mkdirhier(pkg_rootfs_license_dir) + pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'), pkg_dic[pkg]["PN"]) + + pkg_manifest_licenses = [canonical_license(d, lic) \ + for lic in pkg_dic[pkg]["LICENSES"]] + licenses = os.listdir(pkg_license_dir) for lic in licenses: rootfs_license = os.path.join(rootfs_license_dir, lic) @@ -116,22 +142,164 @@ python license_create_manifest() { pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic) if re.match("^generic_.*$", lic): - generic_lic = re.search("^generic_(.*)$", lic).group(1) - if oe.license.license_ok(canonical_license(d, - generic_lic), bad_licenses) == False: + generic_lic = canonical_license(d, + re.search("^generic_(.*)$", lic).group(1)) + + # Do not copy generic license into package if isn't + # declared into LICENSES of the package. + if not re.sub('\+$', '', generic_lic) in \ + [re.sub('\+', '', lic) for lic in \ + pkg_manifest_licenses]: + continue + + if oe.license.license_ok(generic_lic, + bad_licenses) == False: continue if not os.path.exists(rootfs_license): os.link(pkg_license, rootfs_license) - os.symlink(os.path.join('..', lic), pkg_rootfs_license) + if not os.path.exists(pkg_rootfs_license): + os.symlink(os.path.join('..', lic), pkg_rootfs_license) else: - if oe.license.license_ok(canonical_license(d, - lic), bad_licenses) == False: + if (oe.license.license_ok(canonical_license(d, + lic), bad_licenses) == False or + os.path.exists(pkg_rootfs_license)): continue os.link(pkg_license, pkg_rootfs_license) -} + + +def license_deployed_manifest(d): + """ + Write the license manifest for the deployed recipes. + The deployed recipes usually includes the bootloader + and extra files to boot the target. + """ + + dep_dic = {} + man_dic = {} + lic_dir = d.getVar("LICENSE_DIRECTORY") + + dep_dic = get_deployed_dependencies(d) + for dep in dep_dic.keys(): + man_dic[dep] = {} + # It is necessary to mark this will be used for image manifest + man_dic[dep]["IMAGE_MANIFEST"] = True + man_dic[dep]["PN"] = dep + man_dic[dep]["FILES"] = \ + " ".join(get_deployed_files(dep_dic[dep])) + with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f: + for line in f.readlines(): + key,val = line.split(": ", 1) + man_dic[dep][key] = val[:-1] + + lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'), + d.getVar('IMAGE_NAME')) + bb.utils.mkdirhier(lic_manifest_dir) + image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest') + write_license_files(d, image_license_manifest, man_dic) + +def get_deployed_dependencies(d): + """ + Get all the deployed dependencies of an image + """ + + deploy = {} + # Get all the dependencies for the current task (rootfs). + # Also get EXTRA_IMAGEDEPENDS because the bootloader is + # usually in this var and not listed in rootfs. + # At last, get the dependencies from boot classes because + # it might contain the bootloader. + taskdata = d.getVar("BB_TASKDEPDATA", False) + depends = list(set([dep[0] for dep + in list(taskdata.values()) + if not dep[0].endswith("-native")])) + extra_depends = d.getVar("EXTRA_IMAGEDEPENDS") + boot_depends = get_boot_dependencies(d) + depends.extend(extra_depends.split()) + depends.extend(boot_depends) + depends = list(set(depends)) + + # To verify what was deployed it checks the rootfs dependencies against + # the SSTATE_MANIFESTS for "deploy" task. + # The manifest file name contains the arch. Because we are not running + # in the recipe context it is necessary to check every arch used. + sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS") + sstate_archs = d.getVar("SSTATE_ARCHS") + extra_archs = d.getVar("PACKAGE_EXTRA_ARCHS") + archs = list(set(("%s %s" % (sstate_archs, extra_archs)).split())) + for dep in depends: + # Some recipes have an arch on their own, so we try that first. + special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep) + if special_arch: + sstate_manifest_file = os.path.join(sstate_manifest_dir, + "manifest-%s-%s.deploy" % (special_arch, dep)) + if os.path.exists(sstate_manifest_file): + deploy[dep] = sstate_manifest_file + continue + + for arch in archs: + sstate_manifest_file = os.path.join(sstate_manifest_dir, + "manifest-%s-%s.deploy" % (arch, dep)) + if os.path.exists(sstate_manifest_file): + deploy[dep] = sstate_manifest_file + break + + return deploy +get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA" + +def get_boot_dependencies(d): + """ + Return the dependencies from boot tasks + """ + + depends = [] + boot_depends_string = "" + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + # Only bootimg and bootdirectdisk include the depends flag + boot_tasks = ["do_bootimg", "do_bootdirectdisk",] + + for task in boot_tasks: + boot_depends_string = "%s %s" % (boot_depends_string, + d.getVarFlag(task, "depends") or "") + boot_depends = [dep.split(":")[0] for dep + in boot_depends_string.split() + if not dep.split(":")[0].endswith("-native")] + for dep in boot_depends: + info_file = os.path.join(d.getVar("LICENSE_DIRECTORY"), + dep, "recipeinfo") + # If the recipe and dependency name is the same + if os.path.exists(info_file): + depends.append(dep) + # We need to search for the provider of the dependency + else: + for taskdep in taskdepdata.values(): + # The fifth field contains what the task provides + if dep in taskdep[4]: + info_file = os.path.join( + d.getVar("LICENSE_DIRECTORY"), + taskdep[0], "recipeinfo") + if os.path.exists(info_file): + depends.append(taskdep[0]) + break + return depends +get_boot_dependencies[vardepsexclude] = "BB_TASKDEPDATA" + +def get_deployed_files(man_file): + """ + Get the files deployed from the sstate manifest + """ + + dep_files = [] + excluded_files = [] + with open(man_file, "r") as manifest: + all_files = manifest.read() + for f in all_files.splitlines(): + if ((not (os.path.islink(f) or os.path.isdir(f))) and + not os.path.basename(f) in excluded_files): + dep_files.append(os.path.basename(f)) + return dep_files python do_populate_lic() { """ @@ -140,26 +308,38 @@ python do_populate_lic() { lic_files_paths = find_license_files(d) # The base directory we wrangle licenses to - destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True)) + destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('PN')) copy_license_files(lic_files_paths, destdir) + info = get_recipe_info(d) + with open(os.path.join(destdir, "recipeinfo"), "w") as f: + for key in sorted(info.keys()): + f.write("%s: %s\n" % (key, info[key])) } # it would be better to copy them in do_install_append, but find_license_filesa is python python perform_packagecopy_prepend () { enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d) - if d.getVar('CLASSOVERRIDE', True) == 'class-target' and enabled: + if d.getVar('CLASSOVERRIDE') == 'class-target' and enabled: lic_files_paths = find_license_files(d) # LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY - destdir = d.getVar('D', True) + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY', True), d.getVar('PN', True)) + destdir = d.getVar('D') + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY'), d.getVar('PN')) copy_license_files(lic_files_paths, destdir) add_package_and_files(d) } +perform_packagecopy[vardeps] += "LICENSE_CREATE_PACKAGE" + +def get_recipe_info(d): + info = {} + info["PV"] = d.getVar("PV") + info["PR"] = d.getVar("PR") + info["LICENSE"] = d.getVar("LICENSE") + return info def add_package_and_files(d): - packages = d.getVar('PACKAGES', True) - files = d.getVar('LICENSE_FILES_DIRECTORY', True) - pn = d.getVar('PN', True) + packages = d.getVar('PACKAGES') + files = d.getVar('LICENSE_FILES_DIRECTORY') + pn = d.getVar('PN') pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False)) if pn_lic in packages: bb.warn("%s package already existed in %s." % (pn_lic, pn)) @@ -167,7 +347,7 @@ def add_package_and_files(d): # first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY d.setVar('PACKAGES', "%s %s" % (pn_lic, packages)) d.setVar('FILES_' + pn_lic, files) - rrecommends_pn = d.getVar('RRECOMMENDS_' + pn, True) + rrecommends_pn = d.getVar('RRECOMMENDS_' + pn) if rrecommends_pn: d.setVar('RRECOMMENDS_' + pn, "%s %s" % (pn_lic, rrecommends_pn)) else: @@ -175,18 +355,41 @@ def add_package_and_files(d): def copy_license_files(lic_files_paths, destdir): import shutil + import errno bb.utils.mkdirhier(destdir) - for (basename, path) in lic_files_paths: + for (basename, path, beginline, endline) in lic_files_paths: try: src = path dst = os.path.join(destdir, basename) if os.path.exists(dst): os.remove(dst) - if os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev): - os.link(src, dst) - else: - shutil.copyfile(src, dst) + if os.path.islink(src): + src = os.path.realpath(src) + canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev) and beginline is None and endline is None + if canlink: + try: + os.link(src, dst) + except OSError as err: + if err.errno == errno.EXDEV: + # Copy license files if hard-link is not possible even if st_dev is the + # same on source and destination (docker container with device-mapper?) + canlink = False + else: + raise + # Only chown if we did hardling, and, we're running under pseudo + if canlink and os.environ.get('PSEUDO_DISABLED') == '0': + os.chown(dst,0,0) + if not canlink: + begin_idx = int(beginline)-1 if beginline is not None else None + end_idx = int(endline) if endline is not None else None + if begin_idx is None and end_idx is None: + shutil.copyfile(src, dst) + else: + with open(src, 'rb') as src_f: + with open(dst, 'wb') as dst_f: + dst_f.write(b''.join(src_f.readlines()[begin_idx:end_idx])) + except Exception as e: bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e)) @@ -196,34 +399,25 @@ def find_license_files(d): """ import shutil import oe.license + from collections import defaultdict, OrderedDict - pn = d.getVar('PN', True) - for package in d.getVar('PACKAGES', True): - if d.getVar('LICENSE_' + package, True): - license_types = license_types + ' & ' + \ - d.getVar('LICENSE_' + package, True) - - #If we get here with no license types, then that means we have a recipe - #level license. If so, we grab only those. - try: - license_types - except NameError: - # All the license types at the recipe level - license_types = d.getVar('LICENSE', True) - # All the license files for the package - lic_files = d.getVar('LIC_FILES_CHKSUM', True) - pn = d.getVar('PN', True) + lic_files = d.getVar('LIC_FILES_CHKSUM') or "" + pn = d.getVar('PN') # The license files are located in S/LIC_FILE_CHECKSUM. - srcdir = d.getVar('S', True) + srcdir = d.getVar('S') # Directory we store the generic licenses as set in the distro configuration - generic_directory = d.getVar('COMMON_LICENSE_DIR', True) + generic_directory = d.getVar('COMMON_LICENSE_DIR') # List of basename, path tuples lic_files_paths = [] + # hash for keep track generic lics mappings + non_generic_lics = {} + # Entries from LIC_FILES_CHKSUM + lic_chksums = {} license_source_dirs = [] license_source_dirs.append(generic_directory) try: - additional_lic_dirs = d.getVar('LICENSE_PATH', True).split() + additional_lic_dirs = d.getVar('LICENSE_PATH').split() for lic_dir in additional_lic_dirs: license_source_dirs.append(lic_dir) except: @@ -249,7 +443,6 @@ def find_license_files(d): license_source = None # If the generic does not exist we need to check to see if there is an SPDX mapping to it, # unless NO_GENERIC_LICENSE is set. - for lic_dir in license_source_dirs: if not os.path.isfile(os.path.join(lic_dir, license_type)): if d.getVarFlag('SPDXLICENSEMAP', license_type) != None: @@ -263,24 +456,25 @@ def find_license_files(d): license_source = lic_dir break + non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type) if spdx_generic and license_source: # we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest # audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes) - lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic))) + lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic), + None, None)) # The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense # and should not be allowed, warn the user in this case. if d.getVarFlag('NO_GENERIC_LICENSE', license_type): bb.warn("%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type)) - elif d.getVarFlag('NO_GENERIC_LICENSE', license_type): + elif non_generic_lic and non_generic_lic in lic_chksums: # if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source # of the package rather than the license_source_dirs. - for (basename, path) in lic_files_paths: - if d.getVarFlag('NO_GENERIC_LICENSE', license_type) == basename: - lic_files_paths.append(("generic_" + license_type, path)) - break + lic_files_paths.append(("generic_" + license_type, + os.path.join(srcdir, non_generic_lic), None, None)) + non_generic_lics[non_generic_lic] = license_type else: # Add explicity avoid of CLOSED license because this isn't generic if license_type != 'CLOSED': @@ -289,30 +483,42 @@ def find_license_files(d): pass if not generic_directory: - raise bb.build.FuncFailed("COMMON_LICENSE_DIR is unset. Please set this in your distro config") - - if not lic_files: - # No recipe should have an invalid license file. This is checked else - # where, but let's be pedantic - bb.note(pn + ": Recipe file does not have license file information.") - return lic_files_paths + bb.fatal("COMMON_LICENSE_DIR is unset. Please set this in your distro config") for url in lic_files.split(): try: (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) except bb.fetch.MalformedUrl: - raise bb.build.FuncFailed("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF', True), url)) + bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF'), url)) # We want the license filename and path - srclicfile = os.path.join(srcdir, path) - lic_files_paths.append((os.path.basename(path), srclicfile)) + chksum = parm['md5'] if 'md5' in parm else parm['sha256'] + beginline = parm.get('beginline') + endline = parm.get('endline') + lic_chksums[path] = (chksum, beginline, endline) v = FindVisitor() try: - v.visit_string(license_types) + v.visit_string(d.getVar('LICENSE')) except oe.license.InvalidLicense as exc: - bb.fatal('%s: %s' % (d.getVar('PF', True), exc)) + bb.fatal('%s: %s' % (d.getVar('PF'), exc)) except SyntaxError: - bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True))) + bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF'))) + # Add files from LIC_FILES_CHKSUM to list of license files + lic_chksum_paths = defaultdict(OrderedDict) + for path, data in sorted(lic_chksums.items()): + lic_chksum_paths[os.path.basename(path)][data] = (os.path.join(srcdir, path), data[1], data[2]) + for basename, files in lic_chksum_paths.items(): + if len(files) == 1: + # Don't copy again a LICENSE already handled as non-generic + if basename in non_generic_lics: + continue + data = list(files.values())[0] + lic_files_paths.append(tuple([basename] + list(data))) + else: + # If there are multiple different license files with identical + # basenames we rename them to <file>.0, <file>.1, ... + for i, data in enumerate(files.values()): + lic_files_paths.append(tuple(["%s.%d" % (basename, i)] + list(data))) return lic_files_paths @@ -320,7 +526,7 @@ def return_spdx(d, license): """ This function returns the spdx mapping of a license if it exists. """ - return d.getVarFlag('SPDXLICENSEMAP', license, True) + return d.getVarFlag('SPDXLICENSEMAP', license) def canonical_license(d, license): """ @@ -329,9 +535,9 @@ def canonical_license(d, license): 'X' if availabel and the tailing '+' (so GPLv3+ becomes GPL-3.0+), or the passed license if there is no canonical form. """ - lic = d.getVarFlag('SPDXLICENSEMAP', license, True) or "" + lic = d.getVarFlag('SPDXLICENSEMAP', license) or "" if not lic and license.endswith('+'): - lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'), True) + lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+')) if lic: lic += '+' return lic or license @@ -357,7 +563,7 @@ def expand_wildcard_licenses(d, wildcard_licenses): def incompatible_license_contains(license, truevalue, falsevalue, d): license = canonical_license(d, license) - bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE', True) or "").split() + bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() bad_licenses = expand_wildcard_licenses(d, bad_licenses) return truevalue if license in bad_licenses else falsevalue @@ -368,9 +574,9 @@ def incompatible_license(d, dont_want_licenses, package=None): as canonical (SPDX) names. """ import oe.license - license = d.getVar("LICENSE_%s" % package, True) if package else None + license = d.getVar("LICENSE_%s" % package) if package else None if not license: - license = d.getVar('LICENSE', True) + license = d.getVar('LICENSE') # Handles an "or" or two license sets provided by # flattened_licenses(), pick one that works if possible. @@ -381,7 +587,7 @@ def incompatible_license(d, dont_want_licenses, package=None): try: licenses = oe.license.flattened_licenses(license, choose_lic_set) except oe.license.LicenseError as exc: - bb.fatal('%s: %s' % (d.getVar('P', True), exc)) + bb.fatal('%s: %s' % (d.getVar('P'), exc)) return any(not oe.license.license_ok(canonical_license(d, l), \ dont_want_licenses) for l in licenses) @@ -429,16 +635,16 @@ def check_license_flags(d): def all_license_flags_match(license_flags, whitelist): """ Return first unmatched flag, None if all flags match """ - pn = d.getVar('PN', True) + pn = d.getVar('PN') split_whitelist = whitelist.split() for flag in license_flags.split(): if not license_flag_matches(flag, split_whitelist, pn): return flag return None - license_flags = d.getVar('LICENSE_FLAGS', True) + license_flags = d.getVar('LICENSE_FLAGS') if license_flags: - whitelist = d.getVar('LICENSE_FLAGS_WHITELIST', True) + whitelist = d.getVar('LICENSE_FLAGS_WHITELIST') if not whitelist: return license_flags unmatched_flag = all_license_flags_match(license_flags, whitelist) @@ -452,11 +658,11 @@ def check_license_format(d): Validate operators in LICENSES. No spaces are allowed between LICENSES. """ - pn = d.getVar('PN', True) - licenses = d.getVar('LICENSE', True) + pn = d.getVar('PN') + licenses = d.getVar('LICENSE') from oe.license import license_operator, license_operator_chars, license_pattern - elements = filter(lambda x: x.strip(), license_operator.split(licenses)) + elements = list(filter(lambda x: x.strip(), license_operator.split(licenses))) for pos, element in enumerate(elements): if license_pattern.match(element): if pos > 0 and license_pattern.match(elements[pos - 1]): @@ -474,9 +680,11 @@ do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}" do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/" ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; " +do_rootfs[recrdeptask] += "do_populate_lic" + +IMAGE_POSTPROCESS_COMMAND_prepend = "write_deploy_manifest; " +do_image[recrdeptask] += "do_populate_lic" -do_populate_lic_setscene[dirs] = "${LICSSTATEDIR}/${PN}" -do_populate_lic_setscene[cleandirs] = "${LICSSTATEDIR}" python do_populate_lic_setscene () { sstate_setscene(d) } diff --git a/meta/classes/linuxloader.bbclass b/meta/classes/linuxloader.bbclass new file mode 100644 index 0000000000..117b030746 --- /dev/null +++ b/meta/classes/linuxloader.bbclass @@ -0,0 +1,30 @@ + +linuxloader () { + case ${TARGET_ARCH} in + powerpc | microblaze ) + dynamic_loader="${base_libdir}/ld.so.1" + ;; + mipsisa32r6el | mipsisa32r6 | mipsisa64r6el | mipsisa64r6) + dynamic_loader="${base_libdir}/ld-linux-mipsn8.so.1" + ;; + mips* ) + dynamic_loader="${base_libdir}/ld.so.1" + ;; + powerpc64) + dynamic_loader="${base_libdir}/ld64.so.1" + ;; + x86_64) + dynamic_loader="${base_libdir}/ld-linux-x86-64.so.2" + ;; + i*86 ) + dynamic_loader="${base_libdir}/ld-linux.so.2" + ;; + arm ) + dynamic_loader="${base_libdir}/ld-linux.so.3" + ;; + * ) + dynamic_loader="/unknown_dynamic_linker" + ;; + esac + echo $dynamic_loader +} diff --git a/meta/classes/live-vm-common.bbclass b/meta/classes/live-vm-common.bbclass new file mode 100644 index 0000000000..27b137dec6 --- /dev/null +++ b/meta/classes/live-vm-common.bbclass @@ -0,0 +1,62 @@ +# Some of the vars for vm and live image are conflicted, this function +# is used for fixing the problem. +def set_live_vm_vars(d, suffix): + vars = ['GRUB_CFG', 'SYSLINUX_CFG', 'ROOT', 'LABELS', 'INITRD'] + for var in vars: + var_with_suffix = var + '_' + suffix + if d.getVar(var): + bb.warn('Found potential conflicted var %s, please use %s rather than %s' % \ + (var, var_with_suffix, var)) + elif d.getVar(var_with_suffix): + d.setVar(var, d.getVar(var_with_suffix)) + + +EFI = "${@bb.utils.contains("MACHINE_FEATURES", "efi", "1", "0", d)}" +EFI_PROVIDER ?= "grub-efi" +EFI_CLASS = "${@bb.utils.contains("MACHINE_FEATURES", "efi", "${EFI_PROVIDER}", "", d)}" + +# Include legacy boot if MACHINE_FEATURES includes "pcbios" or if it does not +# contain "efi". This way legacy is supported by default if neither is +# specified, maintaining the original behavior. +def pcbios(d): + pcbios = bb.utils.contains("MACHINE_FEATURES", "pcbios", "1", "0", d) + if pcbios == "0": + pcbios = bb.utils.contains("MACHINE_FEATURES", "efi", "0", "1", d) + return pcbios + +PCBIOS = "${@pcbios(d)}" +PCBIOS_CLASS = "${@['','syslinux'][d.getVar('PCBIOS') == '1']}" + +inherit ${EFI_CLASS} +inherit ${PCBIOS_CLASS} + +KERNEL_IMAGETYPE ??= "bzImage" +VM_DEFAULT_KERNEL ??= "${KERNEL_IMAGETYPE}" + +populate_kernel() { + dest=$1 + install -d $dest + + # Install bzImage, initrd, and rootfs.img in DEST for all loaders to use. + bbnote "Trying to install ${DEPLOY_DIR_IMAGE}/${VM_DEFAULT_KERNEL} as $dest/vmlinuz" + if [ -e ${DEPLOY_DIR_IMAGE}/${VM_DEFAULT_KERNEL} ]; then + install -m 0644 ${DEPLOY_DIR_IMAGE}/${VM_DEFAULT_KERNEL} $dest/vmlinuz + else + bbwarn "${DEPLOY_DIR_IMAGE}/${VM_DEFAULT_KERNEL} doesn't exist" + fi + + # initrd is made of concatenation of multiple filesystem images + if [ -n "${INITRD}" ]; then + rm -f $dest/initrd + for fs in ${INITRD} + do + if [ -s "$fs" ]; then + cat $fs >> $dest/initrd + else + bbfatal "$fs is invalid. initrd image creation failed." + fi + done + chmod 0644 $dest/initrd + fi +} + diff --git a/meta/classes/manpages.bbclass b/meta/classes/manpages.bbclass new file mode 100644 index 0000000000..d16237b898 --- /dev/null +++ b/meta/classes/manpages.bbclass @@ -0,0 +1,5 @@ +# Inherit this class to enable or disable building and installation of manpages +# depending on whether 'api-documentation' is in DISTRO_FEATURES. Such building +# tends to pull in the entire XML stack and other tools, so it's not enabled +# by default. +PACKAGECONFIG_append_class-target = " ${@bb.utils.contains('DISTRO_FEATURES', 'api-documentation', 'manpages', '', d)}" diff --git a/meta/classes/metadata_scm.bbclass b/meta/classes/metadata_scm.bbclass index 237e61821d..fa791f04c4 100644 --- a/meta/classes/metadata_scm.bbclass +++ b/meta/classes/metadata_scm.bbclass @@ -4,8 +4,7 @@ METADATA_REVISION ?= "${@base_detect_revision(d)}" def base_detect_revision(d): path = base_get_scmbasepath(d) - scms = [base_get_metadata_git_revision, \ - base_get_metadata_svn_revision] + scms = [base_get_metadata_git_revision] for scm in scms: rev = scm(path, d) @@ -27,7 +26,7 @@ def base_detect_branch(d): return "<unknown>" def base_get_scmbasepath(d): - return d.getVar( 'COREBASE', True) + return os.path.join(d.getVar('COREBASE'), 'meta') def base_get_metadata_monotone_branch(path, d): monotone_branch = "<unknown>" @@ -65,18 +64,19 @@ def base_get_metadata_svn_revision(path, d): return revision def base_get_metadata_git_branch(path, d): - branch = os.popen('cd %s; git branch 2>&1 | grep "^* " | tr -d "* "' % path).read() + import bb.process - if len(branch) != 0: - return branch - return "<unknown>" + try: + rev, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', cwd=path) + except bb.process.ExecutionError: + rev = '<unknown>' + return rev.strip() def base_get_metadata_git_revision(path, d): - f = os.popen("cd %s; git log -n 1 --pretty=oneline -- 2>&1" % path) - data = f.read() - if f.close() is None: - rev = data.split(" ")[0] - if len(rev) != 0: - return rev - return "<unknown>" + import bb.process + try: + rev, _ = bb.process.run('git rev-parse HEAD', cwd=path) + except bb.process.ExecutionError: + rev = '<unknown>' + return rev.strip() diff --git a/meta/classes/migrate_localcount.bbclass b/meta/classes/migrate_localcount.bbclass index aa0df8bb76..810a541316 100644 --- a/meta/classes/migrate_localcount.bbclass +++ b/meta/classes/migrate_localcount.bbclass @@ -6,12 +6,12 @@ python migrate_localcount_handler () { if not e.data: return - pv = e.data.getVar('PV', True) + pv = e.data.getVar('PV') if not 'AUTOINC' in pv: return localcounts = bb.persist_data.persist('BB_URI_LOCALCOUNT', e.data) - pn = e.data.getVar('PN', True) + pn = e.data.getVar('PN') revs = localcounts.get_by_pattern('%%-%s_rev' % pn) counts = localcounts.get_by_pattern('%%-%s_count' % pn) if not revs or not counts: @@ -21,10 +21,10 @@ python migrate_localcount_handler () { bb.warn("The number of revs and localcounts don't match in %s" % pn) return - version = e.data.getVar('PRAUTOINX', True) + version = e.data.getVar('PRAUTOINX') srcrev = bb.fetch2.get_srcrev(e.data) base_ver = 'AUTOINC-%s' % version[:version.find(srcrev)] - pkgarch = e.data.getVar('PACKAGE_ARCH', True) + pkgarch = e.data.getVar('PACKAGE_ARCH') value = max(int(count) for count in counts) if len(revs) == 1: @@ -33,8 +33,8 @@ python migrate_localcount_handler () { else: value += 1 - bb.utils.mkdirhier(e.data.getVar('PRSERV_DUMPDIR', True)) - df = e.data.getVar('LOCALCOUNT_DUMPFILE', True) + bb.utils.mkdirhier(e.data.getVar('PRSERV_DUMPDIR')) + df = e.data.getVar('LOCALCOUNT_DUMPFILE') flock = bb.utils.lockfile("%s.lock" % df) with open(df, 'a') as fd: fd.write('PRAUTO$%s$%s$%s = "%s"\n' % diff --git a/meta/classes/mime.bbclass b/meta/classes/mime.bbclass index 721c73fcff..0df15831c4 100644 --- a/meta/classes/mime.bbclass +++ b/meta/classes/mime.bbclass @@ -1,4 +1,5 @@ -DEPENDS += "shared-mime-info-native shared-mime-info" +DEPENDS += "shared-mime-info" +PACKAGE_WRITE_DEPS += "shared-mime-info-native" mime_postinst() { if [ "$1" = configure ]; then @@ -28,8 +29,8 @@ fi python populate_packages_append () { import re - packages = d.getVar('PACKAGES', True).split() - pkgdest = d.getVar('PKGDEST', True) + packages = d.getVar('PACKAGES').split() + pkgdest = d.getVar('PKGDEST') for pkg in packages: mime_dir = '%s/%s/usr/share/mime/packages' % (pkgdest, pkg) @@ -41,15 +42,15 @@ python populate_packages_append () { mimes.append(f) if mimes: bb.note("adding mime postinst and postrm scripts to %s" % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('mime_postinst', True) + postinst += d.getVar('mime_postinst') d.setVar('pkg_postinst_%s' % pkg, postinst) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) + postrm = d.getVar('pkg_postrm_%s' % pkg) if not postrm: postrm = '#!/bin/sh\n' - postrm += d.getVar('mime_postrm', True) + postrm += d.getVar('mime_postrm') d.setVar('pkg_postrm_%s' % pkg, postrm) bb.note("adding shared-mime-info-data dependency to %s" % pkg) d.appendVar('RDEPENDS_' + pkg, " shared-mime-info-data") diff --git a/meta/classes/mirrors.bbclass b/meta/classes/mirrors.bbclass index b96c071f65..1174ef6823 100644 --- a/meta/classes/mirrors.bbclass +++ b/meta/classes/mirrors.bbclass @@ -21,13 +21,13 @@ ${DEBIAN_MIRROR} ftp://ftp.se.debian.org/debian/pool \n \ ${DEBIAN_MIRROR} ftp://ftp.tr.debian.org/debian/pool \n \ ${GNU_MIRROR} ftp://mirrors.kernel.org/gnu \n \ ${KERNELORG_MIRROR} http://www.kernel.org/pub \n \ -ftp://ftp.gnupg.org/gcrypt/ ftp://ftp.franken.de/pub/crypt/mirror/ftp.gnupg.org/gcrypt/ \n \ -ftp://ftp.gnupg.org/gcrypt/ ftp://ftp.surfnet.nl/pub/security/gnupg/ \n \ -ftp://ftp.gnupg.org/gcrypt/ http://gulus.USherbrooke.ca/pub/appl/GnuPG/ \n \ +${GNUPG_MIRROR} ftp://ftp.gnupg.org/gcrypt \n \ +${GNUPG_MIRROR} ftp://ftp.franken.de/pub/crypt/mirror/ftp.gnupg.org/gcrypt \n \ +${GNUPG_MIRROR} ftp://mirrors.dotsrc.org/gcrypt \n \ ftp://dante.ctan.org/tex-archive ftp://ftp.fu-berlin.de/tex/CTAN \n \ ftp://dante.ctan.org/tex-archive http://sunsite.sut.ac.jp/pub/archives/ctan/ \n \ ftp://dante.ctan.org/tex-archive http://ctan.unsw.edu.au/ \n \ -ftp://ftp.gnutls.org/gcrypt/gnutls ftp://ftp.gnupg.org/gcrypt/gnutls/ \n \ +ftp://ftp.gnutls.org/gcrypt/gnutls ${GNUPG_MIRROR}/gnutls \n \ http://ftp.info-zip.org/pub/infozip/src/ http://mirror.switch.ch/ftp/mirror/infozip/src/ \n \ http://ftp.info-zip.org/pub/infozip/src/ ftp://sunsite.icm.edu.pl/pub/unix/archiving/info-zip/src/ \n \ ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/ ftp://ftp.cerias.purdue.edu/pub/tools/unix/sysutils/lsof/ \n \ @@ -40,7 +40,8 @@ ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/ ftp://ftp.tux.org/pub/sites/vic ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/ ftp://gd.tuwien.ac.at/utils/admin-tools/lsof/ \n \ ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/ ftp://sunsite.ualberta.ca/pub/Mirror/lsof/ \n \ ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/ ftp://the.wiretapped.net/pub/security/host-security/lsof/ \n \ -http://www.apache.org/dist http://archive.apache.org/dist \n \ +${APACHE_MIRROR} http://www.us.apache.org/dist \n \ +${APACHE_MIRROR} http://archive.apache.org/dist \n \ http://downloads.sourceforge.net/watchdog/ http://fossies.org/linux/misc/ \n \ ${SAVANNAH_GNU_MIRROR} http://download-mirror.savannah.gnu.org/releases \n \ ${SAVANNAH_NONGNU_MIRROR} http://download-mirror.savannah.nongnu.org/releases \n \ @@ -53,6 +54,7 @@ p4://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ osc://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ https?$://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ ftp://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ +npm://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ cvs://.*/.* http://sources.openembedded.org/ \n \ svn://.*/.* http://sources.openembedded.org/ \n \ git://.*/.* http://sources.openembedded.org/ \n \ @@ -62,6 +64,7 @@ p4://.*/.* http://sources.openembedded.org/ \n \ osc://.*/.* http://sources.openembedded.org/ \n \ https?$://.*/.* http://sources.openembedded.org/ \n \ ftp://.*/.* http://sources.openembedded.org/ \n \ +npm://.*/.* http://sources.openembedded.org/ \n \ ${CPAN_MIRROR} http://cpan.metacpan.org/ \n \ ${CPAN_MIRROR} http://search.cpan.org/CPAN/ \n \ " diff --git a/meta/classes/module-base.bbclass b/meta/classes/module-base.bbclass index 8be26c4f5e..6fe77c01b7 100644 --- a/meta/classes/module-base.bbclass +++ b/meta/classes/module-base.bbclass @@ -1,7 +1,7 @@ inherit kernel-arch # This is instead of DEPENDS = "virtual/kernel" -do_configure[depends] += "virtual/kernel:do_shared_workdir" +do_configure[depends] += "virtual/kernel:do_compile_kernelmodules" export OS = "${TARGET_OS}" export CROSS_COMPILE = "${TARGET_PREFIX}" @@ -19,7 +19,7 @@ KERNEL_OBJECT_SUFFIX = ".ko" PACKAGE_ARCH = "${MACHINE_ARCH}" # Function to ensure the kernel scripts are created. Expected to -# be called before do_compile. See module.bbclass for an exmaple. +# be called before do_compile. See module.bbclass for an example. do_make_scripts() { unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS make CC="${KERNEL_CC}" LD="${KERNEL_LD}" AR="${KERNEL_AR}" \ diff --git a/meta/classes/module.bbclass b/meta/classes/module.bbclass index 0952c0cca7..802476bc7a 100644 --- a/meta/classes/module.bbclass +++ b/meta/classes/module.bbclass @@ -1,11 +1,23 @@ -inherit module-base kernel-module-split +inherit module-base kernel-module-split pkgconfig -addtask make_scripts after do_patch before do_compile +addtask make_scripts after do_prepare_recipe_sysroot before do_compile do_make_scripts[lockfiles] = "${TMPDIR}/kernel-scripts.lock" do_make_scripts[depends] += "virtual/kernel:do_shared_workdir" EXTRA_OEMAKE += "KERNEL_SRC=${STAGING_KERNEL_DIR}" +MODULES_INSTALL_TARGET ?= "modules_install" +MODULES_MODULE_SYMVERS_LOCATION ?= "" + +python __anonymous () { + depends = d.getVar('DEPENDS') + extra_symbols = [] + for dep in depends.split(): + if dep.startswith("kernel-module-"): + extra_symbols.append("${STAGING_INCDIR}/" + dep + "/Module.symvers") + d.setVar('KBUILD_EXTRA_SYMBOLS', " ".join(extra_symbols)) +} + module_do_compile() { unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS oe_runmake KERNEL_PATH=${STAGING_KERNEL_DIR} \ @@ -13,15 +25,28 @@ module_do_compile() { CC="${KERNEL_CC}" LD="${KERNEL_LD}" \ AR="${KERNEL_AR}" \ O=${STAGING_KERNEL_BUILDDIR} \ + KBUILD_EXTRA_SYMBOLS="${KBUILD_EXTRA_SYMBOLS}" \ ${MAKE_TARGETS} } module_do_install() { unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS - oe_runmake DEPMOD=echo INSTALL_MOD_PATH="${D}" \ + oe_runmake DEPMOD=echo MODLIB="${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}" \ CC="${KERNEL_CC}" LD="${KERNEL_LD}" \ O=${STAGING_KERNEL_BUILDDIR} \ - modules_install + ${MODULES_INSTALL_TARGET} + + if [ ! -e "${B}/${MODULES_MODULE_SYMVERS_LOCATION}/Module.symvers" ] ; then + bbwarn "Module.symvers not found in ${B}/${MODULES_MODULE_SYMVERS_LOCATION}" + bbwarn "Please consider setting MODULES_MODULE_SYMVERS_LOCATION to a" + bbwarn "directory below B to get correct inter-module dependencies" + else + install -Dm0644 "${B}/${MODULES_MODULE_SYMVERS_LOCATION}"/Module.symvers ${D}${includedir}/${BPN}/Module.symvers + # Module.symvers contains absolute path to the build directory. + # While it doesn't actually seem to matter which path is specified, + # clear them out to avoid confusion + sed -e 's:${B}/::g' -i ${D}${includedir}/${BPN}/Module.symvers + fi } EXPORT_FUNCTIONS do_compile do_install diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass index 8f61d8d41a..ab04597f93 100644 --- a/meta/classes/multilib.bbclass +++ b/meta/classes/multilib.bbclass @@ -1,20 +1,20 @@ python multilib_virtclass_handler () { - cls = e.data.getVar("BBEXTENDCURR", True) - variant = e.data.getVar("BBEXTENDVARIANT", True) + cls = e.data.getVar("BBEXTENDCURR") + variant = e.data.getVar("BBEXTENDVARIANT") if cls != "multilib" or not variant: return - e.data.setVar('STAGING_KERNEL_DIR', e.data.getVar('STAGING_KERNEL_DIR', True)) + e.data.setVar('STAGING_KERNEL_DIR', e.data.getVar('STAGING_KERNEL_DIR')) # There should only be one kernel in multilib configs # We also skip multilib setup for module packages. - provides = (e.data.getVar("PROVIDES", True) or "").split() + provides = (e.data.getVar("PROVIDES") or "").split() if "virtual/kernel" in provides or bb.data.inherits_class('module-base', e.data): raise bb.parse.SkipPackage("We shouldn't have multilib variants for the kernel") - save_var_name=e.data.getVar("MULTILIB_SAVE_VARNAME", True) or "" + save_var_name=e.data.getVar("MULTILIB_SAVE_VARNAME") or "" for name in save_var_name.split(): - val=e.data.getVar(name, True) + val=e.data.getVar(name) if val: e.data.setVar(name + "_MULTILIB_ORIGINAL", val) @@ -26,6 +26,7 @@ python multilib_virtclass_handler () { if bb.data.inherits_class('image', e.data): e.data.setVar("MLPREFIX", variant + "-") e.data.setVar("PN", variant + "-" + e.data.getVar("PN", False)) + e.data.setVar('SDKTARGETSYSROOT', e.data.getVar('SDKTARGETSYSROOT')) target_vendor = e.data.getVar("TARGET_VENDOR_" + "virtclass-multilib-" + variant, False) if target_vendor: e.data.setVar("TARGET_VENDOR", target_vendor) @@ -35,7 +36,6 @@ python multilib_virtclass_handler () { e.data.setVar("MLPREFIX", variant + "-") override = ":virtclass-multilib-" + variant e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + override) - bb.data.update_data(e.data) return if bb.data.inherits_class('native', e.data): @@ -49,17 +49,23 @@ python multilib_virtclass_handler () { # Expand this since this won't work correctly once we set a multilib into place - e.data.setVar("ALL_MULTILIB_PACKAGE_ARCHS", e.data.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)) + e.data.setVar("ALL_MULTILIB_PACKAGE_ARCHS", e.data.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) override = ":virtclass-multilib-" + variant + blacklist = e.data.getVarFlag('PNBLACKLIST', e.data.getVar('PN')) + if blacklist: + pn_new = variant + "-" + e.data.getVar('PN') + if not e.data.getVarFlag('PNBLACKLIST', pn_new): + e.data.setVarFlag('PNBLACKLIST', pn_new, blacklist) + e.data.setVar("MLPREFIX", variant + "-") e.data.setVar("PN", variant + "-" + e.data.getVar("PN", False)) e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + override) # Expand the WHITELISTs with multilib prefix - for whitelist in ["HOSTTOOLS_WHITELIST_GPL-3.0", "WHITELIST_GPL-3.0", "LGPLv2_WHITELIST_GPL-3.0"]: - pkgs = e.data.getVar(whitelist, True) + for whitelist in ["WHITELIST_GPL-3.0", "LGPLv2_WHITELIST_GPL-3.0"]: + pkgs = e.data.getVar(whitelist) for pkg in pkgs.split(): pkgs += " " + variant + "-" + pkg e.data.setVar(whitelist, pkgs) @@ -77,7 +83,7 @@ multilib_virtclass_handler[eventmask] = "bb.event.RecipePreFinalise" STAGINGCC_prepend = "${BBEXTENDVARIANT}-" python __anonymous () { - variant = d.getVar("BBEXTENDVARIANT", True) + variant = d.getVar("BBEXTENDVARIANT") import oe.classextend @@ -87,16 +93,12 @@ python __anonymous () { clsextend.map_depends_variable("PACKAGE_INSTALL") clsextend.map_depends_variable("LINGUAS_INSTALL") clsextend.map_depends_variable("RDEPENDS") - pinstall = d.getVar("LINGUAS_INSTALL", True) + " " + d.getVar("PACKAGE_INSTALL", True) + pinstall = d.getVar("LINGUAS_INSTALL") + " " + d.getVar("PACKAGE_INSTALL") d.setVar("PACKAGE_INSTALL", pinstall) d.setVar("LINGUAS_INSTALL", "") # FIXME, we need to map this to something, not delete it! d.setVar("PACKAGE_INSTALL_ATTEMPTONLY", "") - if bb.data.inherits_class('populate_sdk_base', d): - clsextend.map_depends_variable("TOOLCHAIN_TARGET_TASK") - clsextend.map_depends_variable("TOOLCHAIN_TARGET_TASK_ATTEMPTONLY") - if bb.data.inherits_class('image', d): return @@ -107,7 +109,7 @@ python __anonymous () { return clsextend.rename_packages() - clsextend.rename_package_variables((d.getVar("PACKAGEVARS", True) or "").split()) + clsextend.rename_package_variables((d.getVar("PACKAGEVARS") or "").split()) clsextend.map_packagevars() clsextend.map_regexp_variable("PACKAGES_DYNAMIC") @@ -122,7 +124,7 @@ PACKAGEFUNCS_append = " do_package_qa_multilib" python do_package_qa_multilib() { def check_mlprefix(pkg, var, mlprefix): - values = bb.utils.explode_deps(d.getVar('%s_%s' % (var, pkg), True) or d.getVar(var, True) or "") + values = bb.utils.explode_deps(d.getVar('%s_%s' % (var, pkg)) or d.getVar(var) or "") candidates = [] for i in values: if i.startswith('virtual/'): @@ -133,14 +135,14 @@ python do_package_qa_multilib() { candidates.append(i) if len(candidates) > 0: msg = "%s package %s - suspicious values '%s' in %s" \ - % (d.getVar('PN', True), pkg, ' '.join(candidates), var) + % (d.getVar('PN'), pkg, ' '.join(candidates), var) package_qa_handle_error("multilib", msg, d) - ml = d.getVar('MLPREFIX', True) + ml = d.getVar('MLPREFIX') if not ml: return - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') for pkg in packages.split(): check_mlprefix(pkg, 'RDEPENDS', ml) check_mlprefix(pkg, 'RPROVIDES', ml) diff --git a/meta/classes/multilib_global.bbclass b/meta/classes/multilib_global.bbclass index 612cfb6919..fd0bfe1273 100644 --- a/meta/classes/multilib_global.bbclass +++ b/meta/classes/multilib_global.bbclass @@ -1,7 +1,7 @@ def preferred_ml_updates(d): # If any PREFERRED_PROVIDER or PREFERRED_VERSION are set, # we need to mirror these variables in the multilib case; - multilibs = d.getVar('MULTILIBS', True) or "" + multilibs = d.getVar('MULTILIBS') or "" if not multilibs: return @@ -29,7 +29,6 @@ def preferred_ml_updates(d): localdata = bb.data.createCopy(d) override = ":virtclass-multilib-" + p localdata.setVar("OVERRIDES", localdata.getVar("OVERRIDES", False) + override) - bb.data.update_data(localdata) if "-canadian-" in pkg: newname = localdata.expand(v) else: @@ -57,7 +56,6 @@ def preferred_ml_updates(d): localdata = bb.data.createCopy(d) override = ":virtclass-multilib-" + p localdata.setVar("OVERRIDES", localdata.getVar("OVERRIDES", False) + override) - bb.data.update_data(localdata) newname = localdata.expand(prov) if newname != prov: newval = localdata.expand(val) @@ -72,6 +70,7 @@ def preferred_ml_updates(d): pkg = pkg.replace("virtual/", "") virt = "virtual/" for p in prefixes: + newval = None if pkg != "kernel": newval = p + "-" + val @@ -79,37 +78,54 @@ def preferred_ml_updates(d): localdata = bb.data.createCopy(d) override = ":virtclass-multilib-" + p localdata.setVar("OVERRIDES", localdata.getVar("OVERRIDES", False) + override) - bb.data.update_data(localdata) newname = localdata.expand(prov) if newname != prov and not d.getVar(newname, False): d.setVar(newname, localdata.expand(newval)) # implement alternative multilib name newname = localdata.expand("PREFERRED_PROVIDER_" + virt + p + "-" + pkg) - if not d.getVar(newname, False): + if not d.getVar(newname, False) and newval != None: d.setVar(newname, localdata.expand(newval)) # Avoid future variable key expansion provexp = d.expand(prov) if prov != provexp and d.getVar(prov, False): d.renameVar(prov, provexp) + def translate_provide(prefix, prov): + if not prov.startswith("virtual/"): + return prefix + "-" + prov + if prov == "virtual/kernel": + return prov + prov = prov.replace("virtual/", "") + return "virtual/" + prefix + "-" + prov - mp = (d.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split() + mp = (d.getVar("MULTI_PROVIDER_WHITELIST") or "").split() extramp = [] for p in mp: if p.endswith("-native") or "-crosssdk-" in p or p.startswith(("nativesdk-", "virtual/nativesdk-")) or 'cross-canadian' in p: continue - virt = "" - if p.startswith("virtual/"): - p = p.replace("virtual/", "") - virt = "virtual/" for pref in prefixes: - extramp.append(virt + pref + "-" + p) + extramp.append(translate_provide(pref, p)) d.setVar("MULTI_PROVIDER_WHITELIST", " ".join(mp + extramp)) + abisafe = (d.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + extras = [] + for p in prefixes: + for a in abisafe: + extras.append(p + "-" + a) + d.appendVar("SIGGEN_EXCLUDERECIPES_ABISAFE", " " + " ".join(extras)) + + siggen_exclude = (d.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() + extras = [] + for p in prefixes: + for a in siggen_exclude: + a1, a2 = a.split("->") + extras.append(translate_provide(p, a1) + "->" + translate_provide(p, a2)) + d.appendVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", " " + " ".join(extras)) + python multilib_virtclass_handler_vendor () { if isinstance(e, bb.event.ConfigParsed): - for v in e.data.getVar("MULTILIB_VARIANTS", True).split(): + for v in e.data.getVar("MULTILIB_VARIANTS").split(): if e.data.getVar("TARGET_VENDOR_virtclass-multilib-" + v, False) is None: e.data.setVar("TARGET_VENDOR_virtclass-multilib-" + v, e.data.getVar("TARGET_VENDOR", False) + "ml" + v) preferred_ml_updates(e.data) @@ -118,17 +134,15 @@ addhandler multilib_virtclass_handler_vendor multilib_virtclass_handler_vendor[eventmask] = "bb.event.ConfigParsed" python multilib_virtclass_handler_global () { - if not e.data: + variant = e.data.getVar("BBEXTENDVARIANT") + if variant: return - variant = e.data.getVar("BBEXTENDVARIANT", True) - - if isinstance(e, bb.event.RecipeParsed) and not variant: - if bb.data.inherits_class('kernel', e.data) or \ + if bb.data.inherits_class('kernel', e.data) or \ bb.data.inherits_class('module-base', e.data) or \ (bb.data.inherits_class('allarch', e.data) and\ not bb.data.inherits_class('packagegroup', e.data)): - variants = (e.data.getVar("MULTILIB_VARIANTS", True) or "").split() + variants = (e.data.getVar("MULTILIB_VARIANTS") or "").split() import oe.classextend clsextends = [] @@ -136,21 +150,21 @@ python multilib_virtclass_handler_global () { clsextends.append(oe.classextend.ClassExtender(variant, e.data)) # Process PROVIDES - origprovs = provs = e.data.getVar("PROVIDES", True) or "" + origprovs = provs = e.data.getVar("PROVIDES") or "" for clsextend in clsextends: provs = provs + " " + clsextend.map_variable("PROVIDES", setvar=False) e.data.setVar("PROVIDES", provs) # Process RPROVIDES - origrprovs = rprovs = e.data.getVar("RPROVIDES", True) or "" + origrprovs = rprovs = e.data.getVar("RPROVIDES") or "" for clsextend in clsextends: rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES", setvar=False) if rprovs.strip(): e.data.setVar("RPROVIDES", rprovs) # Process RPROVIDES_${PN}... - for pkg in (e.data.getVar("PACKAGES", True) or "").split(): - origrprovs = rprovs = e.data.getVar("RPROVIDES_%s" % pkg, True) or "" + for pkg in (e.data.getVar("PACKAGES") or "").split(): + origrprovs = rprovs = e.data.getVar("RPROVIDES_%s" % pkg) or "" for clsextend in clsextends: rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES_%s" % pkg, setvar=False) rprovs = rprovs + " " + clsextend.extname + "-" + pkg @@ -158,5 +172,5 @@ python multilib_virtclass_handler_global () { } addhandler multilib_virtclass_handler_global -multilib_virtclass_handler_global[eventmask] = "bb.event.RecipePreFinalise bb.event.RecipeParsed" +multilib_virtclass_handler_global[eventmask] = "bb.event.RecipeParsed" diff --git a/meta/classes/multilib_header.bbclass b/meta/classes/multilib_header.bbclass index 5ee0a2d562..e03f5b13b2 100644 --- a/meta/classes/multilib_header.bbclass +++ b/meta/classes/multilib_header.bbclass @@ -13,13 +13,9 @@ oe_multilib_header() { ;; *) esac - # We use - # For ARM: We don't support multilib builds. # For MIPS: "n32" is a special case, which needs to be # distinct from both 64-bit and 32-bit. case ${TARGET_ARCH} in - arm*) return - ;; mips*) case "${MIPSPKGSFX_ABI}" in "-n32") ident=n32 @@ -31,9 +27,6 @@ oe_multilib_header() { ;; *) ident=${SITEINFO_BITS} esac - if echo ${TARGET_ARCH} | grep -q arm; then - return - fi for each_header in "$@" ; do if [ ! -f "${D}/${includedir}/$each_header" ]; then bberror "oe_multilib_header: Unable to find header $each_header." @@ -52,3 +45,8 @@ oe_multilib_header() { oe_multilib_header_class-native () { return } + +# Nor do we need multilib headers for nativesdk builds. +oe_multilib_header_class-nativesdk () { + return +} diff --git a/meta/classes/native.bbclass b/meta/classes/native.bbclass index bcbcd61e03..aec1087af5 100644 --- a/meta/classes/native.bbclass +++ b/meta/classes/native.bbclass @@ -42,7 +42,7 @@ HOST_AS_ARCH = "${BUILD_AS_ARCH}" CPPFLAGS = "${BUILD_CPPFLAGS}" CFLAGS = "${BUILD_CFLAGS}" -CXXFLAGS = "${BUILD_CFLAGS}" +CXXFLAGS = "${BUILD_CXXFLAGS}" LDFLAGS = "${BUILD_LDFLAGS}" LDFLAGS_build-darwin = "-L${STAGING_LIBDIR_NATIVE} " @@ -87,8 +87,7 @@ datadir = "${STAGING_DATADIR_NATIVE}" baselib = "lib" -# Libtool's default paths are correct for the native machine -lt_cv_sys_lib_dlsearch_path_spec[unexport] = "1" +export lt_cv_sys_lib_dlsearch_path_spec = "${libdir} ${base_libdir} /lib /lib64 /usr/lib /usr/lib64" NATIVE_PACKAGE_PATH_SUFFIX ?= "" bindir .= "${NATIVE_PACKAGE_PATH_SUFFIX}" @@ -96,7 +95,7 @@ libdir .= "${NATIVE_PACKAGE_PATH_SUFFIX}" libexecdir .= "${NATIVE_PACKAGE_PATH_SUFFIX}" do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}/" -do_populate_sysroot[sstate-outputdirs] = "${STAGING_DIR_NATIVE}/" +do_populate_sysroot[sstate-outputdirs] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/${PN}" # Since we actually install these into situ there is no staging prefix STAGING_DIR_HOST = "" @@ -106,27 +105,40 @@ PKG_CONFIG_DIR = "${libdir}/pkgconfig" EXTRA_NATIVE_PKGCONFIG_PATH ?= "" PKG_CONFIG_PATH .= "${EXTRA_NATIVE_PKGCONFIG_PATH}" PKG_CONFIG_SYSROOT_DIR = "" +PKG_CONFIG_SYSTEM_LIBRARY_PATH[unexport] = "1" +PKG_CONFIG_SYSTEM_INCLUDE_PATH[unexport] = "1" # we dont want libc-uclibc or libc-glibc to kick in for native recipes LIBCOVERRIDE = "" CLASSOVERRIDE = "class-native" MACHINEOVERRIDES = "" +MACHINE_FEATURES = "" PATH_prepend = "${COREBASE}/scripts/native-intercept:" +# This class encodes staging paths into its scripts data so can only be +# reused if we manipulate the paths. +SSTATE_SCAN_CMD ?= "${SSTATE_SCAN_CMD_NATIVE}" + python native_virtclass_handler () { - classextend = e.data.getVar('BBCLASSEXTEND', True) or "" - if "native" not in classextend: + pn = e.data.getVar("PN") + if not pn.endswith("-native"): return - pn = e.data.getVar("PN", True) - if not pn.endswith("-native"): + # Set features here to prevent appends and distro features backfill + # from modifying native distro features + features = set(d.getVar("DISTRO_FEATURES_NATIVE").split()) + filtered = set(bb.utils.filter("DISTRO_FEATURES", d.getVar("DISTRO_FEATURES_FILTER_NATIVE"), d).split()) + d.setVar("DISTRO_FEATURES", " ".join(features | filtered)) + + classextend = e.data.getVar('BBCLASSEXTEND') or "" + if "native" not in classextend: return def map_dependencies(varname, d, suffix = ""): if suffix: varname = varname + "_" + suffix - deps = d.getVar(varname, True) + deps = d.getVar(varname) if not deps: return deps = bb.utils.explode_deps(deps) @@ -145,14 +157,14 @@ python native_virtclass_handler () { e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + ":virtclass-native") map_dependencies("DEPENDS", e.data) - for pkg in [e.data.getVar("PN", True), "", "${PN}"]: + for pkg in [e.data.getVar("PN"), "", "${PN}"]: map_dependencies("RDEPENDS", e.data, pkg) map_dependencies("RRECOMMENDS", e.data, pkg) map_dependencies("RSUGGESTS", e.data, pkg) map_dependencies("RPROVIDES", e.data, pkg) map_dependencies("RREPLACES", e.data, pkg) - provides = e.data.getVar("PROVIDES", True) + provides = e.data.getVar("PROVIDES") nprovides = [] for prov in provides.split(): if prov.find(pn) != -1: @@ -169,13 +181,12 @@ python native_virtclass_handler () { addhandler native_virtclass_handler native_virtclass_handler[eventmask] = "bb.event.RecipePreFinalise" -deltask package -deltask packagedata -deltask package_qa -deltask package_write_ipk -deltask package_write_deb -deltask package_write_rpm -deltask package_write +python do_addto_recipe_sysroot () { + bb.build.exec_func("extend_recipe_sysroot", d) +} +addtask addto_recipe_sysroot after do_populate_sysroot + +inherit nopackages do_packagedata[stamp-extra-info] = "" do_populate_sysroot[stamp-extra-info] = "" diff --git a/meta/classes/nativesdk.bbclass b/meta/classes/nativesdk.bbclass index f74da6267f..655b4560e7 100644 --- a/meta/classes/nativesdk.bbclass +++ b/meta/classes/nativesdk.bbclass @@ -10,6 +10,8 @@ LIBCOVERRIDE = ":${NATIVESDKLIBC}" CLASSOVERRIDE = "class-nativesdk" MACHINEOVERRIDES = "" +MULTILIBS = "" + # # Update PACKAGE_ARCH and PACKAGE_ARCHS # @@ -23,9 +25,7 @@ PACKAGE_ARCHS = "${SDK_PACKAGE_ARCHS}" DEPENDS_append = " chrpath-replacement-native" EXTRANATIVEPATH += "chrpath-native" -STAGING_DIR_HOST = "${STAGING_DIR}/${MULTIMACH_HOST_SYS}" -STAGING_DIR_TARGET = "${STAGING_DIR}/${MULTIMACH_TARGET_SYS}" -PKGDATA_DIR = "${STAGING_DIR_HOST}/pkgdata" +PKGDATA_DIR = "${TMPDIR}/pkgdata/${SDK_SYS}" HOST_ARCH = "${SDK_ARCH}" HOST_VENDOR = "${SDK_VENDOR}" @@ -43,6 +43,10 @@ TARGET_PREFIX = "${SDK_PREFIX}" TARGET_CC_ARCH = "${SDK_CC_ARCH}" TARGET_LD_ARCH = "${SDK_LD_ARCH}" TARGET_AS_ARCH = "${SDK_AS_ARCH}" +TARGET_CPPFLAGS = "${BUILDSDK_CPPFLAGS}" +TARGET_CFLAGS = "${BUILDSDK_CFLAGS}" +TARGET_CXXFLAGS = "${BUILDSDK_CXXFLAGS}" +TARGET_LDFLAGS = "${BUILDSDK_LDFLAGS}" TARGET_FPU = "" EXTRA_OECONF_GCC_FLOAT = "" @@ -62,17 +66,23 @@ export PKG_CONFIG_DIR = "${STAGING_DIR_HOST}${libdir}/pkgconfig" export PKG_CONFIG_SYSROOT_DIR = "${STAGING_DIR_HOST}" python nativesdk_virtclass_handler () { - pn = e.data.getVar("PN", True) + pn = e.data.getVar("PN") if not (pn.endswith("-nativesdk") or pn.startswith("nativesdk-")): return + # Set features here to prevent appends and distro features backfill + # from modifying nativesdk distro features + features = set(d.getVar("DISTRO_FEATURES_NATIVESDK").split()) + filtered = set(bb.utils.filter("DISTRO_FEATURES", d.getVar("DISTRO_FEATURES_FILTER_NATIVESDK"), d).split()) + d.setVar("DISTRO_FEATURES", " ".join(features | filtered)) + e.data.setVar("MLPREFIX", "nativesdk-") - e.data.setVar("PN", "nativesdk-" + e.data.getVar("PN", True).replace("-nativesdk", "").replace("nativesdk-", "")) + e.data.setVar("PN", "nativesdk-" + e.data.getVar("PN").replace("-nativesdk", "").replace("nativesdk-", "")) e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + ":virtclass-nativesdk") } python () { - pn = d.getVar("PN", True) + pn = d.getVar("PN") if not pn.startswith("nativesdk-"): return @@ -80,7 +90,7 @@ python () { clsextend = oe.classextend.NativesdkClassExtender("nativesdk", d) clsextend.rename_packages() - clsextend.rename_package_variables((d.getVar("PACKAGEVARS", True) or "").split()) + clsextend.rename_package_variables((d.getVar("PACKAGEVARS") or "").split()) clsextend.map_depends_variable("DEPENDS") clsextend.map_packagevars() @@ -95,3 +105,5 @@ do_populate_sysroot[stamp-extra-info] = "" do_packagedata[stamp-extra-info] = "" USE_NLS = "${SDKUSE_NLS}" + +OLDEST_KERNEL = "${SDK_OLDEST_KERNEL}" diff --git a/meta/classes/nopackages.bbclass b/meta/classes/nopackages.bbclass new file mode 100644 index 0000000000..559f5078bd --- /dev/null +++ b/meta/classes/nopackages.bbclass @@ -0,0 +1,12 @@ +deltask do_package +deltask do_package_write_rpm +deltask do_package_write_ipk +deltask do_package_write_deb +deltask do_package_qa +deltask do_packagedata +deltask do_package_setscene +deltask do_package_write_rpm_setscene +deltask do_package_write_ipk_setscene +deltask do_package_write_deb_setscene +deltask do_package_qa_setscene +deltask do_packagedata_setscene diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass new file mode 100644 index 0000000000..a69bedbb28 --- /dev/null +++ b/meta/classes/npm.bbclass @@ -0,0 +1,81 @@ +DEPENDS_prepend = "nodejs-native " +RDEPENDS_${PN}_prepend = "nodejs " +S = "${WORKDIR}/npmpkg" + +NPM_INSTALLDIR = "${D}${libdir}/node_modules/${PN}" + +# function maps arch names to npm arch names +def npm_oe_arch_map(target_arch, d): + import re + if re.match('p(pc|owerpc)(|64)', target_arch): return 'ppc' + elif re.match('i.86$', target_arch): return 'ia32' + elif re.match('x86_64$', target_arch): return 'x64' + elif re.match('arm64$', target_arch): return 'arm' + return target_arch + +NPM_ARCH ?= "${@npm_oe_arch_map(d.getVar('TARGET_ARCH'), d)}" +NPM_INSTALL_DEV = "0" + +npm_do_compile() { + # Copy in any additionally fetched modules + if [ -d ${WORKDIR}/node_modules ] ; then + cp -a ${WORKDIR}/node_modules ${S}/ + fi + # changing the home directory to the working directory, the .npmrc will + # be created in this directory + export HOME=${WORKDIR} + if [ "${NPM_INSTALL_DEV}" = "1" ]; then + npm config set dev true + else + npm config set dev false + fi + npm set cache ${WORKDIR}/npm_cache + # clear cache before every build + npm cache clear + # Install pkg into ${S} without going to the registry + if [ "${NPM_INSTALL_DEV}" = "1" ]; then + npm --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --no-registry install + else + npm --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry install + fi +} + +npm_do_install() { + # changing the home directory to the working directory, the .npmrc will + # be created in this directory + export HOME=${WORKDIR} + mkdir -p ${NPM_INSTALLDIR}/ + npm install --prefix ${D}${prefix} -g --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry + if [ -d ${D}${prefix}/etc ] ; then + # This will be empty + rmdir ${D}${prefix}/etc + fi +} + +python populate_packages_prepend () { + instdir = d.expand('${D}${libdir}/node_modules/${PN}') + extrapackages = oe.package.npm_split_package_dirs(instdir) + pkgnames = extrapackages.keys() + d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames)) + for pkgname in pkgnames: + pkgrelpath, pdata = extrapackages[pkgname] + pkgpath = '${libdir}/node_modules/${PN}/' + pkgrelpath + # package names can't have underscores but npm packages sometimes use them + oe_pkg_name = pkgname.replace('_', '-') + expanded_pkgname = d.expand(oe_pkg_name) + d.setVar('FILES_%s' % expanded_pkgname, pkgpath) + if pdata: + version = pdata.get('version', None) + if version: + d.setVar('PKGV_%s' % expanded_pkgname, version) + description = pdata.get('description', None) + if description: + d.setVar('SUMMARY_%s' % expanded_pkgname, description.replace(u"\u2018", "'").replace(u"\u2019", "'")) + d.appendVar('RDEPENDS_%s' % d.getVar('PN'), ' %s' % ' '.join(pkgnames).replace('_', '-')) +} + +FILES_${PN} += " \ + ${libdir}/node_modules/${PN} \ +" + +EXPORT_FUNCTIONS do_compile do_install diff --git a/meta/classes/oelint.bbclass b/meta/classes/oelint.bbclass index 1b051ca22e..2589d34059 100644 --- a/meta/classes/oelint.bbclass +++ b/meta/classes/oelint.bbclass @@ -1,7 +1,7 @@ addtask lint before do_build do_lint[nostamp] = "1" python do_lint() { - pkgname = d.getVar("PN", True) + pkgname = d.getVar("PN") ############################## # Test that DESCRIPTION exists @@ -35,7 +35,7 @@ python do_lint() { # Check that all patches have Signed-off-by and Upstream-Status # srcuri = d.getVar("SRC_URI", False).split() - fpaths = (d.getVar('FILESPATH', True) or '').split(':') + fpaths = (d.getVar('FILESPATH') or '').split(':') def findPatch(patchname): for dir in fpaths: @@ -45,7 +45,7 @@ python do_lint() { def findKey(path, key): ret = True - f = file('%s' % path, mode = 'r') + f = open('%s' % path, mode = 'r') line = f.readline() while line: if line.find(key) != -1: diff --git a/meta/classes/own-mirrors.bbclass b/meta/classes/own-mirrors.bbclass index e235227ec8..12b42675bc 100644 --- a/meta/classes/own-mirrors.bbclass +++ b/meta/classes/own-mirrors.bbclass @@ -9,4 +9,5 @@ p4://.*/.* ${SOURCE_MIRROR_URL} osc://.*/.* ${SOURCE_MIRROR_URL} https?$://.*/.* ${SOURCE_MIRROR_URL} ftp://.*/.* ${SOURCE_MIRROR_URL} +npm://.*/.* ${SOURCE_MIRROR_URL} } diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass index cd92beb394..cc466bd1b2 100644 --- a/meta/classes/package.bbclass +++ b/meta/classes/package.bbclass @@ -39,7 +39,6 @@ # packaging steps inherit packagedata -inherit prserv inherit chrpath # Need the package_qa_handle_error() in insane.bbclass @@ -55,6 +54,14 @@ ALL_MULTILIB_PACKAGE_ARCHS = "${@all_multilib_tune_values(d, 'PACKAGE_ARCHS')}" # rpm is used for the per-file dependency identification PACKAGE_DEPENDS += "rpm-native" + +# If your postinstall can execute at rootfs creation time rather than on +# target but depends on a native/cross tool in order to execute, you need to +# list that tool in PACKAGE_WRITE_DEPENDS. Target package dependencies belong +# in the package dependencies as normal, this is just for native/cross support +# tools at rootfs build time. +PACKAGE_WRITE_DEPS ??= "" + def legitimize_package_name(s): """ Make sure package names are legitimate strings @@ -64,7 +71,7 @@ def legitimize_package_name(s): def fixutf(m): cp = m.group(1) if cp: - return ('\u%s' % cp).decode('unicode_escape').encode('utf-8') + return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape') # Handle unicode codepoints encoded as <U0123>, as in glibc locale files. s = re.sub('<U([0-9A-Fa-f]{1,4})>', fixutf, s) @@ -121,14 +128,17 @@ def do_split_packages(d, root, file_regex, output_pattern, description, postinst """ - dvar = d.getVar('PKGD', True) + dvar = d.getVar('PKGD') + root = d.expand(root) + output_pattern = d.expand(output_pattern) + extra_depends = d.expand(extra_depends) # If the root directory doesn't exist, don't error out later but silently do # no splitting. if not os.path.exists(dvar + root): return [] - ml = d.getVar("MLPREFIX", True) + ml = d.getVar("MLPREFIX") if ml: if not output_pattern.startswith(ml): output_pattern = ml + output_pattern @@ -143,8 +153,8 @@ def do_split_packages(d, root, file_regex, output_pattern, description, postinst extra_depends = " ".join(newdeps) - packages = d.getVar('PACKAGES', True).split() - split_packages = [] + packages = d.getVar('PACKAGES').split() + split_packages = set() if postinst: postinst = '#!/bin/sh\n' + postinst + '\n' @@ -161,7 +171,7 @@ def do_split_packages(d, root, file_regex, output_pattern, description, postinst objs.append(relpath) if extra_depends == None: - extra_depends = d.getVar("PN", True) + extra_depends = d.getVar("PN") if not summary: summary = description @@ -181,13 +191,13 @@ def do_split_packages(d, root, file_regex, output_pattern, description, postinst continue on = legitimize_package_name(m.group(1)) pkg = output_pattern % on - split_packages.append(pkg) + split_packages.add(pkg) if not pkg in packages: if prepend: packages = [pkg] + packages else: packages.append(pkg) - oldfiles = d.getVar('FILES_' + pkg, True) + oldfiles = d.getVar('FILES_' + pkg) newfile = os.path.join(root, o) # These names will be passed through glob() so if the filename actually # contains * or ? (rare, but possible) we need to handle that specially @@ -212,9 +222,9 @@ def do_split_packages(d, root, file_regex, output_pattern, description, postinst d.setVar('FILES_' + pkg, oldfiles + " " + newfile) if extra_depends != '': d.appendVar('RDEPENDS_' + pkg, ' ' + extra_depends) - if not d.getVar('DESCRIPTION_' + pkg, True): + if not d.getVar('DESCRIPTION_' + pkg): d.setVar('DESCRIPTION_' + pkg, description % on) - if not d.getVar('SUMMARY_' + pkg, True): + if not d.getVar('SUMMARY_' + pkg): d.setVar('SUMMARY_' + pkg, summary % on) if postinst: d.setVar('pkg_postinst_' + pkg, postinst) @@ -224,14 +234,14 @@ def do_split_packages(d, root, file_regex, output_pattern, description, postinst hook(f, pkg, file_regex, output_pattern, m.group(1)) d.setVar('PACKAGES', ' '.join(packages)) - return split_packages + return list(split_packages) PACKAGE_DEPENDS += "file-native" python () { - if d.getVar('PACKAGES', True) != '': + if d.getVar('PACKAGES') != '': deps = "" - for dep in (d.getVar('PACKAGE_DEPENDS', True) or "").split(): + for dep in (d.getVar('PACKAGE_DEPENDS') or "").split(): deps += " %s:do_populate_sysroot" % dep d.appendVarFlag('do_package', 'depends', deps) @@ -257,29 +267,45 @@ def files_from_filevars(filevars): continue files.append(f) - for f in files: + symlink_paths = [] + for ind, f in enumerate(files): + # Handle directory symlinks. Truncate path to the lowest level symlink + parent = '' + for dirname in f.split('/')[:-1]: + parent = os.path.join(parent, dirname) + if dirname == '.': + continue + if cpath.islink(parent): + bb.warn("FILES contains file '%s' which resides under a " + "directory symlink. Please fix the recipe and use the " + "real path for the file." % f[1:]) + symlink_paths.append(f) + files[ind] = parent + f = parent + break + if not cpath.islink(f): if cpath.isdir(f): newfiles = [ os.path.join(f,x) for x in os.listdir(f) ] if newfiles: files += newfiles - return files + return files, symlink_paths # Called in package_<rpm,ipk,deb>.bbclass to get the correct list of configuration files def get_conffiles(pkg, d): - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') root = os.path.join(pkgdest, pkg) cwd = os.getcwd() os.chdir(root) - conffiles = d.getVar('CONFFILES_%s' % pkg, True); + conffiles = d.getVar('CONFFILES_%s' % pkg); if conffiles == None: - conffiles = d.getVar('CONFFILES', True) + conffiles = d.getVar('CONFFILES') if conffiles == None: conffiles = "" conffiles = conffiles.split() - conf_orig_list = files_from_filevars(conffiles) + conf_orig_list = files_from_filevars(conffiles)[0] # Remove links and directories from conf_orig_list to get conf_list which only contains normal files conf_list = [] @@ -299,6 +325,15 @@ def get_conffiles(pkg, d): os.chdir(cwd) return conf_list +def checkbuildpath(file, d): + tmpdir = d.getVar('TMPDIR') + with open(file) as f: + file_content = f.read() + if tmpdir in file_content: + return True + + return False + def splitdebuginfo(file, debugfile, debugsrcdir, sourcefile, d): # Function to split a single file into two components, one is the stripped # target system binary, the other contains any debugging information. The @@ -308,11 +343,9 @@ def splitdebuginfo(file, debugfile, debugsrcdir, sourcefile, d): import stat - dvar = d.getVar('PKGD', True) - objcopy = d.getVar("OBJCOPY", True) - debugedit = d.expand("${STAGING_LIBDIR_NATIVE}/rpm/bin/debugedit") - workdir = d.getVar("WORKDIR", True) - workparentdir = d.getVar("DEBUGSRC_OVERRIDE_PATH", True) or os.path.dirname(os.path.dirname(workdir)) + dvar = d.getVar('PKGD') + objcopy = d.getVar("OBJCOPY") + debugedit = d.expand("${STAGING_LIBDIR_NATIVE}/rpm/debugedit") # We ignore kernel modules, we don't generate debug info files. if file.find("/lib/modules/") != -1 and file.endswith(".ko"): @@ -326,7 +359,7 @@ def splitdebuginfo(file, debugfile, debugsrcdir, sourcefile, d): # We need to extract the debug src information here... if debugsrcdir: - cmd = "'%s' -b '%s' -d '%s' -i -l '%s' '%s'" % (debugedit, workparentdir, debugsrcdir, sourcefile, file) + cmd = "'%s' -i -l '%s' '%s'" % (debugedit, sourcefile, file) (retval, output) = oe.utils.getstatusoutput(cmd) if retval: bb.fatal("debugedit failed with exit code %s (cmd was %s)%s" % (retval, cmd, ":\n%s" % output if output else "")) @@ -357,14 +390,21 @@ def copydebugsources(debugsrcdir, d): sourcefile = d.expand("${WORKDIR}/debugsources.list") if debugsrcdir and os.path.isfile(sourcefile): - dvar = d.getVar('PKGD', True) - strip = d.getVar("STRIP", True) - objcopy = d.getVar("OBJCOPY", True) + dvar = d.getVar('PKGD') + strip = d.getVar("STRIP") + objcopy = d.getVar("OBJCOPY") debugedit = d.expand("${STAGING_LIBDIR_NATIVE}/rpm/bin/debugedit") - workdir = d.getVar("WORKDIR", True) + workdir = d.getVar("WORKDIR") workparentdir = os.path.dirname(os.path.dirname(workdir)) workbasedir = os.path.basename(os.path.dirname(workdir)) + "/" + os.path.basename(workdir) + # If build path exists in sourcefile, it means toolchain did not use + # -fdebug-prefix-map to compile + if checkbuildpath(sourcefile, d): + localsrc_prefix = workparentdir + "/" + else: + localsrc_prefix = "/usr/src/debug/" + nosuchdir = [] basepath = dvar for p in debugsrcdir.split("/"): @@ -374,13 +414,16 @@ def copydebugsources(debugsrcdir, d): bb.utils.mkdirhier(basepath) cpath.updatecache(basepath) - processdebugsrc = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '(<internal>|<built-in>)$' | " + # Ignore files from the recipe sysroots (target and native) + processdebugsrc = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " # We need to ignore files that are not actually ours # we do this by only paying attention to items from this package processdebugsrc += "fgrep -zw '%s' | " + # Remove prefix in the source paths + processdebugsrc += "sed 's#%s##g' | " processdebugsrc += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s' 2>/dev/null)" - cmd = processdebugsrc % (sourcefile, workbasedir, workparentdir, dvar, debugsrcdir) + cmd = processdebugsrc % (sourcefile, workbasedir, localsrc_prefix, workparentdir, dvar, debugsrcdir) (retval, output) = oe.utils.getstatusoutput(cmd) # Can "fail" if internal headers/transient sources are attempted #if retval: @@ -434,20 +477,17 @@ def get_package_additional_metadata (pkg_type, d): return "\n".join(metadata_fields).strip() def runtime_mapping_rename (varname, pkg, d): - #bb.note("%s before: %s" % (varname, d.getVar(varname, True))) - - if bb.data.inherits_class('packagegroup', d): - return + #bb.note("%s before: %s" % (varname, d.getVar(varname))) new_depends = {} - deps = bb.utils.explode_dep_versions2(d.getVar(varname, True) or "") + deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "") for depend in deps: new_depend = get_package_mapping(depend, pkg, d) new_depends[new_depend] = deps[depend] d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False)) - #bb.note("%s after: %s" % (varname, d.getVar(varname, True))) + #bb.note("%s after: %s" % (varname, d.getVar(varname))) # # Package functions suitable for inclusion in PACKAGEFUNCS @@ -458,34 +498,34 @@ python package_get_auto_pr() { import re # Support per recipe PRSERV_HOST - pn = d.getVar('PN', True) - host = d.getVar("PRSERV_HOST_" + pn, True) + pn = d.getVar('PN') + host = d.getVar("PRSERV_HOST_" + pn) if not (host is None): d.setVar("PRSERV_HOST", host) - pkgv = d.getVar("PKGV", True) + pkgv = d.getVar("PKGV") # PR Server not active, handle AUTOINC - if not d.getVar('PRSERV_HOST', True): + if not d.getVar('PRSERV_HOST'): if 'AUTOINC' in pkgv: d.setVar("PKGV", pkgv.replace("AUTOINC", "0")) return auto_pr = None - pv = d.getVar("PV", True) - version = d.getVar("PRAUTOINX", True) - pkgarch = d.getVar("PACKAGE_ARCH", True) - checksum = d.getVar("BB_TASKHASH", True) + pv = d.getVar("PV") + version = d.getVar("PRAUTOINX") + pkgarch = d.getVar("PACKAGE_ARCH") + checksum = d.getVar("BB_TASKHASH") - if d.getVar('PRSERV_LOCKDOWN', True): - auto_pr = d.getVar('PRAUTO_' + version + '_' + pkgarch, True) or d.getVar('PRAUTO_' + version, True) or None + if d.getVar('PRSERV_LOCKDOWN'): + auto_pr = d.getVar('PRAUTO_' + version + '_' + pkgarch) or d.getVar('PRAUTO_' + version) or None if auto_pr is None: bb.fatal("Can NOT get PRAUTO from lockdown exported file") d.setVar('PRAUTO',str(auto_pr)) return try: - conn = d.getVar("__PRSERV_CONN", True) + conn = d.getVar("__PRSERV_CONN") if conn is None: conn = oe.prservice.prserv_make_conn(d) if conn is not None: @@ -506,19 +546,19 @@ python package_get_auto_pr() { LOCALEBASEPN ??= "${PN}" python package_do_split_locales() { - if (d.getVar('PACKAGE_NO_LOCALE', True) == '1'): + if (d.getVar('PACKAGE_NO_LOCALE') == '1'): bb.debug(1, "package requested not splitting locales") return - packages = (d.getVar('PACKAGES', True) or "").split() + packages = (d.getVar('PACKAGES') or "").split() - datadir = d.getVar('datadir', True) + datadir = d.getVar('datadir') if not datadir: bb.note("datadir not defined") return - dvar = d.getVar('PKGD', True) - pn = d.getVar('LOCALEBASEPN', True) + dvar = d.getVar('PKGD') + pn = d.getVar('LOCALEBASEPN') if pn + '-locale' in packages: packages.remove(pn + '-locale') @@ -531,10 +571,10 @@ python package_do_split_locales() { locales = os.listdir(localedir) - summary = d.getVar('SUMMARY', True) or pn - description = d.getVar('DESCRIPTION', True) or "" - locale_section = d.getVar('LOCALE_SECTION', True) - mlprefix = d.getVar('MLPREFIX', True) or "" + summary = d.getVar('SUMMARY') or pn + description = d.getVar('DESCRIPTION') or "" + locale_section = d.getVar('LOCALE_SECTION') + mlprefix = d.getVar('MLPREFIX') or "" for l in sorted(locales): ln = legitimize_package_name(l) pkg = pn + '-locale-' + ln @@ -555,14 +595,14 @@ python package_do_split_locales() { # glibc-localedata-translit* won't install as a dependency # for some other package which breaks meta-toolchain # Probably breaks since virtual-locale- isn't provided anywhere - #rdep = (d.getVar('RDEPENDS_%s' % pn, True) or "").split() + #rdep = (d.getVar('RDEPENDS_%s' % pn) or "").split() #rdep.append('%s-locale*' % pn) #d.setVar('RDEPENDS_%s' % pn, ' '.join(rdep)) } python perform_packagecopy () { - dest = d.getVar('D', True) - dvar = d.getVar('PKGD', True) + dest = d.getVar('D') + dvar = d.getVar('PKGD') # Start by package population by taking a copy of the installed # files to operate on @@ -696,8 +736,8 @@ python fixup_perms () { # paths are resolved via BBPATH def get_fs_perms_list(d): str = "" - bbpath = d.getVar('BBPATH', True) - fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES', True) + bbpath = d.getVar('BBPATH') + fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES') if not fs_perms_tables: fs_perms_tables = 'files/fs-perms.txt' for conf_file in fs_perms_tables.split(): @@ -706,9 +746,10 @@ python fixup_perms () { - dvar = d.getVar('PKGD', True) + dvar = d.getVar('PKGD') fs_perms_table = {} + fs_link_table = {} # By default all of the standard directories specified in # bitbake.conf will get 0755 root:root. @@ -734,10 +775,10 @@ python fixup_perms () { 'oldincludedir' ] for path in target_path_vars: - dir = d.getVar(path, True) or "" + dir = d.getVar(path) or "" if dir == "": continue - fs_perms_table[dir] = fs_perms_entry(bb.data.expand("%s 0755 root root false - - -" % (dir), d)) + fs_perms_table[dir] = fs_perms_entry(d.expand("%s 0755 root root false - - -" % (dir))) # Now we actually load from the configuration files for conf in get_fs_perms_list(d).split(): @@ -755,24 +796,32 @@ python fixup_perms () { continue entry = fs_perms_entry(d.expand(line)) if entry and entry.path: - fs_perms_table[entry.path] = entry + if entry.link: + fs_link_table[entry.path] = entry + if entry.path in fs_perms_table: + fs_perms_table.pop(entry.path) + else: + fs_perms_table[entry.path] = entry + if entry.path in fs_link_table: + fs_link_table.pop(entry.path) f.close() # Debug -- list out in-memory table #for dir in fs_perms_table: # bb.note("Fixup Perms: %s: %s" % (dir, str(fs_perms_table[dir]))) + #for link in fs_link_table: + # bb.note("Fixup Perms: %s: %s" % (link, str(fs_link_table[link]))) # We process links first, so we can go back and fixup directory ownership # for any newly created directories - for dir in fs_perms_table: - if not fs_perms_table[dir].link: - continue - + # Process in sorted order so /run gets created before /run/lock, etc. + for entry in sorted(fs_link_table.values(), key=lambda x: x.link): + link = entry.link + dir = entry.path origin = dvar + dir if not (cpath.exists(origin) and cpath.isdir(origin) and not cpath.islink(origin)): continue - link = fs_perms_table[dir].link if link[0] == "/": target = dvar + link ptarget = link @@ -792,9 +841,6 @@ python fixup_perms () { os.symlink(link, origin) for dir in fs_perms_table: - if fs_perms_table[dir].link: - continue - origin = dvar + dir if not (cpath.exists(origin) and cpath.isdir(origin)): continue @@ -814,17 +860,20 @@ python fixup_perms () { python split_and_strip_files () { import stat, errno - dvar = d.getVar('PKGD', True) - pn = d.getVar('PN', True) + dvar = d.getVar('PKGD') + pn = d.getVar('PN') + + oldcwd = os.getcwd() + os.chdir(dvar) # We default to '.debug' style - if d.getVar('PACKAGE_DEBUG_SPLIT_STYLE', True) == 'debug-file-directory': + if d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory': # Single debug-file-directory style debug info debugappend = ".debug" debugdir = "" debuglibdir = "/usr/lib/debug" debugsrcdir = "/usr/src/debug" - elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE', True) == 'debug-without-src': + elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-without-src': # Original OE-core, a.k.a. ".debug", style debug info, but without sources in /usr/src/debug debugappend = "" debugdir = "/.debug" @@ -840,8 +889,6 @@ python split_and_strip_files () { sourcefile = d.expand("${WORKDIR}/debugsources.list") bb.utils.remove(sourcefile) - os.chdir(dvar) - # Return type (bits): # 0 - not elf # 1 - ELF @@ -877,9 +924,10 @@ python split_and_strip_files () { symlinks = {} kernmods = [] inodes = {} - libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True)) - baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True)) - if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'): + libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir")) + baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir")) + if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \ + d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): for root, dirs, files in cpath.walk(dvar): for f in files: file = os.path.join(root, f) @@ -906,7 +954,7 @@ python split_and_strip_files () { continue # Check its an excutable if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \ - or ((file.startswith(libdir) or file.startswith(baselibdir)) and ".so" in f): + or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)): # If it's a symlink, and points to an ELF file, we capture the readlink target if cpath.islink(file): target = os.readlink(file) @@ -920,7 +968,7 @@ python split_and_strip_files () { elf_file = isELF(file) if elf_file & 1: if elf_file & 2: - if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn, True) or "").split(): + if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split(): bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) else: msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) @@ -949,7 +997,7 @@ python split_and_strip_files () { # # First lets process debug splitting # - if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1'): + if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): for file in elffiles: src = file[len(dvar):] dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend @@ -1012,8 +1060,8 @@ python split_and_strip_files () { # # Now lets go back over things and strip them # - if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'): - strip = d.getVar("STRIP", True) + if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1'): + strip = d.getVar("STRIP") sfiles = [] for file in elffiles: elf_file = int(elffiles[file]) @@ -1027,19 +1075,22 @@ python split_and_strip_files () { # # End of strip # + os.chdir(oldcwd) } python populate_packages () { import glob, re - workdir = d.getVar('WORKDIR', True) - outdir = d.getVar('DEPLOY_DIR', True) - dvar = d.getVar('PKGD', True) - packages = d.getVar('PACKAGES', True) - pn = d.getVar('PN', True) + workdir = d.getVar('WORKDIR') + outdir = d.getVar('DEPLOY_DIR') + dvar = d.getVar('PKGD') + packages = d.getVar('PACKAGES') + pn = d.getVar('PN') bb.utils.mkdirhier(outdir) os.chdir(dvar) + + autodebug = not (d.getVar("NOAUTOPACKAGEDEBUG") or False) # Sanity check PACKAGES for duplicates # Sanity should be moved to sanity.bbclass once we have the infrastucture @@ -1049,28 +1100,43 @@ python populate_packages () { if pkg in package_list: msg = "%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg package_qa_handle_error("packages-list", msg, d) + elif autodebug and pkg.endswith("-dbg"): + package_list.insert(0, pkg) else: package_list.append(pkg) d.setVar('PACKAGES', ' '.join(package_list)) - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') seen = [] # os.mkdir masks the permissions with umask so we have to unset it first oldumask = os.umask(0) + debug = [] + for root, dirs, files in cpath.walk(dvar): + dir = root[len(dvar):] + if not dir: + dir = os.sep + for f in (files + dirs): + path = "." + os.path.join(dir, f) + if "/.debug/" in path or path.endswith("/.debug"): + debug.append(path) + for pkg in package_list: root = os.path.join(pkgdest, pkg) bb.utils.mkdirhier(root) - filesvar = d.getVar('FILES_%s' % pkg, True) or "" + filesvar = d.getVar('FILES_%s' % pkg) or "" if "//" in filesvar: msg = "FILES variable for package %s contains '//' which is invalid. Attempting to fix this but you should correct the metadata.\n" % pkg package_qa_handle_error("files-invalid", msg, d) filesvar.replace("//", "/") origfiles = filesvar.split() - files = files_from_filevars(origfiles) + files, symlink_paths = files_from_filevars(origfiles) + + if autodebug and pkg.endswith("-dbg"): + files.extend(debug) for file in files: if (not cpath.islink(file)) and (not cpath.exists(file)): @@ -1108,13 +1174,19 @@ python populate_packages () { fpath = os.path.join(root,file) if not cpath.islink(file): os.link(file, fpath) - fstat = cpath.stat(file) - os.chmod(fpath, fstat.st_mode) - os.chown(fpath, fstat.st_uid, fstat.st_gid) continue ret = bb.utils.copyfile(file, fpath) if ret is False or ret == 0: - raise bb.build.FuncFailed("File population failed") + bb.fatal("File population failed") + + # Check if symlink paths exist + for file in symlink_paths: + if not os.path.exists(os.path.join(root,file)): + bb.fatal("File '%s' cannot be packaged into '%s' because its " + "parent directory structure does not exist. One of " + "its parent directories is a symlink whose target " + "directory is not included in the package." % + (file, pkg)) os.umask(oldumask) os.chdir(workdir) @@ -1122,7 +1194,7 @@ python populate_packages () { # Handle LICENSE_EXCLUSION package_list = [] for pkg in packages.split(): - if d.getVar('LICENSE_EXCLUSION-' + pkg, True): + if d.getVar('LICENSE_EXCLUSION-' + pkg): msg = "%s has an incompatible license. Excluding from packaging." % pkg package_qa_handle_error("incompatible-license", msg, d) else: @@ -1141,19 +1213,20 @@ python populate_packages () { if unshipped != []: msg = pn + ": Files/directories were installed but not shipped in any package:" - if "installed-vs-shipped" in (d.getVar('INSANE_SKIP_' + pn, True) or "").split(): + if "installed-vs-shipped" in (d.getVar('INSANE_SKIP_' + pn) or "").split(): bb.note("Package %s skipping QA tests: installed-vs-shipped" % pn) else: for f in unshipped: msg = msg + "\n " + f - msg = msg + "\nPlease set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install." + msg = msg + "\nPlease set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.\n" + msg = msg + "%s: %d installed and not shipped files." % (pn, len(unshipped)) package_qa_handle_error("installed-vs-shipped", msg, d) } populate_packages[dirs] = "${D}" python package_fixsymlinks () { import errno - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') packages = d.getVar("PACKAGES", False).split() dangling_links = {} @@ -1188,7 +1261,7 @@ python package_fixsymlinks () { bb.note("%s contains dangling symlink to %s" % (pkg, l)) for pkg in newrdepends: - rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or "") + rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg) or "") for p in newrdepends[pkg]: if p not in rdepends: rdepends[p] = [] @@ -1216,14 +1289,14 @@ python emit_pkgdata() { def write_if_exists(f, pkg, var): def encode(str): import codecs - c = codecs.getencoder("string_escape") - return c(str)[0] + c = codecs.getencoder("unicode_escape") + return c(str)[0].decode("latin1") - val = d.getVar('%s_%s' % (var, pkg), True) + val = d.getVar('%s_%s' % (var, pkg)) if val: f.write('%s_%s: %s\n' % (var, pkg, encode(val))) return val - val = d.getVar('%s' % (var), True) + val = d.getVar('%s' % (var)) if val: f.write('%s: %s\n' % (var, encode(val))) return val @@ -1242,9 +1315,9 @@ python emit_pkgdata() { with open(subdata_file, 'w') as fd: fd.write("PKG_%s: %s" % (ml_pkg, pkg)) - packages = d.getVar('PACKAGES', True) - pkgdest = d.getVar('PKGDEST', True) - pkgdatadir = d.getVar('PKGDESTWORK', True) + packages = d.getVar('PACKAGES') + pkgdest = d.getVar('PKGDEST') + pkgdatadir = d.getVar('PKGDESTWORK') # Take shared lock since we're only reading, not writing lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True) @@ -1254,9 +1327,9 @@ python emit_pkgdata() { f.write("PACKAGES: %s\n" % packages) f.close() - pn = d.getVar('PN', True) - global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS', True) or "").split() - variants = (d.getVar('MULTILIB_VARIANTS', True) or "").split() + pn = d.getVar('PN') + global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() + variants = (d.getVar('MULTILIB_VARIANTS') or "").split() if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): write_extra_pkgs(variants, pn, packages, pkgdatadir) @@ -1264,10 +1337,10 @@ python emit_pkgdata() { if (bb.data.inherits_class('allarch', d) and not bb.data.inherits_class('packagegroup', d)): write_extra_pkgs(global_variants, pn, packages, pkgdatadir) - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') for pkg in packages.split(): - pkgval = d.getVar('PKG_%s' % pkg, True) + pkgval = d.getVar('PKG_%s' % pkg) if pkgval is None: pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) @@ -1275,11 +1348,14 @@ python emit_pkgdata() { pkgdestpkg = os.path.join(pkgdest, pkg) files = {} total_size = 0 + seen = set() for f in pkgfiles[pkg]: relpth = os.path.relpath(f, pkgdestpkg) fstat = os.lstat(f) - total_size += fstat.st_size files[os.sep + relpth] = fstat.st_size + if fstat.st_ino not in seen: + seen.add(fstat.st_ino) + total_size += fstat.st_size d.setVar('FILES_INFO', json.dumps(files)) subdata_file = pkgdatadir + "/runtime/%s" % pkg @@ -1304,17 +1380,18 @@ python emit_pkgdata() { write_if_exists(sf, pkg, 'PKG') write_if_exists(sf, pkg, 'ALLOW_EMPTY') write_if_exists(sf, pkg, 'FILES') + write_if_exists(sf, pkg, 'CONFFILES') write_if_exists(sf, pkg, 'pkg_postinst') write_if_exists(sf, pkg, 'pkg_postrm') write_if_exists(sf, pkg, 'pkg_preinst') write_if_exists(sf, pkg, 'pkg_prerm') write_if_exists(sf, pkg, 'FILERPROVIDESFLIST') write_if_exists(sf, pkg, 'FILES_INFO') - for dfile in (d.getVar('FILERPROVIDESFLIST_' + pkg, True) or "").split(): + for dfile in (d.getVar('FILERPROVIDESFLIST_' + pkg) or "").split(): write_if_exists(sf, pkg, 'FILERPROVIDES_' + dfile) write_if_exists(sf, pkg, 'FILERDEPENDSFLIST') - for dfile in (d.getVar('FILERDEPENDSFLIST_' + pkg, True) or "").split(): + for dfile in (d.getVar('FILERDEPENDSFLIST_' + pkg) or "").split(): write_if_exists(sf, pkg, 'FILERDEPENDS_' + dfile) sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) @@ -1327,9 +1404,9 @@ python emit_pkgdata() { bb.utils.mkdirhier(os.path.dirname(subdata_sym)) oe.path.symlink("../../runtime/%s" % pkg, subdata_sym, True) - allow_empty = d.getVar('ALLOW_EMPTY_%s' % pkg, True) + allow_empty = d.getVar('ALLOW_EMPTY_%s' % pkg) if not allow_empty: - allow_empty = d.getVar('ALLOW_EMPTY', True) + allow_empty = d.getVar('ALLOW_EMPTY') root = "%s/%s" % (pkgdest, pkg) os.chdir(root) g = glob('*') @@ -1357,7 +1434,7 @@ if [ x"$D" = "x" ]; then fi } -RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt" +RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/rpmdeps --rcfile ${STAGING_LIBDIR_NATIVE}/rpm/rpmrc --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmconfigdir ${STAGING_LIBDIR_NATIVE}/rpm/'" # Collect perfile run-time dependency metadata # Output: @@ -1368,24 +1445,25 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LI # FILERDEPENDS_filepath_pkg - per file dep python package_do_filedeps() { - if d.getVar('SKIP_FILEDEPS', True) == '1': + if d.getVar('SKIP_FILEDEPS') == '1': return - pkgdest = d.getVar('PKGDEST', True) - packages = d.getVar('PACKAGES', True) - rpmdeps = d.getVar('RPMDEPS', True) + pkgdest = d.getVar('PKGDEST') + packages = d.getVar('PACKAGES') + rpmdeps = d.getVar('RPMDEPS') + magic = d.expand("${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc") def chunks(files, n): return [files[i:i+n] for i in range(0, len(files), n)] pkglist = [] for pkg in packages.split(): - if d.getVar('SKIP_FILEDEPS_' + pkg, True) == '1': + if d.getVar('SKIP_FILEDEPS_' + pkg) == '1': continue if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-'): continue for files in chunks(pkgfiles[pkg], 100): - pkglist.append((pkg, files, rpmdeps, pkgdest)) + pkglist.append((pkg, files, rpmdeps, pkgdest, magic)) processed = oe.utils.multiprocess_exec( pkglist, oe.package.filedeprunner) @@ -1423,28 +1501,28 @@ python package_do_shlibs() { import re, pipes import subprocess as sub - exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', 0) + exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', False) if exclude_shlibs: bb.note("not generating shlibs") return lib_re = re.compile("^.*\.so") - libdir_re = re.compile(".*/%s$" % d.getVar('baselib', True)) + libdir_re = re.compile(".*/%s$" % d.getVar('baselib')) - packages = d.getVar('PACKAGES', True) - targetos = d.getVar('TARGET_OS', True) + packages = d.getVar('PACKAGES') + targetos = d.getVar('TARGET_OS') - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') - ver = d.getVar('PKGV', True) + ver = d.getVar('PKGV') if not ver: msg = "PKGV not defined" package_qa_handle_error("pkgv-undefined", msg, d) return - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') - shlibswork_dir = d.getVar('SHLIBSWORKDIR', True) + shlibswork_dir = d.getVar('SHLIBSWORKDIR') # Take shared lock since we're only reading, not writing lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}")) @@ -1452,7 +1530,7 @@ python package_do_shlibs() { def linux_so(file, needed, sonames, renames, pkgver): needs_ldconfig = False ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '') - cmd = d.getVar('OBJDUMP', True) + " -p " + pipes.quote(file) + " 2>/dev/null" + cmd = d.getVar('OBJDUMP') + " -p " + pipes.quote(file) + " 2>/dev/null" fd = os.popen(cmd) lines = fd.readlines() fd.close() @@ -1461,7 +1539,7 @@ python package_do_shlibs() { m = re.match("\s+RPATH\s+([^\s]*)", l) if m: rpaths = m.group(1).replace("$ORIGIN", ldir).split(":") - rpath = map(os.path.normpath, rpaths) + rpath = list(map(os.path.normpath, rpaths)) for l in lines: m = re.match("\s+NEEDED\s+([^\s]*)", l) if m: @@ -1512,19 +1590,19 @@ python package_do_shlibs() { if file.endswith('.dylib') or file.endswith('.so'): rpath = [] p = sub.Popen([d.expand("${HOST_PREFIX}otool"), '-l', file],stdout=sub.PIPE,stderr=sub.PIPE) - err, out = p.communicate() - # If returned succesfully, process stderr for results + out, err = p.communicate() + # If returned successfully, process stdout for results if p.returncode == 0: - for l in err.split("\n"): + for l in out.split("\n"): l = l.strip() if l.startswith('path '): rpath.append(l.split()[1]) p = sub.Popen([d.expand("${HOST_PREFIX}otool"), '-L', file],stdout=sub.PIPE,stderr=sub.PIPE) - err, out = p.communicate() - # If returned succesfully, process stderr for results + out, err = p.communicate() + # If returned successfully, process stdout for results if p.returncode == 0: - for l in err.split("\n"): + for l in out.split("\n"): l = l.strip() if not l or l.endswith(":"): continue @@ -1534,28 +1612,44 @@ python package_do_shlibs() { if name and name not in needed[pkg]: needed[pkg].append((name, file, [])) - if d.getVar('PACKAGE_SNAP_LIB_SYMLINKS', True) == "1": + def mingw_dll(file, needed, sonames, renames, pkgver): + if not os.path.exists(file): + return + + if file.endswith(".dll"): + # assume all dlls are shared objects provided by the package + sonames.append((os.path.basename(file), os.path.dirname(file).replace(pkgdest + "/" + pkg, ''), pkgver)) + + if (file.endswith(".dll") or file.endswith(".exe")): + # use objdump to search for "DLL Name: .*\.dll" + p = sub.Popen([d.expand("${HOST_PREFIX}objdump"), "-p", file], stdout = sub.PIPE, stderr= sub.PIPE) + out, err = p.communicate() + # process the output, grabbing all .dll names + if p.returncode == 0: + for m in re.finditer("DLL Name: (.*?\.dll)$", out.decode(), re.MULTILINE | re.IGNORECASE): + dllname = m.group(1) + if dllname: + needed[pkg].append((dllname, file, [])) + + if d.getVar('PACKAGE_SNAP_LIB_SYMLINKS') == "1": snap_symlinks = True else: snap_symlinks = False - if (d.getVar('USE_LDCONFIG', True) or "1") == "1": - use_ldconfig = True - else: - use_ldconfig = False + use_ldconfig = bb.utils.contains('DISTRO_FEATURES', 'ldconfig', True, False, d) needed = {} shlib_provider = oe.package.read_shlib_providers(d) for pkg in packages.split(): - private_libs = d.getVar('PRIVATE_LIBS_' + pkg, True) or d.getVar('PRIVATE_LIBS', True) or "" + private_libs = d.getVar('PRIVATE_LIBS_' + pkg) or d.getVar('PRIVATE_LIBS') or "" private_libs = private_libs.split() needs_ldconfig = False bb.debug(2, "calculating shlib provides for %s" % pkg) - pkgver = d.getVar('PKGV_' + pkg, True) + pkgver = d.getVar('PKGV_' + pkg) if not pkgver: - pkgver = d.getVar('PV_' + pkg, True) + pkgver = d.getVar('PV_' + pkg) if not pkgver: pkgver = ver @@ -1568,6 +1662,8 @@ python package_do_shlibs() { continue if targetos == "darwin" or targetos == "darwin8": darwin_so(file, needed, sonames, renames, pkgver) + elif targetos.startswith("mingw"): + mingw_dll(file, needed, sonames, renames, pkgver) elif os.access(file, os.X_OK) or lib_re.match(file): ldconfig = linux_so(file, needed, sonames, renames, pkgver) needs_ldconfig = needs_ldconfig or ldconfig @@ -1592,18 +1688,18 @@ python package_do_shlibs() { fd.close() if needs_ldconfig and use_ldconfig: bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('ldconfig_postinst_fragment', True) + postinst += d.getVar('ldconfig_postinst_fragment') d.setVar('pkg_postinst_%s' % pkg, postinst) bb.debug(1, 'LIBNAMES: pkg %s sonames %s' % (pkg, sonames)) bb.utils.unlockfile(lf) - assumed_libs = d.getVar('ASSUME_SHLIBS', True) + assumed_libs = d.getVar('ASSUME_SHLIBS') if assumed_libs: - libdir = d.getVar("libdir", True) + libdir = d.getVar("libdir") for e in assumed_libs.split(): l, dep_pkg = e.split(":") lib_ver = None @@ -1615,7 +1711,7 @@ python package_do_shlibs() { shlib_provider[l] = {} shlib_provider[l][libdir] = (dep_pkg, lib_ver) - libsearchpath = [d.getVar('libdir', True), d.getVar('base_libdir', True)] + libsearchpath = [d.getVar('libdir'), d.getVar('base_libdir')] for pkg in packages.split(): bb.debug(2, "calculating shlib requirements for %s" % pkg) @@ -1631,7 +1727,7 @@ python package_do_shlibs() { bb.debug(2, '%s: Dependency %s covered by PRIVATE_LIBS' % (pkg, n[0])) continue if n[0] in shlib_provider.keys(): - shlib_provider_path = list() + shlib_provider_path = [] for k in shlib_provider[n[0]].keys(): shlib_provider_path.append(k) match = None @@ -1669,12 +1765,12 @@ python package_do_shlibs() { python package_do_pkgconfig () { import re - packages = d.getVar('PACKAGES', True) - workdir = d.getVar('WORKDIR', True) - pkgdest = d.getVar('PKGDEST', True) + packages = d.getVar('PACKAGES') + workdir = d.getVar('WORKDIR') + pkgdest = d.getVar('PKGDEST') - shlibs_dirs = d.getVar('SHLIBSDIRS', True).split() - shlibswork_dir = d.getVar('SHLIBSWORKDIR', True) + shlibs_dirs = d.getVar('SHLIBSDIRS').split() + shlibswork_dir = d.getVar('SHLIBSWORKDIR') pc_re = re.compile('(.*)\.pc$') var_re = re.compile('(.*)=(.*)') @@ -1706,7 +1802,7 @@ python package_do_pkgconfig () { m = field_re.match(l) if m: hdr = m.group(1) - exp = bb.data.expand(m.group(2), pd) + exp = pd.expand(m.group(2)) if hdr == 'Requires': pkgconfig_needed[pkg] += exp.replace(',', ' ').split() @@ -1759,7 +1855,7 @@ python package_do_pkgconfig () { def read_libdep_files(d): pkglibdeps = {} - packages = d.getVar('PACKAGES', True).split() + packages = d.getVar('PACKAGES').split() for pkg in packages: pkglibdeps[pkg] = {} for extension in ".shlibdeps", ".pcdeps", ".clilibdeps": @@ -1779,9 +1875,9 @@ def read_libdep_files(d): python read_shlibdeps () { pkglibdeps = read_libdep_files(d) - packages = d.getVar('PACKAGES', True).split() + packages = d.getVar('PACKAGES').split() for pkg in packages: - rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or "") + rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg) or "") for dep in pkglibdeps[pkg]: # Add the dep if it's not already there, or if no comparison is set if dep not in rdepends: @@ -1806,14 +1902,14 @@ python package_depchains() { package. """ - packages = d.getVar('PACKAGES', True) - postfixes = (d.getVar('DEPCHAIN_POST', True) or '').split() - prefixes = (d.getVar('DEPCHAIN_PRE', True) or '').split() + packages = d.getVar('PACKAGES') + postfixes = (d.getVar('DEPCHAIN_POST') or '').split() + prefixes = (d.getVar('DEPCHAIN_PRE') or '').split() def pkg_adddeprrecs(pkg, base, suffix, getname, depends, d): #bb.note('depends for %s is %s' % (base, depends)) - rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg, True) or "") + rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg) or "") for depend in depends: if depend.find('-native') != -1 or depend.find('-cross') != -1 or depend.startswith('virtual/'): @@ -1834,7 +1930,7 @@ python package_depchains() { def pkg_addrrecs(pkg, base, suffix, getname, rdepends, d): #bb.note('rdepends for %s is %s' % (base, rdepends)) - rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg, True) or "") + rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg) or "") for depend in rdepends: if depend.find('virtual-locale-') != -1: @@ -1857,12 +1953,12 @@ python package_depchains() { list.append(dep) depends = [] - for dep in bb.utils.explode_deps(d.getVar('DEPENDS', True) or ""): + for dep in bb.utils.explode_deps(d.getVar('DEPENDS') or ""): add_dep(depends, dep) rdepends = [] for pkg in packages.split(): - for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + pkg, True) or ""): + for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + pkg) or ""): add_dep(rdepends, dep) #bb.note('rdepends is %s' % rdepends) @@ -1892,8 +1988,7 @@ python package_depchains() { for pkg in pkglibdeps: for k in pkglibdeps[pkg]: add_dep(pkglibdeplist, k) - # FIXME this should not look at PN once all task recipes inherit from task.bbclass - dbgdefaultdeps = ((d.getVar('DEPCHAIN_DBGDEFAULTDEPS', True) == '1') or (d.getVar('PN', True) or '').startswith('packagegroup-')) + dbgdefaultdeps = ((d.getVar('DEPCHAIN_DBGDEFAULTDEPS') == '1') or (bb.data.inherits_class('packagegroup', d))) for suffix in pkgs: for pkg in pkgs[suffix]: @@ -1910,19 +2005,19 @@ python package_depchains() { pkg_addrrecs(pkg, base, suffix, func, rdepends, d) else: rdeps = [] - for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + base, True) or ""): + for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + base) or ""): add_dep(rdeps, dep) pkg_addrrecs(pkg, base, suffix, func, rdeps, d) } # Since bitbake can't determine which variables are accessed during package # iteration, we need to list them here: -PACKAGEVARS = "FILES RDEPENDS RRECOMMENDS SUMMARY DESCRIPTION RSUGGESTS RPROVIDES RCONFLICTS PKG ALLOW_EMPTY pkg_postinst pkg_postrm INITSCRIPT_NAME INITSCRIPT_PARAMS DEBIAN_NOAUTONAME ALTERNATIVE PKGE PKGV PKGR USERADD_PARAM GROUPADD_PARAM CONFFILES SYSTEMD_SERVICE LICENSE SECTION pkg_preinst pkg_prerm RREPLACES GROUPMEMS_PARAM SYSTEMD_AUTO_ENABLE" +PACKAGEVARS = "FILES RDEPENDS RRECOMMENDS SUMMARY DESCRIPTION RSUGGESTS RPROVIDES RCONFLICTS PKG ALLOW_EMPTY pkg_postinst pkg_postrm INITSCRIPT_NAME INITSCRIPT_PARAMS DEBIAN_NOAUTONAME ALTERNATIVE PKGE PKGV PKGR USERADD_PARAM GROUPADD_PARAM CONFFILES SYSTEMD_SERVICE LICENSE SECTION pkg_preinst pkg_prerm RREPLACES GROUPMEMS_PARAM SYSTEMD_AUTO_ENABLE SKIP_FILEDEPS PRIVATE_LIBS" def gen_packagevar(d): ret = [] - pkgs = (d.getVar("PACKAGES", True) or "").split() - vars = (d.getVar("PACKAGEVARS", True) or "").split() + pkgs = (d.getVar("PACKAGES") or "").split() + vars = (d.getVar("PACKAGEVARS") or "").split() for p in pkgs: for v in vars: ret.append(v + "_" + p) @@ -1970,16 +2065,16 @@ python do_package () { # Sanity test the setup ########################################################################### - packages = (d.getVar('PACKAGES', True) or "").split() + packages = (d.getVar('PACKAGES') or "").split() if len(packages) < 1: bb.debug(1, "No packages to build, skipping do_package") return - workdir = d.getVar('WORKDIR', True) - outdir = d.getVar('DEPLOY_DIR', True) - dest = d.getVar('D', True) - dvar = d.getVar('PKGD', True) - pn = d.getVar('PN', True) + workdir = d.getVar('WORKDIR') + outdir = d.getVar('DEPLOY_DIR') + dest = d.getVar('D') + dvar = d.getVar('PKGD') + pn = d.getVar('PN') if not workdir or not outdir or not dest or not dvar or not pn: msg = "WORKDIR, DEPLOY_DIR, D, PN and PKGD all must be defined, unable to package" @@ -1997,7 +2092,7 @@ python do_package () { # code pre-expands some frequently used variables def expandVar(x, d): - d.setVar(x, d.getVar(x, True)) + d.setVar(x, d.getVar(x)) for x in 'PN', 'PV', 'BPN', 'TARGET_SYS', 'EXTENDPRAUTO': expandVar(x, d) @@ -2006,7 +2101,7 @@ python do_package () { # Setup PKGD (from D) ########################################################################### - for f in (d.getVar('PACKAGEBUILDPKGD', True) or '').split(): + for f in (d.getVar('PACKAGEBUILDPKGD') or '').split(): bb.build.exec_func(f, d) ########################################################################### @@ -2015,7 +2110,7 @@ python do_package () { cpath = oe.cachedpath.CachedPath() - for f in (d.getVar('PACKAGESPLITFUNCS', True) or '').split(): + for f in (d.getVar('PACKAGESPLITFUNCS') or '').split(): bb.build.exec_func(f, d) ########################################################################### @@ -2025,16 +2120,20 @@ python do_package () { # Build global list of files in each split package global pkgfiles pkgfiles = {} - packages = d.getVar('PACKAGES', True).split() - pkgdest = d.getVar('PKGDEST', True) + packages = d.getVar('PACKAGES').split() + pkgdest = d.getVar('PKGDEST') for pkg in packages: pkgfiles[pkg] = [] for walkroot, dirs, files in cpath.walk(pkgdest + "/" + pkg): for file in files: pkgfiles[pkg].append(walkroot + os.sep + file) - for f in (d.getVar('PACKAGEFUNCS', True) or '').split(): + for f in (d.getVar('PACKAGEFUNCS') or '').split(): bb.build.exec_func(f, d) + + qa_sane = d.getVar("QA_SANE") + if not qa_sane: + bb.fatal("Fatal QA errors found, failing task.") } do_package[dirs] = "${SHLIBSWORKDIR} ${PKGDESTWORK} ${D}" @@ -2079,8 +2178,7 @@ def mapping_rename_hook(d): Rewrite variables to account for package renaming in things like debian.bbclass or manual PKG variable name changes """ - pkg = d.getVar("PKG", True) + pkg = d.getVar("PKG") runtime_mapping_rename("RDEPENDS", pkg, d) runtime_mapping_rename("RRECOMMENDS", pkg, d) runtime_mapping_rename("RSUGGESTS", pkg, d) - diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass index d2fea4fcc2..eacabcdb61 100644 --- a/meta/classes/package_deb.bbclass +++ b/meta/classes/package_deb.bbclass @@ -6,14 +6,39 @@ inherit package IMAGE_PKGTYPE ?= "deb" -DPKG_ARCH ?= "${TARGET_ARCH}" +DPKG_ARCH ?= "${@debian_arch_map(d.getVar('TARGET_ARCH'), d.getVar('TUNE_FEATURES'))}" +DPKG_ARCH[vardepvalue] = "${DPKG_ARCH}" PKGWRITEDIRDEB = "${WORKDIR}/deploy-debs" APTCONF_TARGET = "${WORKDIR}" -APT_ARGS = "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS", True) == "1"]}" - +APT_ARGS = "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" + +def debian_arch_map(arch, tune): + tune_features = tune.split() + if arch == "allarch": + return "all" + if arch in ["i586", "i686"]: + return "i386" + if arch == "x86_64": + if "mx32" in tune_features: + return "x32" + return "amd64" + if arch.startswith("mips"): + endian = ["el", ""]["bigendian" in tune_features] + if "n64" in tune_features: + return "mips64" + endian + if "n32" in tune_features: + return "mipsn32" + endian + return "mips" + endian + if arch == "powerpc": + return arch + ["", "spe"]["spe" in tune_features] + if arch == "aarch64": + return "arm64" + if arch == "arm": + return arch + ["el", "hf"]["callconvention-hard" in tune_features] + return arch # # install a bunch of packages using apt # the following shell variables needs to be set before calling this func: @@ -21,7 +46,7 @@ APT_ARGS = "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS", Tr # INSTALL_BASEARCH_DEB - install base architecutre # INSTALL_ARCHS_DEB - list of available archs # INSTALL_PACKAGES_NORMAL_DEB - packages to be installed -# INSTALL_PACKAGES_ATTEMPTONLY_DEB - packages attemped to be installed only +# INSTALL_PACKAGES_ATTEMPTONLY_DEB - packages attempted to be installed only # INSTALL_PACKAGES_LINGUAS_DEB - additional packages for uclibc # INSTALL_TASK_DEB - task name @@ -29,23 +54,27 @@ python do_package_deb () { import re, copy import textwrap import subprocess + import collections + import codecs + + oldcwd = os.getcwd() - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') if not workdir: bb.error("WORKDIR not defined, unable to package") return - outdir = d.getVar('PKGWRITEDIRDEB', True) + outdir = d.getVar('PKGWRITEDIRDEB') if not outdir: bb.error("PKGWRITEDIRDEB not defined, unable to package") return - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') if not packages: bb.debug(1, "PACKAGES not defined, nothing to package") return - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') if os.access(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN"),os.R_OK): os.unlink(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN")) @@ -54,7 +83,7 @@ python do_package_deb () { bb.debug(1, "No packages; nothing to do") return - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') def cleanupcontrol(root): for p in ['CONTROL', 'DEBIAN']: @@ -70,17 +99,16 @@ python do_package_deb () { localdata.setVar('ROOT', '') localdata.setVar('ROOT_%s' % pkg, root) - pkgname = localdata.getVar('PKG_%s' % pkg, True) + pkgname = localdata.getVar('PKG_%s' % pkg) if not pkgname: pkgname = pkg localdata.setVar('PKG', pkgname) localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) - bb.data.update_data(localdata) basedir = os.path.join(os.path.dirname(root)) - pkgoutdir = os.path.join(outdir, localdata.getVar('PACKAGE_ARCH', True)) + pkgoutdir = os.path.join(outdir, localdata.getVar('PACKAGE_ARCH')) bb.utils.mkdirhier(pkgoutdir) os.chdir(root) @@ -88,22 +116,18 @@ python do_package_deb () { from glob import glob g = glob('*') if not g and localdata.getVar('ALLOW_EMPTY', False) != "1": - bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV', True), localdata.getVar('PKGR', True))) + bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV'), localdata.getVar('PKGR'))) bb.utils.unlockfile(lf) continue controldir = os.path.join(root, 'DEBIAN') bb.utils.mkdirhier(controldir) - os.chmod(controldir, 0755) - try: - import codecs - ctrlfile = codecs.open(os.path.join(controldir, 'control'), 'w', 'utf-8') - except OSError: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("unable to open control file for writing.") + os.chmod(controldir, 0o755) + + ctrlfile = codecs.open(os.path.join(controldir, 'control'), 'w', 'utf-8') fields = [] - pe = d.getVar('PKGE', True) + pe = d.getVar('PKGE') if pe and int(pe) > 0: fields.append(["Version: %s:%s-%s\n", ['PKGE', 'PKGV', 'PKGR']]) else: @@ -115,7 +139,7 @@ python do_package_deb () { fields.append(["Architecture: %s\n", ['DPKG_ARCH']]) fields.append(["OE: %s\n", ['PN']]) fields.append(["PackageArch: %s\n", ['PACKAGE_ARCH']]) - if d.getVar('HOMEPAGE', True): + if d.getVar('HOMEPAGE'): fields.append(["Homepage: %s\n", ['HOMEPAGE']]) # Package, Version, Maintainer, Description - mandatory @@ -125,10 +149,10 @@ python do_package_deb () { def pullData(l, d): l2 = [] for i in l: - data = d.getVar(i, True) + data = d.getVar(i) if data is None: - raise KeyError(f) - if i == 'DPKG_ARCH' and d.getVar('PACKAGE_ARCH', True) == 'all': + raise KeyError(i) + if i == 'DPKG_ARCH' and d.getVar('PACKAGE_ARCH') == 'all': data = 'all' elif i == 'PACKAGE_ARCH' or i == 'DPKG_ARCH': # The params in deb package control don't allow character @@ -139,47 +163,32 @@ python do_package_deb () { return l2 ctrlfile.write("Package: %s\n" % pkgname) + if d.getVar('PACKAGE_ARCH') == "all": + ctrlfile.write("Multi-Arch: foreign\n") # check for required fields - try: - for (c, fs) in fields: - for f in fs: - if localdata.getVar(f, False) is None: - raise KeyError(f) - # Special behavior for description... - if 'DESCRIPTION' in fs: - summary = localdata.getVar('SUMMARY', True) or localdata.getVar('DESCRIPTION', True) or "." - ctrlfile.write('Description: %s\n' % unicode(summary,'utf-8')) - description = localdata.getVar('DESCRIPTION', True) or "." - description = textwrap.dedent(description).strip() - if '\\n' in description: - # Manually indent - for t in description.split('\\n'): - # We don't limit the width when manually indent, but we do - # need the textwrap.fill() to set the initial_indent and - # subsequent_indent, so set a large width - ctrlfile.write('%s\n' % unicode(textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' '),'utf-8')) - else: - # Auto indent - ctrlfile.write('%s\n' % unicode(textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' '),'utf-8')) - - else: - ctrlfile.write(unicode(c % tuple(pullData(fs, localdata)),'utf-8')) - except KeyError: - import sys - (type, value, traceback) = sys.exc_info() - bb.utils.unlockfile(lf) - ctrlfile.close() - raise bb.build.FuncFailed("Missing field for deb generation: %s" % value) - except UnicodeDecodeError: - bb.utils.unlockfile(lf) - ctrlfile.close() - raise bb.build.FuncFailed("Non UTF-8 characters found in one of the fields") + for (c, fs) in fields: + # Special behavior for description... + if 'DESCRIPTION' in fs: + summary = localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or "." + ctrlfile.write('Description: %s\n' % summary) + description = localdata.getVar('DESCRIPTION') or "." + description = textwrap.dedent(description).strip() + if '\\n' in description: + # Manually indent + for t in description.split('\\n'): + ctrlfile.write(' %s\n' % (t.strip() or '.')) + else: + # Auto indent + ctrlfile.write('%s\n' % textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' ')) + + else: + ctrlfile.write(c % tuple(pullData(fs, localdata))) # more fields custom_fields_chunk = get_package_additional_metadata("deb", localdata) - if custom_fields_chunk is not None: - ctrlfile.write(unicode(custom_fields_chunk)) + if custom_fields_chunk: + ctrlfile.write(custom_fields_chunk) ctrlfile.write("\n") mapping_rename_hook(localdata) @@ -207,49 +216,53 @@ python do_package_deb () { elif (v or "").startswith("> "): var[dep][i] = var[dep][i].replace("> ", ">> ") - rdepends = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS", True) or "") + rdepends = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") debian_cmp_remap(rdepends) - for dep in rdepends: + for dep in list(rdepends.keys()): + if dep == pkg: + del rdepends[dep] + continue if '*' in dep: del rdepends[dep] - rrecommends = bb.utils.explode_dep_versions2(localdata.getVar("RRECOMMENDS", True) or "") + rrecommends = bb.utils.explode_dep_versions2(localdata.getVar("RRECOMMENDS") or "") debian_cmp_remap(rrecommends) - for dep in rrecommends: + for dep in list(rrecommends.keys()): if '*' in dep: del rrecommends[dep] - rsuggests = bb.utils.explode_dep_versions2(localdata.getVar("RSUGGESTS", True) or "") + rsuggests = bb.utils.explode_dep_versions2(localdata.getVar("RSUGGESTS") or "") debian_cmp_remap(rsuggests) # Deliberately drop version information here, not wanted/supported by deb - rprovides = dict.fromkeys(bb.utils.explode_dep_versions2(localdata.getVar("RPROVIDES", True) or ""), []) + rprovides = dict.fromkeys(bb.utils.explode_dep_versions2(localdata.getVar("RPROVIDES") or ""), []) + # Remove file paths if any from rprovides, debian does not support custom providers + for key in list(rprovides.keys()): + if key.startswith('/'): + del rprovides[key] + rprovides = collections.OrderedDict(sorted(rprovides.items(), key=lambda x: x[0])) debian_cmp_remap(rprovides) - rreplaces = bb.utils.explode_dep_versions2(localdata.getVar("RREPLACES", True) or "") + rreplaces = bb.utils.explode_dep_versions2(localdata.getVar("RREPLACES") or "") debian_cmp_remap(rreplaces) - rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS", True) or "") + rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS") or "") debian_cmp_remap(rconflicts) if rdepends: - ctrlfile.write("Depends: %s\n" % unicode(bb.utils.join_deps(rdepends))) + ctrlfile.write("Depends: %s\n" % bb.utils.join_deps(rdepends)) if rsuggests: - ctrlfile.write("Suggests: %s\n" % unicode(bb.utils.join_deps(rsuggests))) + ctrlfile.write("Suggests: %s\n" % bb.utils.join_deps(rsuggests)) if rrecommends: - ctrlfile.write("Recommends: %s\n" % unicode(bb.utils.join_deps(rrecommends))) + ctrlfile.write("Recommends: %s\n" % bb.utils.join_deps(rrecommends)) if rprovides: - ctrlfile.write("Provides: %s\n" % unicode(bb.utils.join_deps(rprovides))) + ctrlfile.write("Provides: %s\n" % bb.utils.join_deps(rprovides)) if rreplaces: - ctrlfile.write("Replaces: %s\n" % unicode(bb.utils.join_deps(rreplaces))) + ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces)) if rconflicts: - ctrlfile.write("Conflicts: %s\n" % unicode(bb.utils.join_deps(rconflicts))) + ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts)) ctrlfile.close() for script in ["preinst", "postinst", "prerm", "postrm"]: - scriptvar = localdata.getVar('pkg_%s' % script, True) + scriptvar = localdata.getVar('pkg_%s' % script) if not scriptvar: continue scriptvar = scriptvar.strip() - try: - scriptfile = open(os.path.join(controldir, script), 'w') - except OSError: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("unable to open %s script file for writing." % script) + scriptfile = open(os.path.join(controldir, script), 'w') if scriptvar.startswith("#!"): pos = scriptvar.find("\n") + 1 @@ -265,29 +278,25 @@ python do_package_deb () { scriptfile.write(scriptvar[pos:]) scriptfile.write('\n') scriptfile.close() - os.chmod(os.path.join(controldir, script), 0755) + os.chmod(os.path.join(controldir, script), 0o755) conffiles_str = ' '.join(get_conffiles(pkg, d)) if conffiles_str: - try: - conffiles = open(os.path.join(controldir, 'conffiles'), 'w') - except OSError: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("unable to open conffiles for writing.") + conffiles = open(os.path.join(controldir, 'conffiles'), 'w') for f in conffiles_str.split(): if os.path.exists(oe.path.join(root, f)): conffiles.write('%s\n' % f) conffiles.close() os.chdir(basedir) - ret = subprocess.call("PATH=\"%s\" dpkg-deb -b %s %s" % (localdata.getVar("PATH", True), root, pkgoutdir), shell=True) - if ret != 0: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("dpkg-deb execution failed") + subprocess.check_output("PATH=\"%s\" dpkg-deb -b %s %s" % (localdata.getVar("PATH"), root, pkgoutdir), shell=True) cleanupcontrol(root) bb.utils.unlockfile(lf) + os.chdir(oldcwd) } +# Indirect references to these vars +do_package_write_deb[vardeps] += "PKGV PKGR PKGV DESCRIPTION SECTION PRIORITY MAINTAINER DPKG_ARCH PN HOMEPAGE" # Otherwise allarch packages may change depending on override configuration do_package_deb[vardepsexclude] = "OVERRIDES" @@ -297,7 +306,7 @@ do_package_write_deb[sstate-inputdirs] = "${PKGWRITEDIRDEB}" do_package_write_deb[sstate-outputdirs] = "${DEPLOY_DIR_DEB}" python do_package_write_deb_setscene () { - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') if os.access(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN"),os.R_OK): os.unlink(os.path.join(tmpdir, "stamps", "DEB_PACKAGE_INDEX_CLEAN")) @@ -307,19 +316,10 @@ python do_package_write_deb_setscene () { addtask do_package_write_deb_setscene python () { - if d.getVar('PACKAGES', True) != '': + if d.getVar('PACKAGES') != '': deps = ' dpkg-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot' d.appendVarFlag('do_package_write_deb', 'depends', deps) d.setVarFlag('do_package_write_deb', 'fakeroot', "1") - - # Map TARGET_ARCH to Debian's ideas about architectures - darch = d.getVar('DPKG_ARCH', True) - if darch in ["x86", "i486", "i586", "i686", "pentium"]: - d.setVar('DPKG_ARCH', 'i386') - elif darch == "x86_64": - d.setVar('DPKG_ARCH', 'amd64') - elif darch == "arm": - d.setVar('DPKG_ARCH', 'armel') } python do_package_write_deb () { @@ -329,6 +329,7 @@ python do_package_write_deb () { do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" do_package_write_deb[umask] = "022" +do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" addtask package_write_deb after do_packagedata do_package diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass index 4dd7a7e5c9..c7cec9d63d 100644 --- a/meta/classes/package_ipk.bbclass +++ b/meta/classes/package_ipk.bbclass @@ -10,9 +10,9 @@ PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages OPKGBUILDCMD ??= "opkg-build" -OPKG_ARGS = "--force_postinstall --prefer-arch-to-version" -OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS", True) == "1"]}" -OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKAGE_EXCLUDE', True) or "").split())][(d.getVar("PACKAGE_EXCLUDE", True) or "") != ""]}" +OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" +OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" +OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKAGE_EXCLUDE') or "").split())][(d.getVar("PACKAGE_EXCLUDE") or "") != ""]}" OPKGLIBDIR = "${localstatedir}/lib" @@ -20,16 +20,19 @@ python do_package_ipk () { import re, copy import textwrap import subprocess + import collections - workdir = d.getVar('WORKDIR', True) - outdir = d.getVar('PKGWRITEDIRIPK', True) - tmpdir = d.getVar('TMPDIR', True) - pkgdest = d.getVar('PKGDEST', True) + oldcwd = os.getcwd() + + workdir = d.getVar('WORKDIR') + outdir = d.getVar('PKGWRITEDIRIPK') + tmpdir = d.getVar('TMPDIR') + pkgdest = d.getVar('PKGDEST') if not workdir or not outdir or not tmpdir: bb.error("Variables incorrectly set, unable to package") return - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') if not packages or packages == '': bb.debug(1, "No packages; nothing to do") return @@ -53,16 +56,15 @@ python do_package_ipk () { localdata.setVar('ROOT', '') localdata.setVar('ROOT_%s' % pkg, root) - pkgname = localdata.getVar('PKG_%s' % pkg, True) + pkgname = localdata.getVar('PKG_%s' % pkg) if not pkgname: pkgname = pkg localdata.setVar('PKG', pkgname) localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) - bb.data.update_data(localdata) basedir = os.path.join(os.path.dirname(root)) - arch = localdata.getVar('PACKAGE_ARCH', True) + arch = localdata.getVar('PACKAGE_ARCH') if localdata.getVar('IPK_HIERARCHICAL_FEED', False) == "1": # Spread packages across subdirectories so each isn't too crowded @@ -95,20 +97,16 @@ python do_package_ipk () { from glob import glob g = glob('*') if not g and localdata.getVar('ALLOW_EMPTY', False) != "1": - bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV', True), localdata.getVar('PKGR', True))) + bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV'), localdata.getVar('PKGR'))) bb.utils.unlockfile(lf) continue controldir = os.path.join(root, 'CONTROL') bb.utils.mkdirhier(controldir) - try: - ctrlfile = open(os.path.join(controldir, 'control'), 'w') - except OSError: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("unable to open control file for writing.") + ctrlfile = open(os.path.join(controldir, 'control'), 'w') fields = [] - pe = d.getVar('PKGE', True) + pe = d.getVar('PKGE') if pe and int(pe) > 0: fields.append(["Version: %s:%s-%s\n", ['PKGE', 'PKGV', 'PKGR']]) else: @@ -120,46 +118,43 @@ python do_package_ipk () { fields.append(["License: %s\n", ['LICENSE']]) fields.append(["Architecture: %s\n", ['PACKAGE_ARCH']]) fields.append(["OE: %s\n", ['PN']]) - if d.getVar('HOMEPAGE', True): + if d.getVar('HOMEPAGE'): fields.append(["Homepage: %s\n", ['HOMEPAGE']]) def pullData(l, d): l2 = [] for i in l: - l2.append(d.getVar(i, True)) + l2.append(d.getVar(i)) return l2 ctrlfile.write("Package: %s\n" % pkgname) # check for required fields - try: - for (c, fs) in fields: - for f in fs: - if localdata.getVar(f, False) is None: - raise KeyError(f) - # Special behavior for description... - if 'DESCRIPTION' in fs: - summary = localdata.getVar('SUMMARY', True) or localdata.getVar('DESCRIPTION', True) or "." - ctrlfile.write('Description: %s\n' % summary) - description = localdata.getVar('DESCRIPTION', True) or "." - description = textwrap.dedent(description).strip() - if '\\n' in description: - # Manually indent - for t in description.split('\\n'): - # We don't limit the width when manually indent, but we do - # need the textwrap.fill() to set the initial_indent and - # subsequent_indent, so set a large width - ctrlfile.write('%s\n' % textwrap.fill(t.strip(), width=100000, initial_indent=' ', subsequent_indent=' ')) - else: - # Auto indent - ctrlfile.write('%s\n' % textwrap.fill(description, width=74, initial_indent=' ', subsequent_indent=' ')) + for (c, fs) in fields: + for f in fs: + if localdata.getVar(f, False) is None: + raise KeyError(f) + # Special behavior for description... + if 'DESCRIPTION' in fs: + summary = localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or "." + ctrlfile.write('Description: %s\n' % summary) + description = localdata.getVar('DESCRIPTION') or "." + description = textwrap.dedent(description).strip() + if '\\n' in description: + # Manually indent + for t in description.split('\\n'): + # We don't limit the width when manually indent, but we do + # need the textwrap.fill() to set the initial_indent and + # subsequent_indent, so set a large width + line = textwrap.fill(t.strip(), + width=100000, + initial_indent=' ', + subsequent_indent=' ') or '.' + ctrlfile.write('%s\n' % line) else: - ctrlfile.write(c % tuple(pullData(fs, localdata))) - except KeyError: - import sys - (type, value, traceback) = sys.exc_info() - ctrlfile.close() - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("Missing field for ipk generation: %s" % value) + # Auto indent + ctrlfile.write('%s\n' % textwrap.fill(description, width=74, initial_indent=' ', subsequent_indent=' ')) + else: + ctrlfile.write(c % tuple(pullData(fs, localdata))) # more fields custom_fields_chunk = get_package_additional_metadata("ipk", localdata) @@ -182,18 +177,19 @@ python do_package_ipk () { elif (v or "").startswith("> "): var[dep][i] = var[dep][i].replace("> ", ">> ") - rdepends = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS", True) or "") + rdepends = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") debian_cmp_remap(rdepends) - rrecommends = bb.utils.explode_dep_versions2(localdata.getVar("RRECOMMENDS", True) or "") + rrecommends = bb.utils.explode_dep_versions2(localdata.getVar("RRECOMMENDS") or "") debian_cmp_remap(rrecommends) - rsuggests = bb.utils.explode_dep_versions2(localdata.getVar("RSUGGESTS", True) or "") + rsuggests = bb.utils.explode_dep_versions2(localdata.getVar("RSUGGESTS") or "") debian_cmp_remap(rsuggests) # Deliberately drop version information here, not wanted/supported by ipk - rprovides = dict.fromkeys(bb.utils.explode_dep_versions2(localdata.getVar("RPROVIDES", True) or ""), []) + rprovides = dict.fromkeys(bb.utils.explode_dep_versions2(localdata.getVar("RPROVIDES") or ""), []) + rprovides = collections.OrderedDict(sorted(rprovides.items(), key=lambda x: x[0])) debian_cmp_remap(rprovides) - rreplaces = bb.utils.explode_dep_versions2(localdata.getVar("RREPLACES", True) or "") + rreplaces = bb.utils.explode_dep_versions2(localdata.getVar("RREPLACES") or "") debian_cmp_remap(rreplaces) - rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS", True) or "") + rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS") or "") debian_cmp_remap(rconflicts) if rdepends: @@ -208,47 +204,42 @@ python do_package_ipk () { ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces)) if rconflicts: ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts)) - src_uri = localdata.getVar("SRC_URI", True).strip() or "None" + src_uri = localdata.getVar("SRC_URI").strip() or "None" if src_uri: src_uri = re.sub("\s+", " ", src_uri) ctrlfile.write("Source: %s\n" % " ".join(src_uri.split())) ctrlfile.close() for script in ["preinst", "postinst", "prerm", "postrm"]: - scriptvar = localdata.getVar('pkg_%s' % script, True) + scriptvar = localdata.getVar('pkg_%s' % script) if not scriptvar: continue - try: - scriptfile = open(os.path.join(controldir, script), 'w') - except OSError: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("unable to open %s script file for writing." % script) + scriptfile = open(os.path.join(controldir, script), 'w') scriptfile.write(scriptvar) scriptfile.close() - os.chmod(os.path.join(controldir, script), 0755) + os.chmod(os.path.join(controldir, script), 0o755) conffiles_str = ' '.join(get_conffiles(pkg, d)) if conffiles_str: - try: - conffiles = open(os.path.join(controldir, 'conffiles'), 'w') - except OSError: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("unable to open conffiles for writing.") + conffiles = open(os.path.join(controldir, 'conffiles'), 'w') for f in conffiles_str.split(): if os.path.exists(oe.path.join(root, f)): conffiles.write('%s\n' % f) conffiles.close() os.chdir(basedir) - ret = subprocess.call("PATH=\"%s\" %s %s %s" % (localdata.getVar("PATH", True), - d.getVar("OPKGBUILDCMD",1), pkg, pkgoutdir), shell=True) - if ret != 0: - bb.utils.unlockfile(lf) - raise bb.build.FuncFailed("opkg-build execution failed") + subprocess.check_output("PATH=\"%s\" %s %s %s" % (localdata.getVar("PATH"), + d.getVar("OPKGBUILDCMD"), pkg, pkgoutdir), shell=True) + + if d.getVar('IPK_SIGN_PACKAGES') == '1': + ipkver = "%s-%s" % (d.getVar('PKGV'), d.getVar('PKGR')) + ipk_to_sign = "%s/%s_%s_%s.ipk" % (pkgoutdir, pkgname, ipkver, d.getVar('PACKAGE_ARCH')) + sign_ipk(d, ipk_to_sign) cleanupcontrol(root) bb.utils.unlockfile(lf) + os.chdir(oldcwd) } # Otherwise allarch packages may change depending on override configuration do_package_ipk[vardepsexclude] = "OVERRIDES" @@ -258,7 +249,7 @@ do_package_write_ipk[sstate-inputdirs] = "${PKGWRITEDIRIPK}" do_package_write_ipk[sstate-outputdirs] = "${DEPLOY_DIR_IPK}" python do_package_write_ipk_setscene () { - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') if os.access(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"), os.R_OK): os.unlink(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN")) @@ -268,7 +259,7 @@ python do_package_write_ipk_setscene () { addtask do_package_write_ipk_setscene python () { - if d.getVar('PACKAGES', True) != '': + if d.getVar('PACKAGES') != '': deps = ' opkg-utils-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot' d.appendVarFlag('do_package_write_ipk', 'depends', deps) d.setVarFlag('do_package_write_ipk', 'fakeroot', "1") @@ -281,6 +272,7 @@ python do_package_write_ipk () { do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" do_package_write_ipk[umask] = "022" +do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" addtask package_write_ipk after do_packagedata do_package PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass index 1fa1634d29..084546f733 100644 --- a/meta/classes/package_rpm.bbclass +++ b/meta/classes/package_rpm.bbclass @@ -13,9 +13,9 @@ MERGEPERFILEDEPS = "1" # Construct per file dependencies file def write_rpm_perfiledata(srcname, d): - workdir = d.getVar('WORKDIR', True) - packages = d.getVar('PACKAGES', True) - pkgd = d.getVar('PKGD', True) + workdir = d.getVar('WORKDIR') + packages = d.getVar('PACKAGES') + pkgd = d.getVar('PKGD') def dump_filerdeps(varname, outfile, d): outfile.write("#!/usr/bin/env python\n\n") @@ -23,10 +23,10 @@ def write_rpm_perfiledata(srcname, d): outfile.write('deps = {\n') for pkg in packages.split(): dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg - dependsflist = (d.getVar(dependsflist_key, True) or "") + dependsflist = (d.getVar(dependsflist_key) or "") for dfile in dependsflist.split(): key = "FILE" + varname + "_" + dfile + "_" + pkg - depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "") + depends_dict = bb.utils.explode_dep_versions(d.getVar(key) or "") file = dfile.replace("@underscore@", "_") file = file.replace("@closebrace@", "]") file = file.replace("@openbrace@", "[") @@ -55,28 +55,22 @@ def write_rpm_perfiledata(srcname, d): # OE-core dependencies a.k.a. RPM requires outdepends = workdir + "/" + srcname + ".requires" - try: - dependsfile = open(outdepends, 'w') - except OSError: - raise bb.build.FuncFailed("unable to open spec file for writing.") + dependsfile = open(outdepends, 'w') dump_filerdeps('RDEPENDS', dependsfile, d) dependsfile.close() - os.chmod(outdepends, 0755) + os.chmod(outdepends, 0o755) # OE-core / RPM Provides outprovides = workdir + "/" + srcname + ".provides" - try: - providesfile = open(outprovides, 'w') - except OSError: - raise bb.build.FuncFailed("unable to open spec file for writing.") + providesfile = open(outprovides, 'w') dump_filerdeps('RPROVIDES', providesfile, d) providesfile.close() - os.chmod(outprovides, 0755) + os.chmod(outprovides, 0o755) return (outdepends, outprovides) @@ -86,15 +80,15 @@ python write_specfile () { # append information for logs and patches to %prep def add_prep(d,spec_files_bottom): - if d.getVarFlag('ARCHIVER_MODE', 'srpm', True) == '1' and bb.data.inherits_class('archiver', d): - spec_files_bottom.append('%%prep -n %s' % d.getVar('PN', True) ) + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): + spec_files_bottom.append('%%prep -n %s' % d.getVar('PN') ) spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"") spec_files_bottom.append('') # append the name of tarball to key word 'SOURCE' in xxx.spec. def tail_source(d): - if d.getVarFlag('ARCHIVER_MODE', 'srpm', True) == '1' and bb.data.inherits_class('archiver', d): - ar_outdir = d.getVar('ARCHIVER_OUTDIR', True) + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): + ar_outdir = d.getVar('ARCHIVER_OUTDIR') if not os.path.exists(ar_outdir): return source_list = os.listdir(ar_outdir) @@ -107,27 +101,6 @@ python write_specfile () { os.chown(f, 0, 0) spec_preamble_top.append('Source%s: %s' % (source_number, source)) source_number += 1 - # We need a simple way to remove the MLPREFIX from the package name, - # and dependency information... - def strip_multilib(name, d): - multilibs = d.getVar('MULTILIBS', True) or "" - for ext in multilibs.split(): - eext = ext.split(':') - if len(eext) > 1 and eext[0] == 'multilib' and name and name.find(eext[1] + '-') >= 0: - name = "".join(name.split(eext[1] + '-')) - return name - - def strip_multilib_deps(deps, d): - depends = bb.utils.explode_dep_versions2(deps or "") - newdeps = {} - for dep in depends: - newdeps[strip_multilib(dep, d)] = depends[dep] - return bb.utils.join_deps(newdeps) - -# ml = d.getVar("MLPREFIX", True) -# if ml and name and len(ml) != 0 and name.find(ml) == 0: -# return ml.join(name.split(ml, 1)[1:]) -# return name # In RPM, dependencies are of the format: pkg <>= Epoch:Version-Release # This format is similar to OE, however there are restrictions on the @@ -144,7 +117,7 @@ python write_specfile () { # after renaming we cannot look up the dependencies in the packagedata # store. def translate_vers(varname, d): - depends = d.getVar(varname, True) + depends = d.getVar(varname) if depends: depends_dict = bb.utils.explode_dep_versions2(depends) newdeps_dict = {} @@ -197,6 +170,8 @@ python write_specfile () { if path.endswith("DEBIAN") or path.endswith("CONTROL"): continue path = path.replace("%", "%%%%%%%%") + path = path.replace("[", "?") + path = path.replace("]", "?") # Treat all symlinks to directories as normal files. # os.walk() lists them as directories. @@ -216,6 +191,8 @@ python write_specfile () { if dir == "CONTROL" or dir == "DEBIAN": continue dir = dir.replace("%", "%%%%%%%%") + dir = dir.replace("[", "?") + dir = dir.replace("]", "?") # All packages own the directories their files are in... target.append('%dir "' + path + '/' + dir + '"') else: @@ -230,6 +207,8 @@ python write_specfile () { if file == "CONTROL" or file == "DEBIAN": continue file = file.replace("%", "%%%%%%%%") + file = file.replace("[", "?") + file = file.replace("]", "?") if conffiles.count(path + '/' + file): target.append('%config "' + path + '/' + file + '"') else: @@ -248,10 +227,10 @@ python write_specfile () { def get_perfile(varname, pkg, d): deps = [] dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg - dependsflist = (d.getVar(dependsflist_key, True) or "") + dependsflist = (d.getVar(dependsflist_key) or "") for dfile in dependsflist.split(): key = "FILE" + varname + "_" + dfile + "_" + pkg - depends = d.getVar(key, True) + depends = d.getVar(key) if depends: deps.append(depends) return " ".join(deps) @@ -269,33 +248,33 @@ python write_specfile () { else: spec_preamble.append('%s' % textwrap.fill(dedent_text, width=75)) - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') if not packages or packages == '': bb.debug(1, "No packages; nothing to do") return - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') if not pkgdest: bb.fatal("No PKGDEST") - outspecfile = d.getVar('OUTSPECFILE', True) + outspecfile = d.getVar('OUTSPECFILE') if not outspecfile: bb.fatal("No OUTSPECFILE") # Construct the SPEC file... - srcname = strip_multilib(d.getVar('PN', True), d) - srcsummary = (d.getVar('SUMMARY', True) or d.getVar('DESCRIPTION', True) or ".") - srcversion = d.getVar('PKGV', True).replace('-', '+') - srcrelease = d.getVar('PKGR', True) - srcepoch = (d.getVar('PKGE', True) or "") - srclicense = d.getVar('LICENSE', True) - srcsection = d.getVar('SECTION', True) - srcmaintainer = d.getVar('MAINTAINER', True) - srchomepage = d.getVar('HOMEPAGE', True) - srcdescription = d.getVar('DESCRIPTION', True) or "." + srcname = d.getVar('PN') + srcsummary = (d.getVar('SUMMARY') or d.getVar('DESCRIPTION') or ".") + srcversion = d.getVar('PKGV').replace('-', '+') + srcrelease = d.getVar('PKGR') + srcepoch = (d.getVar('PKGE') or "") + srclicense = d.getVar('LICENSE') + srcsection = d.getVar('SECTION') + srcmaintainer = d.getVar('MAINTAINER') + srchomepage = d.getVar('HOMEPAGE') + srcdescription = d.getVar('DESCRIPTION') or "." srccustomtagschunk = get_package_additional_metadata("rpm", d) - srcdepends = strip_multilib_deps(d.getVar('DEPENDS', True), d) + srcdepends = d.getVar('DEPENDS') srcrdepends = [] srcrrecommends = [] srcrsuggests = [] @@ -318,8 +297,8 @@ python write_specfile () { spec_files_top = [] spec_files_bottom = [] - perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0" - extra_pkgdata = (d.getVar("RPM_EXTRA_PKGDATA", True) or "0") == "1" + perfiledeps = (d.getVar("MERGEPERFILEDEPS") or "0") == "0" + extra_pkgdata = (d.getVar("RPM_EXTRA_PKGDATA") or "0") == "1" for pkg in packages.split(): localdata = bb.data.createCopy(d) @@ -328,29 +307,27 @@ python write_specfile () { localdata.setVar('ROOT', '') localdata.setVar('ROOT_%s' % pkg, root) - pkgname = localdata.getVar('PKG_%s' % pkg, True) + pkgname = localdata.getVar('PKG_%s' % pkg) if not pkgname: pkgname = pkg localdata.setVar('PKG', pkgname) localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) - bb.data.update_data(localdata) - conffiles = get_conffiles(pkg, d) - dirfiles = localdata.getVar('DIRFILES', True) + dirfiles = localdata.getVar('DIRFILES') if dirfiles is not None: dirfiles = dirfiles.split() - splitname = strip_multilib(pkgname, d) + splitname = pkgname - splitsummary = (localdata.getVar('SUMMARY', True) or localdata.getVar('DESCRIPTION', True) or ".") - splitversion = (localdata.getVar('PKGV', True) or "").replace('-', '+') - splitrelease = (localdata.getVar('PKGR', True) or "") - splitepoch = (localdata.getVar('PKGE', True) or "") - splitlicense = (localdata.getVar('LICENSE', True) or "") - splitsection = (localdata.getVar('SECTION', True) or "") - splitdescription = (localdata.getVar('DESCRIPTION', True) or ".") + splitsummary = (localdata.getVar('SUMMARY') or localdata.getVar('DESCRIPTION') or ".") + splitversion = (localdata.getVar('PKGV') or "").replace('-', '+') + splitrelease = (localdata.getVar('PKGR') or "") + splitepoch = (localdata.getVar('PKGE') or "") + splitlicense = (localdata.getVar('LICENSE') or "") + splitsection = (localdata.getVar('SECTION') or "") + splitdescription = (localdata.getVar('DESCRIPTION') or ".") splitcustomtagschunk = get_package_additional_metadata("rpm", localdata) translate_vers('RDEPENDS', localdata) @@ -363,18 +340,18 @@ python write_specfile () { # Map the dependencies into their final form mapping_rename_hook(localdata) - splitrdepends = strip_multilib_deps(localdata.getVar('RDEPENDS', True), d) - splitrrecommends = strip_multilib_deps(localdata.getVar('RRECOMMENDS', True), d) - splitrsuggests = strip_multilib_deps(localdata.getVar('RSUGGESTS', True), d) - splitrprovides = strip_multilib_deps(localdata.getVar('RPROVIDES', True), d) - splitrreplaces = strip_multilib_deps(localdata.getVar('RREPLACES', True), d) - splitrconflicts = strip_multilib_deps(localdata.getVar('RCONFLICTS', True), d) + splitrdepends = localdata.getVar('RDEPENDS') + splitrrecommends = localdata.getVar('RRECOMMENDS') + splitrsuggests = localdata.getVar('RSUGGESTS') + splitrprovides = localdata.getVar('RPROVIDES') + splitrreplaces = localdata.getVar('RREPLACES') + splitrconflicts = localdata.getVar('RCONFLICTS') splitrobsoletes = [] - splitrpreinst = localdata.getVar('pkg_preinst', True) - splitrpostinst = localdata.getVar('pkg_postinst', True) - splitrprerm = localdata.getVar('pkg_prerm', True) - splitrpostrm = localdata.getVar('pkg_postrm', True) + splitrpreinst = localdata.getVar('pkg_preinst') + splitrpostinst = localdata.getVar('pkg_postinst') + splitrprerm = localdata.getVar('pkg_prerm') + splitrpostrm = localdata.getVar('pkg_postrm') if not perfiledeps: @@ -452,25 +429,10 @@ python write_specfile () { if splitrpostrm: print_deps(splitrdepends, "Requires(postun)", spec_preamble_bottom, d) - # Suggests in RPM are like recommends in OE-core! - print_deps(splitrrecommends, "Suggests", spec_preamble_bottom, d) - # While there is no analog for suggests... (So call them recommends for now) - print_deps(splitrsuggests, "Recommends", spec_preamble_bottom, d) + print_deps(splitrrecommends, "Recommends", spec_preamble_bottom, d) + print_deps(splitrsuggests, "Suggests", spec_preamble_bottom, d) print_deps(splitrprovides, "Provides", spec_preamble_bottom, d) print_deps(splitrobsoletes, "Obsoletes", spec_preamble_bottom, d) - - # conflicts can not be in a provide! We will need to filter it. - if splitrconflicts: - depends_dict = bb.utils.explode_dep_versions2(splitrconflicts) - newdeps_dict = {} - for dep in depends_dict: - if dep not in splitrprovides: - newdeps_dict[dep] = depends_dict[dep] - if newdeps_dict: - splitrconflicts = bb.utils.join_deps(newdeps_dict) - else: - splitrconflicts = "" - print_deps(splitrconflicts, "Conflicts", spec_preamble_bottom, d) spec_preamble_bottom.append('') @@ -562,25 +524,10 @@ python write_specfile () { if srcrpostrm: print_deps(srcrdepends, "Requires(postun)", spec_preamble_top, d) - # Suggests in RPM are like recommends in OE-core! - print_deps(srcrrecommends, "Suggests", spec_preamble_top, d) - # While there is no analog for suggests... (So call them recommends for now) - print_deps(srcrsuggests, "Recommends", spec_preamble_top, d) - print_deps(srcrprovides, "Provides", spec_preamble_top, d) + print_deps(srcrrecommends, "Recommends", spec_preamble_top, d) + print_deps(srcrsuggests, "Suggests", spec_preamble_top, d) + print_deps(srcrprovides + (" /bin/sh" if srcname.startswith("nativesdk-") else ""), "Provides", spec_preamble_top, d) print_deps(srcrobsoletes, "Obsoletes", spec_preamble_top, d) - - # conflicts can not be in a provide! We will need to filter it. - if srcrconflicts: - depends_dict = bb.utils.explode_dep_versions2(srcrconflicts) - newdeps_dict = {} - for dep in depends_dict: - if dep not in srcrprovides: - newdeps_dict[dep] = depends_dict[dep] - if newdeps_dict: - srcrconflicts = bb.utils.join_deps(newdeps_dict) - else: - srcrconflicts = "" - print_deps(srcrconflicts, "Conflicts", spec_preamble_top, d) spec_preamble_top.append('') @@ -614,14 +561,11 @@ python write_specfile () { spec_scriptlets_top.append('') # Write the SPEC file - try: - specfile = open(outspecfile, 'w') - except OSError: - raise bb.build.FuncFailed("unable to open spec file for writing.") + specfile = open(outspecfile, 'w') # RPMSPEC_PREAMBLE is a way to add arbitrary text to the top # of the generated spec file - external_preamble = d.getVar("RPMSPEC_PREAMBLE", True) + external_preamble = d.getVar("RPMSPEC_PREAMBLE") if external_preamble: specfile.write(external_preamble + "\n") @@ -649,23 +593,15 @@ python write_specfile () { write_specfile[vardepsexclude] = "OVERRIDES" python do_package_rpm () { - # We need a simple way to remove the MLPREFIX from the package name, - # and dependency information... - def strip_multilib(name, d): - ml = d.getVar("MLPREFIX", True) - if ml and name and len(ml) != 0 and name.find(ml) >= 0: - return "".join(name.split(ml)) - return name - - workdir = d.getVar('WORKDIR', True) - tmpdir = d.getVar('TMPDIR', True) - pkgd = d.getVar('PKGD', True) - pkgdest = d.getVar('PKGDEST', True) + workdir = d.getVar('WORKDIR') + tmpdir = d.getVar('TMPDIR') + pkgd = d.getVar('PKGD') + pkgdest = d.getVar('PKGDEST') if not workdir or not pkgd or not tmpdir: bb.error("Variables incorrectly set, unable to package") return - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') if not packages or packages == '': bb.debug(1, "No packages; nothing to do") return @@ -674,40 +610,37 @@ python do_package_rpm () { # If the spec file already exist, and has not been stored into # pseudo's files.db, it maybe cause rpmbuild src.rpm fail, # so remove it before doing rpmbuild src.rpm. - srcname = strip_multilib(d.getVar('PN', True), d) + srcname = d.getVar('PN') outspecfile = workdir + "/" + srcname + ".spec" if os.path.isfile(outspecfile): os.remove(outspecfile) d.setVar('OUTSPECFILE', outspecfile) bb.build.exec_func('write_specfile', d) - perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0" + perfiledeps = (d.getVar("MERGEPERFILEDEPS") or "0") == "0" if perfiledeps: outdepends, outprovides = write_rpm_perfiledata(srcname, d) # Setup the rpmbuild arguments... - rpmbuild = d.getVar('RPMBUILD', True) - targetsys = d.getVar('TARGET_SYS', True) - targetvendor = d.getVar('HOST_VENDOR', True) - package_arch = (d.getVar('PACKAGE_ARCH', True) or "").replace("-", "_") - sdkpkgsuffix = (d.getVar('SDKPKGSUFFIX', True) or "nativesdk").replace("-", "_") - if package_arch not in "all any noarch".split() and not package_arch.endswith(sdkpkgsuffix): - ml_prefix = (d.getVar('MLPREFIX', True) or "").replace("-", "_") - d.setVar('PACKAGE_ARCH_EXTEND', ml_prefix + package_arch) - else: - d.setVar('PACKAGE_ARCH_EXTEND', package_arch) + rpmbuild = d.getVar('RPMBUILD') + targetsys = d.getVar('TARGET_SYS') + targetvendor = d.getVar('HOST_VENDOR') + # Too many places in dnf stack assume that arch-independent packages are "noarch". + # Let's not fight against this. + package_arch = (d.getVar('PACKAGE_ARCH') or "").replace("-", "_").replace("all", "noarch") + sdkpkgsuffix = (d.getVar('SDKPKGSUFFIX') or "nativesdk").replace("-", "_") + d.setVar('PACKAGE_ARCH_EXTEND', package_arch) pkgwritedir = d.expand('${PKGWRITEDIRRPM}/${PACKAGE_ARCH_EXTEND}') d.setVar('RPM_PKGWRITEDIR', pkgwritedir) - bb.debug(1, 'PKGWRITEDIR: %s' % d.getVar('RPM_PKGWRITEDIR', True)) - pkgarch = d.expand('${PACKAGE_ARCH_EXTEND}${HOST_VENDOR}-${HOST_OS}') - magicfile = d.expand('${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc') + bb.debug(1, 'PKGWRITEDIR: %s' % d.getVar('RPM_PKGWRITEDIR')) + pkgarch = d.expand('${PACKAGE_ARCH_EXTEND}${HOST_VENDOR}-linux') bb.utils.mkdirhier(pkgwritedir) - os.chmod(pkgwritedir, 0755) + os.chmod(pkgwritedir, 0o755) cmd = rpmbuild - cmd = cmd + " --nodeps --short-circuit --target " + pkgarch + " --buildroot " + pkgd + cmd = cmd + " --noclean --nodeps --short-circuit --target " + pkgarch + " --buildroot " + pkgd cmd = cmd + " --define '_topdir " + workdir + "' --define '_rpmdir " + pkgwritedir + "'" - cmd = cmd + " --define '_builddir " + d.getVar('S', True) + "'" + cmd = cmd + " --define '_builddir " + d.getVar('S') + "'" cmd = cmd + " --define '_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm'" cmd = cmd + " --define '_use_internal_dependency_generator 0'" if perfiledeps: @@ -718,11 +651,10 @@ python do_package_rpm () { cmd = cmd + " --define '__find_provides %{nil}'" cmd = cmd + " --define '_unpackaged_files_terminate_build 0'" cmd = cmd + " --define 'debug_package %{nil}'" - cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'" cmd = cmd + " --define '_tmppath " + workdir + "'" - if d.getVarFlag('ARCHIVER_MODE', 'srpm', True) == '1' and bb.data.inherits_class('archiver', d): - cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR', True) + "'" - cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_OUTDIR', True) + "'" + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): + cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR') + "'" + cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_OUTDIR') + "'" cmdsrpm = cmdsrpm + " -bs " + outspecfile # Build the .src.rpm d.setVar('SBUILDSPEC', cmdsrpm + "\n") @@ -730,20 +662,23 @@ python do_package_rpm () { bb.build.exec_func('SBUILDSPEC', d) cmd = cmd + " -bb " + outspecfile + # rpm 4 creates various empty directories in _topdir, let's clean them up + cleanupcmd = "rm -rf %s/BUILDROOT %s/SOURCES %s/SPECS %s/SRPMS" % (workdir, workdir, workdir, workdir) + # Build the rpm package! - d.setVar('BUILDSPEC', cmd + "\n") + d.setVar('BUILDSPEC', cmd + "\n" + cleanupcmd + "\n") d.setVarFlag('BUILDSPEC', 'func', '1') bb.build.exec_func('BUILDSPEC', d) - if d.getVar('RPM_SIGN_PACKAGES', True) == '1': + if d.getVar('RPM_SIGN_PACKAGES') == '1': bb.build.exec_func("sign_rpm", d) } python () { - if d.getVar('PACKAGES', True) != '': + if d.getVar('PACKAGES') != '': deps = ' rpm-native:do_populate_sysroot virtual/fakeroot-native:do_populate_sysroot' d.appendVarFlag('do_package_write_rpm', 'depends', deps) - d.setVarFlag('do_package_write_rpm', 'fakeroot', 1) + d.setVarFlag('do_package_write_rpm', 'fakeroot', '1') } SSTATETASKS += "do_package_write_rpm" @@ -766,9 +701,10 @@ python do_package_write_rpm () { do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" do_package_write_rpm[umask] = "022" +do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" addtask package_write_rpm after do_packagedata do_package PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" -PACKAGEINDEXDEPS += "createrepo-native:do_populate_sysroot" +PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" do_build[recrdeptask] += "do_package_write_rpm" diff --git a/meta/classes/package_tar.bbclass b/meta/classes/package_tar.bbclass index f9e2292deb..ce3ab4c8e2 100644 --- a/meta/classes/package_tar.bbclass +++ b/meta/classes/package_tar.bbclass @@ -4,27 +4,30 @@ IMAGE_PKGTYPE ?= "tar" python do_package_tar () { import subprocess - workdir = d.getVar('WORKDIR', True) + + oldcwd = os.getcwd() + + workdir = d.getVar('WORKDIR') if not workdir: bb.error("WORKDIR not defined, unable to package") return - outdir = d.getVar('DEPLOY_DIR_TAR', True) + outdir = d.getVar('DEPLOY_DIR_TAR') if not outdir: bb.error("DEPLOY_DIR_TAR not defined, unable to package") return - dvar = d.getVar('D', True) + dvar = d.getVar('D') if not dvar: bb.error("D not defined, unable to package") return - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') if not packages: bb.debug(1, "PACKAGES not defined, nothing to package") return - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') bb.utils.mkdirhier(outdir) bb.utils.mkdirhier(dvar) @@ -35,7 +38,6 @@ python do_package_tar () { overrides = localdata.getVar('OVERRIDES', False) localdata.setVar('OVERRIDES', '%s:%s' % (overrides, pkg)) - bb.data.update_data(localdata) bb.utils.mkdirhier(root) basedir = os.path.dirname(root) @@ -43,16 +45,18 @@ python do_package_tar () { os.chdir(root) dlist = os.listdir(root) if not dlist: - bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV', True), localdata.getVar('PKGR', True))) + bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV'), localdata.getVar('PKGR'))) continue args = "tar -cz --exclude=CONTROL --exclude=DEBIAN -f".split() ret = subprocess.call(args + [tarfn] + dlist) if ret != 0: bb.error("Creation of tar %s failed." % tarfn) + + os.chdir(oldcwd) } python () { - if d.getVar('PACKAGES', True) != '': + if d.getVar('PACKAGES') != '': deps = (d.getVarFlag('do_package_write_tar', 'depends') or "").split() deps.append('tar-native:do_populate_sysroot') deps.append('virtual/fakeroot-native:do_populate_sysroot') diff --git a/meta/classes/packagedata.bbclass b/meta/classes/packagedata.bbclass index 3397f1e36b..a903e5cfd2 100644 --- a/meta/classes/packagedata.bbclass +++ b/meta/classes/packagedata.bbclass @@ -2,10 +2,10 @@ python read_subpackage_metadata () { import oe.packagedata vars = { - "PN" : d.getVar('PN', True), - "PE" : d.getVar('PE', True), - "PV" : d.getVar('PV', True), - "PR" : d.getVar('PR', True), + "PN" : d.getVar('PN'), + "PE" : d.getVar('PE'), + "PV" : d.getVar('PV'), + "PR" : d.getVar('PR'), } data = oe.packagedata.read_pkgdata(vars["PN"], d) @@ -13,7 +13,7 @@ python read_subpackage_metadata () { for key in data.keys(): d.setVar(key, data[key]) - for pkg in d.getVar('PACKAGES', True).split(): + for pkg in d.getVar('PACKAGES').split(): sdata = oe.packagedata.read_subpkgdata(pkg, d) for key in sdata.keys(): if key in vars: diff --git a/meta/classes/packagefeed-stability.bbclass b/meta/classes/packagefeed-stability.bbclass new file mode 100644 index 0000000000..c0e9be549d --- /dev/null +++ b/meta/classes/packagefeed-stability.bbclass @@ -0,0 +1,252 @@ +# Class to avoid copying packages into the feed if they haven't materially changed +# +# Copyright (C) 2015 Intel Corporation +# Released under the MIT license (see COPYING.MIT for details) +# +# This class effectively intercepts packages as they are written out by +# do_package_write_*, causing them to be written into a different +# directory where we can compare them to whatever older packages might +# be in the "real" package feed directory, and avoid copying the new +# package to the feed if it has not materially changed. The idea is to +# avoid unnecessary churn in the packages when dependencies trigger task +# reexecution (and thus repackaging). Enabling the class is simple: +# +# INHERIT += "packagefeed-stability" +# +# Caveats: +# 1) Latest PR values in the build system may not match those in packages +# seen on the target (naturally) +# 2) If you rebuild from sstate without the existing package feed present, +# you will lose the "state" of the package feed i.e. the preserved old +# package versions. Not the end of the world, but would negate the +# entire purpose of this class. +# +# Note that running -c cleanall on a recipe will purposely delete the old +# package files so they will definitely be copied the next time. + +python() { + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d): + return + # Package backend agnostic intercept + # This assumes that the package_write task is called package_write_<pkgtype> + # and that the directory in which packages should be written is + # pointed to by the variable DEPLOY_DIR_<PKGTYPE> + for pkgclass in (d.getVar('PACKAGE_CLASSES') or '').split(): + if pkgclass.startswith('package_'): + pkgtype = pkgclass.split('_', 1)[1] + pkgwritefunc = 'do_package_write_%s' % pkgtype + sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False) + deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper() + deploydirvarref = '${' + deploydirvar + '}' + pkgcomparefunc = 'do_package_compare_%s' % pkgtype + + if bb.data.inherits_class('image', d): + d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc) + + if bb.data.inherits_class('populate_sdk_base', d): + d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc) + + if bb.data.inherits_class('populate_sdk_ext', d): + d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc) + + d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc) + + if d.getVarFlag(pkgwritefunc, 'noexec') or not d.getVarFlag(pkgwritefunc, 'task'): + # Packaging is disabled for this recipe, we shouldn't do anything + continue + + if deploydirvarref in sstate_outputdirs: + deplor_dir_pkgtype = d.expand(deploydirvarref + '-prediff') + # Set intermediate output directory + d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deplor_dir_pkgtype)) + # Update SSTATE_DUPWHITELIST to avoid shared location conflicted error + d.appendVar('SSTATE_DUPWHITELIST', ' %s' % deplor_dir_pkgtype) + + d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False)) + d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False)) + d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot') + bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d) +} + +# This isn't the real task function - it's a template that we use in the +# anonymous python code above +fakeroot python do_package_compare () { + currenttask = d.getVar('BB_CURRENTTASK') + pkgtype = currenttask.rsplit('_', 1)[1] + package_compare_impl(pkgtype, d) +} + +def package_compare_impl(pkgtype, d): + import errno + import fnmatch + import glob + import subprocess + import oe.sstatesig + + pn = d.getVar('PN') + deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper()) + prepath = deploydir + '-prediff/' + + # Find out PKGR values are + pkgdatadir = d.getVar('PKGDATA_DIR') + packages = [] + try: + with open(os.path.join(pkgdatadir, pn), 'r') as f: + for line in f: + if line.startswith('PACKAGES:'): + packages = line.split(':', 1)[1].split() + break + except IOError as e: + if e.errno == errno.ENOENT: + pass + + if not packages: + bb.debug(2, '%s: no packages, nothing to do' % pn) + return + + pkgrvalues = {} + rpkgnames = {} + rdepends = {} + pkgvvalues = {} + for pkg in packages: + with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f: + for line in f: + if line.startswith('PKGR:'): + pkgrvalues[pkg] = line.split(':', 1)[1].strip() + if line.startswith('PKGV:'): + pkgvvalues[pkg] = line.split(':', 1)[1].strip() + elif line.startswith('PKG_%s:' % pkg): + rpkgnames[pkg] = line.split(':', 1)[1].strip() + elif line.startswith('RDEPENDS_%s:' % pkg): + rdepends[pkg] = line.split(':', 1)[1].strip() + + # Prepare a list of the runtime package names for packages that were + # actually produced + rpkglist = [] + for pkg, rpkg in rpkgnames.items(): + if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')): + rpkglist.append((rpkg, pkg)) + rpkglist.sort(key=lambda x: len(x[0]), reverse=True) + + pvu = d.getVar('PV', False) + if '$' + '{SRCPV}' in pvu: + pvprefix = pvu.split('$' + '{SRCPV}', 1)[0] + else: + pvprefix = None + + pkgwritetask = 'package_write_%s' % pkgtype + files = [] + docopy = False + manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d) + mlprefix = d.getVar('MLPREFIX') + # Copy recipe's all packages if one of the packages are different to make + # they have the same PR. + with open(manifest, 'r') as f: + for line in f: + if line.startswith(prepath): + srcpath = line.rstrip() + if os.path.isfile(srcpath): + destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath)) + + # This is crude but should work assuming the output + # package file name starts with the package name + # and rpkglist is sorted by length (descending) + pkgbasename = os.path.basename(destpath) + pkgname = None + for rpkg, pkg in rpkglist: + if mlprefix and pkgtype == 'rpm' and rpkg.startswith(mlprefix): + rpkg = rpkg[len(mlprefix):] + if pkgbasename.startswith(rpkg): + pkgr = pkgrvalues[pkg] + destpathspec = destpath.replace(pkgr, '*') + if pvprefix: + pkgv = pkgvvalues[pkg] + if pkgv.startswith(pvprefix): + pkgvsuffix = pkgv[len(pvprefix):] + if '+' in pkgvsuffix: + newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1] + destpathspec = destpathspec.replace(pkgv, newpkgv) + pkgname = pkg + break + else: + bb.warn('Unable to map %s back to package' % pkgbasename) + destpathspec = destpath + + oldfile = None + if not docopy: + oldfiles = glob.glob(destpathspec) + if oldfiles: + oldfile = oldfiles[-1] + result = subprocess.call(['pkg-diff.sh', oldfile, srcpath]) + if result != 0: + docopy = True + bb.note("%s and %s are different, will copy packages" % (oldfile, srcpath)) + else: + docopy = True + bb.note("No old packages found for %s, will copy packages" % pkgname) + + files.append((pkgname, pkgbasename, srcpath, destpath)) + + # Remove all the old files and copy again if docopy + if docopy: + bb.plain('Copying packages for recipe %s' % pn) + pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) + try: + with open(pcmanifest, 'r') as f: + for line in f: + fn = line.rstrip() + if fn: + try: + os.remove(fn) + bb.note('Removed old package %s' % fn) + except OSError as e: + if e.errno == errno.ENOENT: + pass + except IOError as e: + if e.errno == errno.ENOENT: + pass + + # Create new manifest + with open(pcmanifest, 'w') as f: + for pkgname, pkgbasename, srcpath, destpath in files: + destdir = os.path.dirname(destpath) + bb.utils.mkdirhier(destdir) + # Remove allarch rpm pkg if it is already existed (for + # multilib), they're identical in theory, but sstate.bbclass + # copies it again, so keep align with that. + if os.path.exists(destpath) and pkgtype == 'rpm' \ + and d.getVar('PACKAGE_ARCH') == 'all': + os.unlink(destpath) + if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev): + # Use a hard link to save space + os.link(srcpath, destpath) + else: + shutil.copyfile(srcpath, destpath) + f.write('%s\n' % destpath) + else: + bb.plain('Not copying packages for recipe %s' % pn) + +do_cleansstate[postfuncs] += "pfs_cleanpkgs" +python pfs_cleanpkgs () { + import errno + for pkgclass in (d.getVar('PACKAGE_CLASSES') or '').split(): + if pkgclass.startswith('package_'): + pkgtype = pkgclass.split('_', 1)[1] + deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper()) + prepath = deploydir + '-prediff' + pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) + try: + with open(pcmanifest, 'r') as f: + for line in f: + fn = line.rstrip() + if fn: + try: + os.remove(fn) + except OSError as e: + if e.errno == errno.ENOENT: + pass + os.remove(pcmanifest) + except IOError as e: + if e.errno == errno.ENOENT: + pass +} diff --git a/meta/classes/packagegroup.bbclass b/meta/classes/packagegroup.bbclass index 56cfead82a..eea2e5b9fc 100644 --- a/meta/classes/packagegroup.bbclass +++ b/meta/classes/packagegroup.bbclass @@ -14,21 +14,25 @@ PACKAGE_ARCH ?= "all" # Fully expanded - so it applies the overrides as well PACKAGE_ARCH_EXPANDED := "${PACKAGE_ARCH}" -inherit ${@oe.utils.ifelse(d.getVar('PACKAGE_ARCH_EXPANDED', True) == 'all', 'allarch', '')} +LICENSE ?= "MIT" + +inherit ${@oe.utils.ifelse(d.getVar('PACKAGE_ARCH_EXPANDED') == 'all', 'allarch', '')} # This automatically adds -dbg and -dev flavours of all PACKAGES # to the list. Their dependencies (RRECOMMENDS) are handled as usual # by package_depchains in a following step. # Also mark all packages as ALLOW_EMPTY python () { - packages = d.getVar('PACKAGES', True).split() - genpackages = [] + packages = d.getVar('PACKAGES').split() + if d.getVar('PACKAGEGROUP_DISABLE_COMPLEMENTARY') != '1': + types = ['', '-dbg', '-dev'] + if bb.utils.contains('DISTRO_FEATURES', 'ptest', True, False, d): + types.append('-ptest') + packages = [pkg + suffix for pkg in packages + for suffix in types] + d.setVar('PACKAGES', ' '.join(packages)) for pkg in packages: - d.setVar("ALLOW_EMPTY_%s" % pkg, "1") - for postfix in ['-dbg', '-dev', '-ptest']: - genpackages.append(pkg+postfix) - if d.getVar('PACKAGEGROUP_DISABLE_COMPLEMENTARY', True) != '1': - d.setVar('PACKAGES', ' '.join(packages+genpackages)) + d.setVar('ALLOW_EMPTY_%s' % pkg, '1') } # We don't want to look at shared library dependencies for the @@ -36,16 +40,18 @@ python () { DEPCHAIN_DBGDEFAULTDEPS = "1" # We only need the packaging tasks - disable the rest -do_fetch[noexec] = "1" -do_unpack[noexec] = "1" -do_patch[noexec] = "1" -do_configure[noexec] = "1" -do_compile[noexec] = "1" -do_install[noexec] = "1" -do_populate_sysroot[noexec] = "1" +deltask do_fetch +deltask do_unpack +deltask do_patch +deltask do_configure +deltask do_compile +deltask do_install +deltask do_populate_sysroot python () { - initman = d.getVar("VIRTUAL-RUNTIME_init_manager", True) + if bb.data.inherits_class('nativesdk', d): + return + initman = d.getVar("VIRTUAL-RUNTIME_init_manager") if initman and initman in ['sysvinit', 'systemd'] and not bb.utils.contains('DISTRO_FEATURES', initman, True, False, d): bb.fatal("Please ensure that your setting of VIRTUAL-RUNTIME_init_manager (%s) matches the entries enabled in DISTRO_FEATURES" % initman) } diff --git a/meta/classes/packageinfo.bbclass b/meta/classes/packageinfo.bbclass deleted file mode 100644 index 7d60ace1dc..0000000000 --- a/meta/classes/packageinfo.bbclass +++ /dev/null @@ -1,22 +0,0 @@ -python packageinfo_handler () { - import oe.packagedata - pkginfolist = [] - - pkgdata_dir = e.data.getVar("PKGDATA_DIR", True) + '/runtime/' - if os.path.exists(pkgdata_dir): - for root, dirs, files in os.walk(pkgdata_dir): - for pkgname in files: - if pkgname.endswith('.packaged'): - pkgname = pkgname[:-9] - pkgdatafile = root + pkgname - try: - sdata = oe.packagedata.read_pkgdatafile(pkgdatafile) - sdata['PKG'] = pkgname - pkginfolist.append(sdata) - except Exception as e: - bb.warn("Failed to read pkgdata file %s: %s: %s" % (pkgdatafile, e.__class__, str(e))) - bb.event.fire(bb.event.PackageInfo(pkginfolist), e.data) -} - -addhandler packageinfo_handler -packageinfo_handler[eventmask] = "bb.event.RequestPackageInfo" diff --git a/meta/classes/patch.bbclass b/meta/classes/patch.bbclass index 1e2aab0418..8f35cb4f95 100644 --- a/meta/classes/patch.bbclass +++ b/meta/classes/patch.bbclass @@ -5,112 +5,70 @@ QUILTRCFILE ?= "${STAGING_ETCDIR_NATIVE}/quiltrc" PATCHDEPENDENCY = "${PATCHTOOL}-native:do_populate_sysroot" -inherit terminal - -def src_patches(d, all = False ): - workdir = d.getVar('WORKDIR', True) - fetch = bb.fetch2.Fetch([], d) - patches = [] - sources = [] - for url in fetch.urls: - local = patch_path(url, fetch, workdir) - if not local: - if all: - local = fetch.localpath(url) - sources.append(local) - continue - - urldata = fetch.ud[url] - parm = urldata.parm - patchname = parm.get('pname') or os.path.basename(local) - - apply, reason = should_apply(parm, d) - if not apply: - if reason: - bb.note("Patch %s %s" % (patchname, reason)) - continue - - patchparm = {'patchname': patchname} - if "striplevel" in parm: - striplevel = parm["striplevel"] - elif "pnum" in parm: - #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url) - striplevel = parm["pnum"] - else: - striplevel = '1' - patchparm['striplevel'] = striplevel +PATCH_GIT_USER_NAME ?= "OpenEmbedded" +PATCH_GIT_USER_EMAIL ?= "oe.patch@oe" - patchdir = parm.get('patchdir') - if patchdir: - patchparm['patchdir'] = patchdir - - localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm)) - patches.append(localurl) - - if all: - return sources +inherit terminal - return patches +python () { + if d.getVar('PATCHTOOL') == 'git' and d.getVar('PATCH_COMMIT_FUNCTIONS') == '1': + extratasks = bb.build.tasksbetween('do_unpack', 'do_patch', d) + try: + extratasks.remove('do_unpack') + except ValueError: + # For some recipes do_unpack doesn't exist, ignore it + pass + + d.appendVarFlag('do_patch', 'prefuncs', ' patch_task_patch_prefunc') + for task in extratasks: + d.appendVarFlag(task, 'postfuncs', ' patch_task_postfunc') +} -def patch_path(url, fetch, workdir): - """Return the local path of a patch, or None if this isn't a patch""" +python patch_task_patch_prefunc() { + # Prefunc for do_patch + func = d.getVar('BB_RUNTASK') + srcsubdir = d.getVar('S') - local = fetch.localpath(url) - base, ext = os.path.splitext(os.path.basename(local)) - if ext in ('.gz', '.bz2', '.Z'): - local = os.path.join(workdir, base) - ext = os.path.splitext(base)[1] + patchdir = os.path.join(srcsubdir, 'patches') + if os.path.exists(patchdir): + if os.listdir(patchdir): + d.setVar('PATCH_HAS_PATCHES_DIR', '1') + else: + os.rmdir(patchdir) +} - urldata = fetch.ud[url] - if "apply" in urldata.parm: - apply = oe.types.boolean(urldata.parm["apply"]) - if not apply: - return - elif ext not in (".diff", ".patch"): - return +python patch_task_postfunc() { + # Prefunc for task functions between do_unpack and do_patch + import oe.patch + import shutil + func = d.getVar('BB_RUNTASK') + srcsubdir = d.getVar('S') + + if os.path.exists(srcsubdir): + if func == 'do_patch': + haspatches = (d.getVar('PATCH_HAS_PATCHES_DIR') == '1') + patchdir = os.path.join(srcsubdir, 'patches') + if os.path.exists(patchdir): + shutil.rmtree(patchdir) + if haspatches: + stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir) + if stdout: + bb.process.run('git checkout patches', cwd=srcsubdir) + stdout, _ = bb.process.run('git status --porcelain .', cwd=srcsubdir) + if stdout: + useroptions = [] + oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) + bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(useroptions), func, oe.patch.GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir) +} - return local +def src_patches(d, all=False, expand=True): + import oe.patch + return oe.patch.src_patches(d, all, expand) def should_apply(parm, d): """Determine if we should apply the given patch""" - - if "mindate" in parm or "maxdate" in parm: - pn = d.getVar('PN', True) - srcdate = d.getVar('SRCDATE_%s' % pn, True) - if not srcdate: - srcdate = d.getVar('SRCDATE', True) - - if srcdate == "now": - srcdate = d.getVar('DATE', True) - - if "maxdate" in parm and parm["maxdate"] < srcdate: - return False, 'is outdated' - - if "mindate" in parm and parm["mindate"] > srcdate: - return False, 'is predated' - - - if "minrev" in parm: - srcrev = d.getVar('SRCREV', True) - if srcrev and srcrev < parm["minrev"]: - return False, 'applies to later revisions' - - if "maxrev" in parm: - srcrev = d.getVar('SRCREV', True) - if srcrev and srcrev > parm["maxrev"]: - return False, 'applies to earlier revisions' - - if "rev" in parm: - srcrev = d.getVar('SRCREV', True) - if srcrev and parm["rev"] not in srcrev: - return False, "doesn't apply to revision" - - if "notrev" in parm: - srcrev = d.getVar('SRCREV', True) - if srcrev and parm["notrev"] in srcrev: - return False, "doesn't apply to revision" - - return True, None + import oe.patch + return oe.patch.should_apply(parm, d) should_apply[vardepsexclude] = "DATE SRCDATE" @@ -123,21 +81,20 @@ python patch_do_patch() { "git": oe.patch.GitApplyTree, } - cls = patchsetmap[d.getVar('PATCHTOOL', True) or 'quilt'] + cls = patchsetmap[d.getVar('PATCHTOOL') or 'quilt'] resolvermap = { "noop": oe.patch.NOOPResolver, "user": oe.patch.UserResolver, } - rcls = resolvermap[d.getVar('PATCHRESOLVE', True) or 'user'] + rcls = resolvermap[d.getVar('PATCHRESOLVE') or 'user'] classes = {} - s = d.getVar('S', True) + s = d.getVar('S') - path = os.getenv('PATH') - os.putenv('PATH', d.getVar('PATH', True)) + os.putenv('PATH', d.getVar('PATH')) # We must use one TMPDIR per process so that the "patch" processes # don't generate the same temp file name. diff --git a/meta/classes/pixbufcache.bbclass b/meta/classes/pixbufcache.bbclass index 349967d74b..b3e507f61b 100644 --- a/meta/classes/pixbufcache.bbclass +++ b/meta/classes/pixbufcache.bbclass @@ -8,6 +8,8 @@ inherit qemu PIXBUF_PACKAGES ??= "${PN}" +PACKAGE_WRITE_DEPS += "qemu-native gdk-pixbuf-native" + pixbufcache_common() { if [ "x$D" != "x" ]; then $INTERCEPT_DIR/postinst_intercept update_pixbuf_cache ${PKG} mlprefix=${MLPREFIX} libdir=${libdir} \ @@ -15,7 +17,7 @@ if [ "x$D" != "x" ]; then else # Update the pixbuf loaders in case they haven't been registered yet - GDK_PIXBUF_MODULEDIR=${libdir}/gdk-pixbuf-2.0/2.10.0/loaders gdk-pixbuf-query-loaders --update-cache + ${libdir}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders --update-cache if [ -x ${bindir}/gtk-update-icon-cache ] && [ -d ${datadir}/icons ]; then for icondir in /usr/share/icons/*; do @@ -28,40 +30,35 @@ fi } python populate_packages_append() { - pixbuf_pkgs = d.getVar('PIXBUF_PACKAGES', True).split() + pixbuf_pkgs = d.getVar('PIXBUF_PACKAGES').split() for pkg in pixbuf_pkgs: bb.note("adding pixbuf postinst and postrm scripts to %s" % pkg) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True) + postinst = d.getVar('pkg_postinst_%s' % pkg) or d.getVar('pkg_postinst') if not postinst: postinst = '#!/bin/sh\n' - postinst += d.getVar('pixbufcache_common', True) + postinst += d.getVar('pixbufcache_common') d.setVar('pkg_postinst_%s' % pkg, postinst) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True) + postrm = d.getVar('pkg_postrm_%s' % pkg) or d.getVar('pkg_postrm') if not postrm: postrm = '#!/bin/sh\n' - postrm += d.getVar('pixbufcache_common', True) + postrm += d.getVar('pixbufcache_common') d.setVar('pkg_postrm_%s' % pkg, postrm) } gdkpixbuf_complete() { - GDK_PIXBUF_FATAL_LOADER=1 ${STAGING_BINDIR_NATIVE}/gdk-pixbuf-query-loaders --update-cache || exit 1 +GDK_PIXBUF_FATAL_LOADER=1 ${STAGING_LIBDIR_NATIVE}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders --update-cache || exit 1 } -# -# Add an sstate postinst hook to update the cache for native packages. -# An error exit during populate_sysroot_setscene allows bitbake to -# try to recover by re-building the package. -# -SSTATEPOSTINSTFUNCS_append_class-native = " pixbufcache_sstate_postinst" +DEPENDS_append_class-native = " gdk-pixbuf-native" +SYSROOT_PREPROCESS_FUNCS_append_class-native = " pixbufcache_sstate_postinst" # See base.bbclass for the other half of this pixbufcache_sstate_postinst() { - if [ "${BB_CURRENTTASK}" = "populate_sysroot" ]; then - ${gdkpixbuf_complete} - elif [ "${BB_CURRENTTASK}" = "populate_sysroot_setscene" ]; then - echo "${gdkpixbuf_complete}" >> ${STAGING_DIR}/sstatecompletions - fi + mkdir -p ${SYSROOT_DESTDIR}${bindir} + dest=${SYSROOT_DESTDIR}${bindir}/postinst-${PN} + echo '#!/bin/sh' > $dest + echo "${gdkpixbuf_complete}" >> $dest + chmod 0755 $dest } - diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass index a9e9bd71d2..563582e0a0 100644 --- a/meta/classes/populate_sdk_base.bbclass +++ b/meta/classes/populate_sdk_base.bbclass @@ -11,13 +11,13 @@ COMPLEMENTARY_GLOB[ptest-pkgs] = '*-ptest' def complementary_globs(featurevar, d): all_globs = d.getVarFlags('COMPLEMENTARY_GLOB') globs = [] - features = set((d.getVar(featurevar, True) or '').split()) + features = set((d.getVar(featurevar) or '').split()) for name, glob in all_globs.items(): if name in features: globs.append(glob) return ' '.join(globs) -SDKIMAGE_FEATURES ??= "dev-pkgs dbg-pkgs" +SDKIMAGE_FEATURES ??= "dev-pkgs dbg-pkgs ${@bb.utils.contains('DISTRO_FEATURES', 'api-documentation', 'doc-pkgs', '', d)}" SDKIMAGE_INSTALL_COMPLEMENTARY = '${@complementary_globs("SDKIMAGE_FEATURES", d)}' inherit rootfs_${IMAGE_PKGTYPE} @@ -26,21 +26,20 @@ SDK_DIR = "${WORKDIR}/sdk" SDK_OUTPUT = "${SDK_DIR}/image" SDK_DEPLOY = "${DEPLOY_DIR}/sdk" +SDKDEPLOYDIR = "${WORKDIR}/${SDKMACHINE}-deploy-${PN}-populate-sdk" + B_task-populate-sdk = "${SDK_DIR}" SDKTARGETSYSROOT = "${SDKPATH}/sysroots/${REAL_MULTIMACH_TARGET_SYS}" TOOLCHAIN_HOST_TASK ?= "nativesdk-packagegroup-sdk-host packagegroup-cross-canadian-${MACHINE}" TOOLCHAIN_HOST_TASK_ATTEMPTONLY ?= "" -TOOLCHAIN_TARGET_TASK ?= " \ - ${@multilib_pkg_extend(d, 'packagegroup-core-standalone-sdk-target')} \ - ${@multilib_pkg_extend(d, 'packagegroup-core-standalone-sdk-target-dbg')} \ - " +TOOLCHAIN_TARGET_TASK ?= "${@multilib_pkg_extend(d, 'packagegroup-core-standalone-sdk-target')}" TOOLCHAIN_TARGET_TASK_ATTEMPTONLY ?= "" TOOLCHAIN_OUTPUTNAME ?= "${SDK_NAME}-toolchain-${SDK_VERSION}" SDK_RDEPENDS = "${TOOLCHAIN_TARGET_TASK} ${TOOLCHAIN_HOST_TASK}" -SDK_DEPENDS = "virtual/fakeroot-native pbzip2-native" +SDK_DEPENDS = "virtual/fakeroot-native pixz-native" # We want the MULTIARCH_TARGET_SYS to point to the TUNE_PKGARCH, not PACKAGE_ARCH as it # could be set to the MACHINE_ARCH @@ -55,36 +54,50 @@ SDK_PRE_INSTALL_COMMAND ?= "" SDK_POST_INSTALL_COMMAND ?= "" SDK_RELOCATE_AFTER_INSTALL ?= "1" -SDK_TITLE ?= "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} SDK" +SDKEXTPATH ?= "~/${@d.getVar('DISTRO')}_sdk" +SDK_TITLE ?= "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} SDK" -SDK_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest" -SDK_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest" +SDK_TARGET_MANIFEST = "${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.target.manifest" +SDK_HOST_MANIFEST = "${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.host.manifest" python write_target_sdk_manifest () { from oe.sdk import sdk_list_installed_packages - sdkmanifestdir = os.path.dirname(d.getVar("SDK_TARGET_MANIFEST", True)) + from oe.utils import format_pkg_list + sdkmanifestdir = os.path.dirname(d.getVar("SDK_TARGET_MANIFEST")) + pkgs = sdk_list_installed_packages(d, True) if not os.path.exists(sdkmanifestdir): bb.utils.mkdirhier(sdkmanifestdir) - with open(d.getVar('SDK_TARGET_MANIFEST', True), 'w') as output: - output.write(sdk_list_installed_packages(d, True, 'ver')) + with open(d.getVar('SDK_TARGET_MANIFEST'), 'w') as output: + output.write(format_pkg_list(pkgs, 'ver')) +} + +python write_sdk_test_data() { + from oe.data import export2json + testdata = "%s/%s.testdata.json" % (d.getVar('SDKDEPLOYDIR'), d.getVar('TOOLCHAIN_OUTPUTNAME')) + bb.utils.mkdirhier(os.path.dirname(testdata)) + export2json(d, testdata) } python write_host_sdk_manifest () { from oe.sdk import sdk_list_installed_packages - sdkmanifestdir = os.path.dirname(d.getVar("SDK_HOST_MANIFEST", True)) + from oe.utils import format_pkg_list + sdkmanifestdir = os.path.dirname(d.getVar("SDK_HOST_MANIFEST")) + pkgs = sdk_list_installed_packages(d, False) if not os.path.exists(sdkmanifestdir): bb.utils.mkdirhier(sdkmanifestdir) - with open(d.getVar('SDK_HOST_MANIFEST', True), 'w') as output: - output.write(sdk_list_installed_packages(d, False, 'ver')) + with open(d.getVar('SDK_HOST_MANIFEST'), 'w') as output: + output.write(format_pkg_list(pkgs, 'ver')) } -POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; " +POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; write_sdk_test_data ; " POPULATE_SDK_POST_HOST_COMMAND_append = " write_host_sdk_manifest; " +SDK_PACKAGING_COMMAND = "${@'${SDK_PACKAGING_FUNC};' if '${SDK_PACKAGING_FUNC}' else ''}" +SDK_POSTPROCESS_COMMAND = " create_sdk_files; check_sdk_sysroots; tar_sdk; ${SDK_PACKAGING_COMMAND} " -fakeroot python do_populate_sdk() { +def populate_sdk_common(d): from oe.sdk import populate_sdk from oe.manifest import create_manifest, Manifest - pn = d.getVar('PN', True) + pn = d.getVar('PN') runtime_mapping_rename("TOOLCHAIN_TARGET_TASK", pn, d) runtime_mapping_rename("TOOLCHAIN_TARGET_TASK_ATTEMPTONLY", pn, d) @@ -92,26 +105,26 @@ fakeroot python do_populate_sdk() { ld.setVar("PKGDATA_DIR", "${STAGING_DIR}/${SDK_ARCH}-${SDKPKGSUFFIX}${SDK_VENDOR}-${SDK_OS}/pkgdata") runtime_mapping_rename("TOOLCHAIN_HOST_TASK", pn, ld) runtime_mapping_rename("TOOLCHAIN_HOST_TASK_ATTEMPTONLY", pn, ld) - d.setVar("TOOLCHAIN_HOST_TASK", ld.getVar("TOOLCHAIN_HOST_TASK", True)) - d.setVar("TOOLCHAIN_HOST_TASK_ATTEMPTONLY", ld.getVar("TOOLCHAIN_HOST_TASK_ATTEMPTONLY", True)) + d.setVar("TOOLCHAIN_HOST_TASK", ld.getVar("TOOLCHAIN_HOST_TASK")) + d.setVar("TOOLCHAIN_HOST_TASK_ATTEMPTONLY", ld.getVar("TOOLCHAIN_HOST_TASK_ATTEMPTONLY")) # create target/host SDK manifests - create_manifest(d, manifest_dir=d.getVar('SDK_DIR', True), + create_manifest(d, manifest_dir=d.getVar('SDK_DIR'), manifest_type=Manifest.MANIFEST_TYPE_SDK_HOST) - create_manifest(d, manifest_dir=d.getVar('SDK_DIR', True), + create_manifest(d, manifest_dir=d.getVar('SDK_DIR'), manifest_type=Manifest.MANIFEST_TYPE_SDK_TARGET) populate_sdk(d) - # Process DEFAULTTUNE - bb.build.exec_func("create_sdk_files", d) - - bb.build.exec_func("tar_sdk", d) - - sdk_packaging_func = d.getVar("SDK_PACKAGING_FUNC", True) or "" - if sdk_packaging_func.strip(): - bb.build.exec_func(d.getVar("SDK_PACKAGING_FUNC", True), d) +fakeroot python do_populate_sdk() { + populate_sdk_common(d) } +SSTATETASKS += "do_populate_sdk" +SSTATE_SKIP_CREATION_task-populate-sdk = '1' +do_populate_sdk[cleandirs] = "${SDKDEPLOYDIR}" +do_populate_sdk[sstate-inputdirs] = "${SDKDEPLOYDIR}" +do_populate_sdk[sstate-outputdirs] = "${SDK_DEPLOY}" +do_populate_sdk[stamp-extra-info] = "${MACHINE}${SDKMACHINE}" fakeroot create_sdk_files() { cp ${COREBASE}/scripts/relocate_sdk.py ${SDK_OUTPUT}/${SDKPATH}/ @@ -122,18 +135,70 @@ fakeroot create_sdk_files() { sed -i -e "s:##DEFAULT_INSTALL_DIR##:$escaped_sdkpath:" ${SDK_OUTPUT}/${SDKPATH}/relocate_sdk.py } +python check_sdk_sysroots() { + # Fails build if there are broken or dangling symlinks in SDK sysroots + + if d.getVar('CHECK_SDK_SYSROOTS') != '1': + # disabled, bail out + return + + def norm_path(path): + return os.path.abspath(path) + + # Get scan root + SCAN_ROOT = norm_path("%s/%s/sysroots/" % (d.getVar('SDK_OUTPUT'), + d.getVar('SDKPATH'))) + + bb.note('Checking SDK sysroots at ' + SCAN_ROOT) + + def check_symlink(linkPath): + if not os.path.islink(linkPath): + return + + linkDirPath = os.path.dirname(linkPath) + + targetPath = os.readlink(linkPath) + if not os.path.isabs(targetPath): + targetPath = os.path.join(linkDirPath, targetPath) + targetPath = norm_path(targetPath) + + if SCAN_ROOT != os.path.commonprefix( [SCAN_ROOT, targetPath] ): + bb.error("Escaping symlink {0!s} --> {1!s}".format(linkPath, targetPath)) + return + + if not os.path.exists(targetPath): + bb.error("Broken symlink {0!s} --> {1!s}".format(linkPath, targetPath)) + return + + if os.path.isdir(targetPath): + dir_walk(targetPath) + + def walk_error_handler(e): + bb.error(str(e)) + + def dir_walk(rootDir): + for dirPath,subDirEntries,fileEntries in os.walk(rootDir, followlinks=False, onerror=walk_error_handler): + entries = subDirEntries + fileEntries + for e in entries: + ePath = os.path.join(dirPath, e) + check_symlink(ePath) + + # start + dir_walk(SCAN_ROOT) +} + SDKTAROPTS = "--owner=root --group=root" fakeroot tar_sdk() { # Package it up - mkdir -p ${SDK_DEPLOY} + mkdir -p ${SDKDEPLOYDIR} cd ${SDK_OUTPUT}/${SDKPATH} - tar ${SDKTAROPTS} -cf - . | pbzip2 > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.tar.bz2 + tar ${SDKTAROPTS} -cf - . | pixz > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz } fakeroot create_shar() { # copy in the template shar extractor script - cp ${COREBASE}/meta/files/toolchain-shar-extract.sh ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + cp ${COREBASE}/meta/files/toolchain-shar-extract.sh ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh rm -f ${T}/pre_install_command ${T}/post_install_command @@ -149,27 +214,29 @@ ${SDK_POST_INSTALL_COMMAND} EOF sed -i -e '/@SDK_PRE_INSTALL_COMMAND@/r ${T}/pre_install_command' \ -e '/@SDK_POST_INSTALL_COMMAND@/r ${T}/post_install_command' \ - ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh # substitute variables sed -i -e 's#@SDK_ARCH@#${SDK_ARCH}#g' \ -e 's#@SDKPATH@#${SDKPATH}#g' \ - -e 's#@OLDEST_KERNEL@#${OLDEST_KERNEL}#g' \ + -e 's#@SDKEXTPATH@#${SDKEXTPATH}#g' \ + -e 's#@OLDEST_KERNEL@#${SDK_OLDEST_KERNEL}#g' \ -e 's#@REAL_MULTIMACH_TARGET_SYS@#${REAL_MULTIMACH_TARGET_SYS}#g' \ - -e 's#@SDK_TITLE@#${SDK_TITLE}#g' \ + -e 's#@SDK_TITLE@#${@d.getVar("SDK_TITLE").replace('&', '\&')}#g' \ -e 's#@SDK_VERSION@#${SDK_VERSION}#g' \ -e '/@SDK_PRE_INSTALL_COMMAND@/d' \ -e '/@SDK_POST_INSTALL_COMMAND@/d' \ - ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + -e 's#@SDK_GCC_VER@#${@oe.utils.host_gcc_version(d)}#g' \ + ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh # add execution permission - chmod +x ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + chmod +x ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh # append the SDK tarball - cat ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.tar.bz2 >> ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + cat ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz >> ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh # delete the old tarball, we don't need it anymore - rm ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.tar.bz2 + rm ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz } populate_sdk_log_check() { @@ -179,8 +246,7 @@ populate_sdk_log_check() { echo "log_check: Using $lf_path as logfile" - if test -e "$lf_path" - then + if [ -e "$lf_path" ]; then ${IMAGE_PKGTYPE}_log_check $target $lf_path else echo "Cannot find logfile [$lf_path]" @@ -189,8 +255,24 @@ populate_sdk_log_check() { done } -do_populate_sdk[dirs] = "${TOPDIR}" -do_populate_sdk[depends] += "${@' '.join([x + ':do_populate_sysroot' for x in d.getVar('SDK_DEPENDS', True).split()])} ${@d.getVarFlag('do_rootfs', 'depends', False)}" -do_populate_sdk[rdepends] = "${@' '.join([x + ':do_populate_sysroot' for x in d.getVar('SDK_RDEPENDS', True).split()])}" +def sdk_command_variables(d): + return ['OPKG_PREPROCESS_COMMANDS','OPKG_POSTPROCESS_COMMANDS','POPULATE_SDK_POST_HOST_COMMAND','POPULATE_SDK_POST_TARGET_COMMAND','SDK_POSTPROCESS_COMMAND','RPM_PREPROCESS_COMMANDS', + 'RPM_POSTPROCESS_COMMANDS'] + +def sdk_variables(d): + variables = ['BUILD_IMAGES_FROM_FEEDS','SDK_OS','SDK_OUTPUT','SDKPATHNATIVE','SDKTARGETSYSROOT','SDK_DIR','SDK_VENDOR','SDKIMAGE_INSTALL_COMPLEMENTARY','SDK_PACKAGE_ARCHS','SDK_OUTPUT', + 'SDKTARGETSYSROOT','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS', + 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI', 'PACKAGE_EXCLUDE_COMPLEMENTARY'] + variables.extend(sdk_command_variables(d)) + return " ".join(variables) + +do_populate_sdk[vardeps] += "${@sdk_variables(d)}" + +do_populate_sdk[file-checksums] += "${COREBASE}/meta/files/toolchain-shar-relocate.sh:True \ + ${COREBASE}/meta/files/toolchain-shar-extract.sh:True" + +do_populate_sdk[dirs] = "${PKGDATA_DIR} ${TOPDIR}" +do_populate_sdk[depends] += "${@' '.join([x + ':do_populate_sysroot' for x in d.getVar('SDK_DEPENDS').split()])} ${@d.getVarFlag('do_rootfs', 'depends', False)}" +do_populate_sdk[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('SDK_RDEPENDS').split()])}" do_populate_sdk[recrdeptask] += "do_packagedata do_package_write_rpm do_package_write_ipk do_package_write_deb" addtask populate_sdk diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass index a36bf16c78..4dfb94caf1 100644 --- a/meta/classes/populate_sdk_ext.bbclass +++ b/meta/classes/populate_sdk_ext.bbclass @@ -11,13 +11,57 @@ TOOLCHAIN_HOST_TASK_task-populate-sdk-ext = " \ TOOLCHAIN_TARGET_TASK_task-populate-sdk-ext = "" -SDK_RDEPENDS_append_task-populate-sdk-ext = " ${SDK_TARGETS}" - SDK_RELOCATE_AFTER_INSTALL_task-populate-sdk-ext = "0" -SDK_META_CONF_WHITELIST ?= "MACHINE DISTRO PACKAGE_CLASSES" +SDK_EXT = "" +SDK_EXT_task-populate-sdk-ext = "-ext" + +# Options are full or minimal +SDK_EXT_TYPE ?= "full" +SDK_INCLUDE_PKGDATA ?= "0" +SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}" + +SDK_RECRDEP_TASKS ?= "" + +SDK_LOCAL_CONF_WHITELIST ?= "" +SDK_LOCAL_CONF_BLACKLIST ?= "CONF_VERSION \ + BB_NUMBER_THREADS \ + BB_NUMBER_PARSE_THREADS \ + PARALLEL_MAKE \ + PRSERV_HOST \ + SSTATE_MIRRORS \ + DL_DIR \ + SSTATE_DIR \ + TMPDIR \ + " +SDK_INHERIT_BLACKLIST ?= "buildhistory icecc" +SDK_UPDATE_URL ?= "" SDK_TARGETS ?= "${PN}" + +def get_sdk_install_targets(d, images_only=False): + sdk_install_targets = '' + if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal': + sdk_install_targets = d.getVar('SDK_TARGETS') + + depd = d.getVar('BB_TASKDEPDATA', False) + tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) + tasklist.remove('do_build') + for v in depd.values(): + if v[1] in tasklist: + if v[0] not in sdk_install_targets: + sdk_install_targets += ' {}'.format(v[0]) + + if not images_only: + if d.getVar('SDK_INCLUDE_PKGDATA') == '1': + sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata' + if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1': + sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot' + + return sdk_install_targets + +get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA" + OE_INIT_ENV_SCRIPT ?= "oe-init-build-env" # The files from COREBASE that you want preserved in the COREBASE copied @@ -33,27 +77,102 @@ COREBASE_FILES ?= " \ SDK_DIR_task-populate-sdk-ext = "${WORKDIR}/sdk-ext" B_task-populate-sdk-ext = "${SDK_DIR}" -TOOLCHAIN_OUTPUTNAME_task-populate-sdk-ext = "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" +TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" +TOOLCHAIN_OUTPUTNAME_task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}" + +SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest" +SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest" + +SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" + +def clean_esdk_builddir(d, sdkbasepath): + """Clean up traces of the fake build for create_filtered_tasklist()""" + import shutil + cleanpaths = 'cache conf/sanity_info tmp'.split() + for pth in cleanpaths: + fullpth = os.path.join(sdkbasepath, pth) + if os.path.isdir(fullpth): + shutil.rmtree(fullpth) + elif os.path.isfile(fullpth): + os.remove(fullpth) + +def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath): + """ + Create a filtered list of tasks. Also double-checks that the build system + within the SDK basically works and required sstate artifacts are available. + """ + import tempfile + import shutil + import oe.copy_buildsystem -SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} Extensible SDK" + # Create a temporary build directory that we can pass to the env setup script + shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak') + try: + with open(sdkbasepath + '/conf/local.conf', 'a') as f: + # Force the use of sstate from the build system + f.write('\nSSTATE_DIR_forcevariable = "%s"\n' % d.getVar('SSTATE_DIR')) + f.write('SSTATE_MIRRORS_forcevariable = ""\n') + # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it + f.write('TMPDIR_forcevariable = "${TOPDIR}/tmp"\n') + f.write('TCLIBCAPPEND_forcevariable = ""\n') + # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will + # be different and we won't be able to find our native sstate) + if not bb.data.inherits_class('uninative', d): + f.write('INHERIT_remove = "uninative"\n') + + # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake + # will not allow in its COREBASE path, so we need to rename the directory temporarily + temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk' + # Delete any existing temp dir + try: + shutil.rmtree(temp_sdkbasepath) + except FileNotFoundError: + pass + os.rename(sdkbasepath, temp_sdkbasepath) + try: + cmdprefix = '. %s .; ' % conf_initpath + logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt' + try: + oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile) + except bb.process.ExecutionError as e: + msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip() + if 'attempted to execute unexpectedly and should have been setscened' in e.stdout: + msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n' + bb.fatal(msg) + finally: + os.rename(temp_sdkbasepath, sdkbasepath) + # Clean out residue of running bitbake, which check_sstate_task_list() + # will effectively do + clean_esdk_builddir(d, sdkbasepath) + finally: + os.replace(sdkbasepath + '/conf/local.conf.bak', sdkbasepath + '/conf/local.conf') python copy_buildsystem () { import re + import shutil + import glob import oe.copy_buildsystem - oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT', True) + oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT') conf_bbpath = '' conf_initpath = '' core_meta_subdir = '' # Copy in all metadata layers + bitbake (as repositories) - buildsystem = oe.copy_buildsystem.BuildSystem(d) - baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True) - layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers') + buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) + baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH') + + # Determine if we're building a derivative extensible SDK (from devtool build-sdk) + derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1' + if derivative: + workspace_name = 'orig-workspace' + else: + workspace_name = None + layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) sdkbblayers = [] - corebase = os.path.basename(d.getVar('COREBASE', True)) + corebase = os.path.basename(d.getVar('COREBASE')) for layer in layers_copied: if corebase == os.path.basename(layer): conf_bbpath = os.path.join('layers', layer, 'bitbake') @@ -77,18 +196,27 @@ python copy_buildsystem () { d.setVar('scriptrelpath', scriptrelpath) # Write out config file for devtool - import ConfigParser - config = ConfigParser.SafeConfigParser() + import configparser + config = configparser.SafeConfigParser() config.add_section('General') config.set('General', 'bitbake_subdir', conf_bbpath) config.set('General', 'init_path', conf_initpath) config.set('General', 'core_meta_subdir', core_meta_subdir) + config.add_section('SDK') + config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS')) + updateurl = d.getVar('SDK_UPDATE_URL') + if updateurl: + config.set('SDK', 'updateserver', updateurl) bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf')) with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: config.write(f) + unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') + with open(unlockedsigs, 'w') as f: + pass + # Create a layer for new recipes / appends - bbpath = d.getVar('BBPATH', True) + bbpath = d.getVar('BBPATH') bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')]) # Create bblayers.conf @@ -99,7 +227,12 @@ python copy_buildsystem () { f.write('# this configuration provides, it is strongly suggested that you set\n') f.write('# up a proper instance of the full build system and use that instead.\n\n') - f.write('LCONF_VERSION = "%s"\n\n' % d.getVar('LCONF_VERSION', False)) + # LCONF_VERSION may not be set, for example when using meta-poky + # so don't error if it isn't found + lconf_version = d.getVar('LCONF_VERSION', False) + if lconf_version is not None: + f.write('LCONF_VERSION = "%s"\n\n' % lconf_version) + f.write('BBPATH = "$' + '{TOPDIR}"\n') f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n') f.write('BBLAYERS := " \\\n') @@ -108,80 +241,321 @@ python copy_buildsystem () { f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n') f.write(' "\n') - # Create local.conf - with open(baseoutpath + '/conf/local.conf', 'w') as f: - f.write('# WARNING: this configuration has been automatically generated and in\n') - f.write('# most cases should not be edited. If you need more flexibility than\n') - f.write('# this configuration provides, it is strongly suggested that you set\n') - f.write('# up a proper instance of the full build system and use that instead.\n\n') - - f.write('INHERIT += "%s"\n\n' % 'uninative') - f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) + # Copy uninative tarball + # For now this is where uninative.bbclass expects the tarball + uninative_file = d.expand('${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2') + uninative_checksum = bb.utils.sha256_file(uninative_file) + uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum) + bb.utils.mkdirhier(uninative_outdir) + shutil.copy(uninative_file, uninative_outdir) - # This is a bit of a hack, but we really don't want these dependencies - # (we're including them in the SDK as nativesdk- versions instead) - f.write('POKYQEMUDEPS_forcevariable = ""\n\n') - f.write('EXTRA_IMAGEDEPENDS_remove = "qemu-native qemu-helper-native"\n\n') + env_whitelist = (d.getVar('BB_ENV_EXTRAWHITE') or '').split() + env_whitelist_values = {} - # Bypass the default connectivity check if any - f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') - - # Another hack, but we want the native part of sstate to be kept the same - # regardless of the host distro - fixedlsbstring = 'SDK-Fixed' - f.write('NATIVELSBSTRING_forcevariable = "%s"\n\n' % fixedlsbstring) - - # Ensure locked sstate cache objects are re-used without error - f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "warn"\n\n') - - for varname in d.getVar('SDK_META_CONF_WHITELIST', True).split(): - f.write('%s = "%s"\n' % (varname, d.getVar(varname, True))) - f.write('require conf/locked-sigs.inc\n') - f.write('require conf/work-config.inc\n') - - sigfile = d.getVar('WORKDIR', True) + '/locked-sigs.inc' - oe.copy_buildsystem.generate_locked_sigs(sigfile, d) + # Create local.conf + builddir = d.getVar('TOPDIR') + if derivative and os.path.exists(builddir + '/conf/auto.conf'): + shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') + if derivative: + shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') + else: + local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST') or '').split() + local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST') or '').split() + def handle_var(varname, origvalue, op, newlines): + if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist): + newlines.append('# Removed original setting of %s\n' % varname) + return None, op, 0, True + else: + if varname in env_whitelist: + env_whitelist_values[varname] = origvalue + return origvalue, op, 0, True + varlist = ['[^#=+ ]*'] + oldlines = [] + if os.path.exists(builddir + '/conf/auto.conf'): + with open(builddir + '/conf/auto.conf', 'r') as f: + oldlines += f.readlines() + with open(builddir + '/conf/local.conf', 'r') as f: + oldlines += f.readlines() + (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) + + with open(baseoutpath + '/conf/local.conf', 'w') as f: + f.write('# WARNING: this configuration has been automatically generated and in\n') + f.write('# most cases should not be edited. If you need more flexibility than\n') + f.write('# this configuration provides, it is strongly suggested that you set\n') + f.write('# up a proper instance of the full build system and use that instead.\n\n') + for line in newlines: + if line.strip() and not line.startswith('#'): + f.write(line) + # Write a newline just in case there's none at the end of the original + f.write('\n') + + f.write('TMPDIR = "${TOPDIR}/tmp"\n') + f.write('TCLIBCAPPEND = ""\n') + f.write('DL_DIR = "${TOPDIR}/downloads"\n') + + f.write('INHERIT += "%s"\n' % 'uninative') + f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum)) + f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) + + # Some classes are not suitable for SDK, remove them from INHERIT + f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False)) + + # Bypass the default connectivity check if any + f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') + + # This warning will come out if reverse dependencies for a task + # don't have sstate as well as the task itself. We already know + # this will be the case for the extensible sdk, so turn off the + # warning. + f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') + + # Warn if the sigs in the locked-signature file don't match + # the sig computed from the metadata. + f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n') + + # Set up whitelist for run on install + f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n') + + # Hide the config information from bitbake output (since it's fixed within the SDK) + f.write('BUILDCFG_HEADER = ""\n\n') + + # Map gcc-dependent uninative sstate cache for installer usage + f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n') + + # Allow additional config through sdk-extra.conf + fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) + if fn: + with open(fn, 'r') as xf: + for line in xf: + f.write(line) + + # If you define a sdk_extraconf() function then it can contain additional config + # (Though this is awkward; sdk-extra.conf should probably be used instead) + extraconf = (d.getVar('sdk_extraconf') or '').strip() + if extraconf: + # Strip off any leading / trailing spaces + for line in extraconf.splitlines(): + f.write(line.strip() + '\n') + + f.write('require conf/locked-sigs.inc\n') + f.write('require conf/unlocked-sigs.inc\n') + + # Write a templateconf.cfg + with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f: + f.write('meta/conf\n') + + # Ensure any variables set from the external environment (by way of + # BB_ENV_EXTRAWHITE) are set in the SDK's configuration + extralines = [] + for name, value in env_whitelist_values.items(): + actualvalue = d.getVar(name) or '' + if value != actualvalue: + extralines.append('%s = "%s"\n' % (name, actualvalue)) + if extralines: + with open(baseoutpath + '/conf/local.conf', 'a') as f: + f.write('\n') + f.write('# Extra settings from environment:\n') + for line in extralines: + f.write(line) + f.write('\n') # Filter the locked signatures file to just the sstate tasks we are interested in - allowed_tasks = ['do_populate_lic', 'do_populate_sysroot', 'do_packagedata', 'do_package_write_ipk', 'do_package_write_rpm', 'do_package_write_deb', 'do_package_qa', 'do_deploy'] - excluded_targets = d.getVar('SDK_TARGETS', True) + excluded_targets = get_sdk_install_targets(d, images_only=True) + sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc' - oe.copy_buildsystem.prune_lockedsigs(allowed_tasks, - excluded_targets, + oe.copy_buildsystem.prune_lockedsigs([], + excluded_targets.split(), sigfile, lockedsigs_pruned) sstate_out = baseoutpath + '/sstate-cache' bb.utils.remove(sstate_out, True) - oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, - d.getVar('SSTATE_DIR', True), - sstate_out, d, - fixedlsbstring) - # Create a dummy config file for additional settings - with open(baseoutpath + '/conf/work-config.inc', 'w') as f: - pass + # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d) + fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d) + + sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1') + sdk_ext_type = d.getVar('SDK_EXT_TYPE') + if sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative: + # Create the filtered task list used to generate the sstate cache shipped with the SDK + tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt' + create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath) + else: + tasklistfn = None + + # Add packagedata if enabled + if d.getVar('SDK_INCLUDE_PKGDATA') == '1': + lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc' + lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc' + shutil.move(lockedsigs_pruned, lockedsigs_base) + oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'], + lockedsigs_base, + d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc', + lockedsigs_pruned, + lockedsigs_copy) + + if sdk_include_toolchain: + lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc' + lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc") + shutil.move(lockedsigs_pruned, lockedsigs_base) + oe.copy_buildsystem.merge_lockedsigs([], + lockedsigs_base, + lockedsigs_toolchain, + lockedsigs_pruned) + oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, + d.getVar('SSTATE_DIR'), + sstate_out, d, + fixedlsbstring, + filterfile=tasklistfn) + + if sdk_ext_type == 'minimal': + if derivative: + # Assume the user is not going to set up an additional sstate + # mirror, thus we need to copy the additional artifacts (from + # workspace recipes) into the derivative SDK + lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc' + if os.path.exists(lockedsigs_orig): + lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc' + oe.copy_buildsystem.merge_lockedsigs(None, + lockedsigs_orig, + lockedsigs_pruned, + None, + lockedsigs_extra) + oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, + d.getVar('SSTATE_DIR'), + sstate_out, d, + fixedlsbstring, + filterfile=tasklistfn) + else: + oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, + d.getVar('SSTATE_DIR'), + sstate_out, d, + fixedlsbstring, + filterfile=tasklistfn) + + # We don't need sstate do_package files + for root, dirs, files in os.walk(sstate_out): + for name in files: + if name.endswith("_package.tgz"): + f = os.path.join(root, name) + os.remove(f) + + # Write manifest file + # Note: at the moment we cannot include the env setup script here to keep + # it updated, since it gets modified during SDK installation (see + # sdk_ext_postinst() below) thus the checksum we take here would always + # be different. + manifest_file_list = ['conf/*'] + manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest') + with open(manifest_file, 'w') as f: + for item in manifest_file_list: + for fn in glob.glob(os.path.join(baseoutpath, item)): + if fn == manifest_file: + continue + chksum = bb.utils.sha256_file(fn) + f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath))) } +def get_current_buildtools(d): + """Get the file name of the current buildtools installer""" + import glob + btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh')) + btfiles.sort(key=os.path.getctime) + return os.path.basename(btfiles[-1]) + +def get_sdk_required_utilities(buildtools_fn, d): + """Find required utilities that aren't provided by the buildtools""" + sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split() + sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc')) + sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++')) + buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn) + filelist, _ = bb.process.run('%s -l' % buildtools_installer) + localdata = bb.data.createCopy(d) + localdata.setVar('SDKPATH', '.') + sdkpathnative = localdata.getVar('SDKPATHNATIVE') + sdkbindirs = [localdata.getVar('bindir_nativesdk'), + localdata.getVar('sbindir_nativesdk'), + localdata.getVar('base_bindir_nativesdk'), + localdata.getVar('base_sbindir_nativesdk')] + for line in filelist.splitlines(): + splitline = line.split() + if len(splitline) > 5: + fn = splitline[5] + if not fn.startswith('./'): + fn = './%s' % fn + if fn.startswith(sdkpathnative): + relpth = '/' + os.path.relpath(fn, sdkpathnative) + for bindir in sdkbindirs: + if relpth.startswith(bindir): + relpth = os.path.relpath(relpth, bindir) + if relpth in sanity_required_utilities: + sanity_required_utilities.remove(relpth) + break + return ' '.join(sanity_required_utilities) + install_tools() { install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk} - lnr ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath}/devtool ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/devtool - lnr ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath}/recipetool ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/recipetool + scripts="devtool recipetool oe-find-native-sysroot runqemu*" + for script in $scripts; do + for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do + lnr ${scriptfn} ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/`basename $scriptfn` + done + done + # We can't use the same method as above because files in the sysroot won't exist at this point + # (they get populated from sstate on installation) + unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd" + if [ "${SDK_INCLUDE_TOOLCHAIN}" = "1" -a ! -e $unfsd_path ] ; then + binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE'), d.getVar('TMPDIR'))} + lnr ${SDK_OUTPUT}/${SDKPATH}/tmp/$binrelpath/unfsd $unfsd_path + fi touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase - install ${SDK_DEPLOY}/${DISTRO}-${TCLIBC}-${SDK_ARCH}-buildtools-tarball-${TUNE_PKGARCH}-buildtools-nativesdk-standalone-${DISTRO_VERSION}.sh ${SDK_OUTPUT}/${SDKPATH} + # find latest buildtools-tarball and install it + install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH} - install ${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2 ${SDK_OUTPUT}/${SDKPATH} + install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH} } +do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True" -# Since bitbake won't run as root it doesn't make sense to try and install -# the extensible sdk as root. sdk_ext_preinst() { + # Since bitbake won't run as root it doesn't make sense to try and install + # the extensible sdk as root. if [ "`id -u`" = "0" ]; then echo "ERROR: The extensible sdk cannot be installed as root." exit 1 fi + if ! command -v locale > /dev/null; then + echo "ERROR: The installer requires the locale command, please install it first" + exit 1 + fi + # Check setting of LC_ALL set above + canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'` + if ! locale -a | grep -q $canonicalised_locale ; then + echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first" + exit 1 + fi + # The relocation script used by buildtools installer requires python + if ! command -v python > /dev/null; then + echo "ERROR: The installer requires python, please install it first" + exit 1 + fi + missing_utils="" + for util in ${SDK_REQUIRED_UTILITIES}; do + if ! command -v $util > /dev/null; then + missing_utils="$missing_utils $util" + fi + done + if [ -n "$missing_utils" ] ; then + echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils" + exit 1 + fi SDK_EXTENSIBLE="1" + if [ "$publish" = "1" ] ; then + EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=ext-sdk-prepare.py" + if [ "${SDK_EXT_TYPE}" = "minimal" ] ; then + EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache" + fi + fi } SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_preinst}" @@ -189,27 +563,41 @@ SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_preinst}" sdk_ext_postinst() { printf "\nExtracting buildtools...\n" cd $target_sdk_dir - printf "buildtools\ny" | ./*buildtools-tarball* > /dev/null + env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}" + printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } + + # Delete the buildtools tar file since it won't be used again + rm -f ./${SDK_BUILDTOOLS_INSTALLER} + # We don't need the log either since it succeeded + rm -f buildtools.log # Make sure when the user sets up the environment, they also get # the buildtools-tarball tools in their path. - echo ". $target_sdk_dir/buildtools/environment-setup*" >> $target_sdk_dir/environment-setup* + echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script # Allow bitbake environment setup to be ran as part of this sdk. - echo "export OE_SKIP_SDK_CHECK=1" >> $target_sdk_dir/environment-setup* + echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script + # Work around runqemu not knowing how to get this information within the eSDK + echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script # A bit of another hack, but we need this in the path only for devtool # so put it at the end of $PATH. - echo "export PATH=\$PATH:$target_sdk_dir/sysroots/${SDK_SYS}/${bindir_nativesdk}" >> $target_sdk_dir/environment-setup* + echo "export PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $env_setup_script + + echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script - # For now this is where uninative.bbclass expects the tarball - mv *-nativesdk-libc.tar.* $target_sdk_dir/`dirname ${oe_init_build_env_path}` + # Warn if trying to use external bitbake and the ext SDK together + echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script - printf "Preparing build system...\n" - # dash which is /bin/sh on Ubuntu will not preserve the - # current working directory when first ran, nor will it set $1 when - # sourcing a script. That is why this has to look so ugly. - sh -c ". buildtools/environment-setup* > preparing_build_system.log && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> preparing_build_system.log && bitbake ${SDK_TARGETS} >> preparing_build_system.log" || { echo "SDK preparation failed: see `pwd`/preparing_build_system.log" ; exit 1 ; } + if [ "$prepare_buildsystem" != "no" ]; then + printf "Preparing build system...\n" + # dash which is /bin/sh on Ubuntu will not preserve the + # current working directory when first ran, nor will it set $1 when + # sourcing a script. That is why this has to look so ugly. + LOGFILE="$target_sdk_dir/preparing_build_system.log" + sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } + rm $target_sdk_dir/ext-sdk-prepare.py + fi echo done } @@ -217,28 +605,84 @@ SDK_POST_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_postinst}" SDK_POSTPROCESS_COMMAND_prepend_task-populate-sdk-ext = "copy_buildsystem; install_tools; " +SDK_INSTALL_TARGETS = "" fakeroot python do_populate_sdk_ext() { - bb.build.exec_func("do_populate_sdk", d) + # FIXME hopefully we can remove this restriction at some point, but uninative + # currently forces this upon us + if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'): + bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH'))) + + d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d)) + buildtools_fn = get_current_buildtools(d) + d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d)) + d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn) + d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}') + + populate_sdk_common(d) +} + +def get_ext_sdk_depends(d): + # Note: the deps varflag is a list not a string, so we need to specify expand=False + deps = d.getVarFlag('do_image_complete', 'deps', False) + pn = d.getVar('PN') + deplist = ['%s:%s' % (pn, dep) for dep in deps] + tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) + tasklist.append('do_rootfs') + for task in tasklist: + deplist.extend((d.getVarFlag(task, 'depends') or '').split()) + return ' '.join(deplist) + +python do_sdk_depends() { + # We have to do this separately in its own task so we avoid recursing into + # dependencies we don't need to (e.g. buildtools-tarball) and bringing those + # into the SDK's sstate-cache + import oe.copy_buildsystem + sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' + oe.copy_buildsystem.generate_locked_sigs(sigfile, d) } +addtask sdk_depends + +do_sdk_depends[dirs] = "${WORKDIR}" +do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot" +do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" +do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}" +do_sdk_depends[rdepends] = "${@get_sdk_ext_rdepends(d)}" def get_sdk_ext_rdepends(d): localdata = d.createCopy() localdata.appendVar('OVERRIDES', ':task-populate-sdk-ext') - bb.data.update_data(localdata) - return localdata.getVarFlag('do_populate_sdk', 'rdepends', True) + return localdata.getVarFlag('do_populate_sdk', 'rdepends') do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}" -do_populate_sdk_ext[depends] += "${@d.getVarFlag('do_populate_sdk', 'depends', False)}" -do_populate_sdk_ext[rdepends] = "${@get_sdk_ext_rdepends(d)}" -do_populate_sdk_ext[recrdeptask] += "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" - - -do_populate_sdk_ext[depends] += "buildtools-tarball:do_populate_sdk uninative-tarball:do_populate_sdk" - -do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':do_build' for x in d.getVar('SDK_TARGETS', True).split()])}" -do_populate_sdk_ext[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy" - -# Make sure codes change in copy_buildsystem can result in rebuilt -do_populate_sdk_ext[vardeps] += "copy_buildsystem" -addtask populate_sdk_ext +do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \ + buildtools-tarball:do_populate_sdk uninative-tarball:do_populate_sdk \ + ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \ + ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}" + +# We must avoid depending on do_build here if rm_work.bbclass is active, +# because otherwise do_rm_work may run before do_populate_sdk_ext itself. +# We can't mark do_populate_sdk_ext and do_sdk_depends as having to +# run before do_rm_work, because then they would also run as part +# of normal builds. +do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}" + +# Make sure code changes can result in rebuild +do_populate_sdk_ext[vardeps] += "copy_buildsystem \ + sdk_ext_postinst" + +# Since any change in the metadata of any layer should cause a rebuild of the +# sdk(since the layers are put in the sdk) set the task to nostamp so it +# always runs. +do_populate_sdk_ext[nostamp] = "1" + +SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext" + +SSTATETASKS += "do_populate_sdk_ext" +SSTATE_SKIP_CREATION_task-populate-sdk-ext = '1' +do_populate_sdk_ext[cleandirs] = "${SDKDEPLOYDIR}" +do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}" +do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}" +do_populate_sdk_ext[stamp-extra-info] = "${MACHINE}" + +addtask populate_sdk_ext after do_sdk_depends diff --git a/meta/classes/prexport.bbclass b/meta/classes/prexport.bbclass index 5a1cb33c6a..6dcf99e29f 100644 --- a/meta/classes/prexport.bbclass +++ b/meta/classes/prexport.bbclass @@ -8,20 +8,21 @@ PRSERV_DUMPFILE ??= "${PRSERV_DUMPDIR}/prserv.inc" python prexport_handler () { import bb.event - if not e.data: + if not e.data or bb.data.inherits_class('native', e.data) or \ + bb.data.inherits_class('crosssdk', e.data): return if isinstance(e, bb.event.RecipeParsed): import oe.prservice #get all PR values for the current PRAUTOINX - ver = e.data.getVar('PRSERV_DUMPOPT_VERSION', True) + ver = e.data.getVar('PRSERV_DUMPOPT_VERSION') ver = ver.replace('%','-') retval = oe.prservice.prserv_dump_db(e.data) if not retval: bb.fatal("prexport_handler: export failed!") (metainfo, datainfo) = retval if not datainfo: - bb.warn("prexport_handler: No AUTOPR values found for %s" % ver) + bb.note("prexport_handler: No AUTOPR values found for %s" % ver) return oe.prservice.prserv_export_tofile(e.data, None, datainfo, False) if 'AUTOINC' in ver: @@ -39,7 +40,7 @@ python prexport_handler () { import oe.prservice oe.prservice.prserv_check_avail(e.data) #remove dumpfile - bb.utils.remove(e.data.getVar('PRSERV_DUMPFILE', True)) + bb.utils.remove(e.data.getVar('PRSERV_DUMPFILE')) elif isinstance(e, bb.event.ParseCompleted): import oe.prservice #dump meta info of tables diff --git a/meta/classes/prserv.bbclass b/meta/classes/prserv.bbclass deleted file mode 100644 index 139597f9cb..0000000000 --- a/meta/classes/prserv.bbclass +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/meta/classes/ptest-gnome.bbclass b/meta/classes/ptest-gnome.bbclass index b2949af9bb..478a33474d 100644 --- a/meta/classes/ptest-gnome.bbclass +++ b/meta/classes/ptest-gnome.bbclass @@ -1,6 +1,6 @@ inherit ptest -EXTRA_OECONF_append_class-target = " ${@bb.utils.contains('PTEST_ENABLED', '1', '--enable-installed-tests', '--disable-installed-tests', d)}" +EXTRA_OECONF_append = " ${@bb.utils.contains('PTEST_ENABLED', '1', '--enable-installed-tests', '--disable-installed-tests', d)}" FILES_${PN}-ptest += "${libexecdir}/installed-tests/ \ ${datadir}/installed-tests/" diff --git a/meta/classes/ptest.bbclass b/meta/classes/ptest.bbclass index 2ac9143abe..c19f65b9bb 100644 --- a/meta/classes/ptest.bbclass +++ b/meta/classes/ptest.bbclass @@ -2,7 +2,7 @@ SUMMARY_${PN}-ptest ?= "${SUMMARY} - Package test files" DESCRIPTION_${PN}-ptest ?= "${DESCRIPTION} \ This package contains a test directory ${PTEST_PATH} for package test purposes." -PTEST_PATH ?= "${libdir}/${PN}/ptest" +PTEST_PATH ?= "${libdir}/${BPN}/ptest" FILES_${PN}-ptest = "${PTEST_PATH}" SECTION_${PN}-ptest = "devel" ALLOW_EMPTY_${PN}-ptest = "1" @@ -39,11 +39,12 @@ do_install_ptest() { do_install_ptest_base() { if [ -f ${WORKDIR}/run-ptest ]; then install -D ${WORKDIR}/run-ptest ${D}${PTEST_PATH}/run-ptest - if grep -q install-ptest: Makefile; then - oe_runmake DESTDIR=${D}${PTEST_PATH} install-ptest - fi - do_install_ptest fi + if grep -q install-ptest: Makefile; then + oe_runmake DESTDIR=${D}${PTEST_PATH} install-ptest + fi + do_install_ptest + chown -R root:root ${D}${PTEST_PATH} } do_configure_ptest_base[dirs] = "${B}" @@ -57,10 +58,10 @@ addtask install_ptest_base after do_install before do_package do_populate_sy python () { if not bb.data.inherits_class('native', d) and not bb.data.inherits_class('cross', d): - d.setVarFlag('do_install_ptest_base', 'fakeroot', 1) + d.setVarFlag('do_install_ptest_base', 'fakeroot', '1') # Remove all '*ptest_base' tasks when ptest is not enabled - if not(d.getVar('PTEST_ENABLED', True) == "1"): + if not(d.getVar('PTEST_ENABLED') == "1"): for i in ['do_configure_ptest_base', 'do_compile_ptest_base', 'do_install_ptest_base']: bb.build.deltask(i, d) } diff --git a/meta/classes/python-dir.bbclass b/meta/classes/python-dir.bbclass index ebfa4b30f6..a11dc350be 100644 --- a/meta/classes/python-dir.bbclass +++ b/meta/classes/python-dir.bbclass @@ -1,5 +1,5 @@ -PYTHON_BASEVERSION ?= "2.7" -PYTHON_ABI ?= "" +PYTHON_BASEVERSION = "2.7" +PYTHON_ABI = "" PYTHON_DIR = "python${PYTHON_BASEVERSION}" -PYTHON_PN = "python${@'' if '${PYTHON_BASEVERSION}'.startswith('2') else '3'}" +PYTHON_PN = "python" PYTHON_SITEPACKAGES_DIR = "${libdir}/${PYTHON_DIR}/site-packages" diff --git a/meta/classes/python3-dir.bbclass b/meta/classes/python3-dir.bbclass new file mode 100644 index 0000000000..06bb046d9c --- /dev/null +++ b/meta/classes/python3-dir.bbclass @@ -0,0 +1,5 @@ +PYTHON_BASEVERSION = "3.5" +PYTHON_ABI = "m" +PYTHON_DIR = "python${PYTHON_BASEVERSION}" +PYTHON_PN = "python3" +PYTHON_SITEPACKAGES_DIR = "${libdir}/${PYTHON_DIR}/site-packages" diff --git a/meta/classes/python3native.bbclass b/meta/classes/python3native.bbclass index cf83017675..ef468b3fde 100644 --- a/meta/classes/python3native.bbclass +++ b/meta/classes/python3native.bbclass @@ -1,7 +1,13 @@ -PYTHON_BASEVERSION = "3.4" - -inherit python-dir +inherit python3-dir PYTHON="${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}" EXTRANATIVEPATH += "${PYTHON_PN}-native" -DEPENDS += " ${PYTHON_PN}-native " +DEPENDS_append = " ${PYTHON_PN}-native " + +# python-config and other scripts are using distutils modules +# which we patch to access these variables +export STAGING_INCDIR +export STAGING_LIBDIR + +# autoconf macros will use their internal default preference otherwise +export PYTHON diff --git a/meta/classes/pythonnative.bbclass b/meta/classes/pythonnative.bbclass index 97029dc525..4e0381b568 100644 --- a/meta/classes/pythonnative.bbclass +++ b/meta/classes/pythonnative.bbclass @@ -5,4 +5,12 @@ PYTHON="${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}" # PYTHON_EXECUTABLE is used by cmake PYTHON_EXECUTABLE="${PYTHON}" EXTRANATIVEPATH += "${PYTHON_PN}-native" -DEPENDS += " ${PYTHON_PN}-native " +DEPENDS_append = " ${PYTHON_PN}-native " + +# python-config and other scripts are using distutils modules +# which we patch to access these variables +export STAGING_INCDIR +export STAGING_LIBDIR + +# autoconf macros will use their internal default preference otherwise +export PYTHON diff --git a/meta/classes/qemu.bbclass b/meta/classes/qemu.bbclass index 601f587534..f5c5780125 100644 --- a/meta/classes/qemu.bbclass +++ b/meta/classes/qemu.bbclass @@ -4,7 +4,12 @@ # def qemu_target_binary(data): - target_arch = data.getVar("TARGET_ARCH", True) + package_arch = data.getVar("PACKAGE_ARCH") + qemu_target_binary = (data.getVar("QEMU_TARGET_BINARY_%s" % package_arch) or "") + if qemu_target_binary: + return qemu_target_binary + + target_arch = data.getVar("TARGET_ARCH") if target_arch in ("i486", "i586", "i686"): target_arch = "i386" elif target_arch == "powerpc": @@ -13,7 +18,19 @@ def qemu_target_binary(data): target_arch = "ppc64" return "qemu-" + target_arch -# + +def qemu_wrapper_cmdline(data, rootfs_path, library_paths): + import string + + qemu_binary = qemu_target_binary(data) + if qemu_binary == "qemu-allarch": + qemu_binary = "qemuwrapper" + + qemu_options = data.getVar("QEMU_OPTIONS") + + return "PSEUDO_UNLOAD=1 " + qemu_binary + " " + qemu_options + " -L " + rootfs_path\ + + " -E LD_LIBRARY_PATH=" + ":".join(library_paths) + " " + # Next function will return a string containing the command that is needed to # to run a certain binary through qemu. For example, in order to make a certain # postinstall scriptlet run at do_rootfs time and running the postinstall is @@ -23,29 +40,25 @@ def qemu_target_binary(data): # ${@qemu_run_binary(d, '$D', '/usr/bin/test_app')} [test_app arguments] # def qemu_run_binary(data, rootfs_path, binary): - qemu_binary = qemu_target_binary(data) - if qemu_binary == "qemu-allarch": - qemu_binary = "qemuwrapper" - libdir = rootfs_path + data.getVar("libdir", False) base_libdir = rootfs_path + data.getVar("base_libdir", False) - qemu_options = data.getVar("QEMU_OPTIONS", True) - return "PSEUDO_UNLOAD=1 " + qemu_binary + " " + qemu_options + " -L " + rootfs_path\ - + " -E LD_LIBRARY_PATH=" + libdir + ":" + base_libdir + " "\ - + rootfs_path + binary - -# QEMU_EXTRAOPTIONS is not meant to be directly used, the extensions are -# PACKAGE_ARCH, not overrides and hence have to do this dance. Simply being arch -# specific isn't good enough. -QEMU_OPTIONS = "-r ${OLDEST_KERNEL} ${@d.getVar("QEMU_EXTRAOPTIONS_%s" % d.getVar('PACKAGE_ARCH', True), True) or ""}" -QEMU_EXTRAOPTIONS_iwmmxt = " -cpu pxa270-c5" -QEMU_EXTRAOPTIONS_armv6 = " -cpu arm1136" -QEMU_EXTRAOPTIONS_armv7a = " -cpu cortex-a8" -QEMU_EXTRAOPTIONS_e500v2 = " -cpu e500v2" -QEMU_EXTRAOPTIONS_e500mc = " -cpu e500mc" -QEMU_EXTRAOPTIONS_e5500 = " -cpu e5500" -QEMU_EXTRAOPTIONS_e5500-64b = " -cpu e5500" -QEMU_EXTRAOPTIONS_e6500 = " -cpu e6500" -QEMU_EXTRAOPTIONS_e6500-64b = " -cpu e6500" -QEMU_EXTRAOPTIONS_ppc7400 = " -cpu 7400" + return qemu_wrapper_cmdline(data, rootfs_path, [libdir, base_libdir]) + rootfs_path + binary + +# QEMU_EXTRAOPTIONS is not meant to be directly used, the extensions are +# PACKAGE_ARCH, *NOT* overrides. +# In some cases (e.g. ppc) simply being arch specific (apparently) isn't good +# enough and a PACKAGE_ARCH specific -cpu option is needed (hence we have to do +# this dance). For others (e.g. arm) a -cpu option is not necessary, since the +# qemu-arm default CPU supports all required architecture levels. + +QEMU_OPTIONS = "-r ${OLDEST_KERNEL} ${@d.getVar("QEMU_EXTRAOPTIONS_%s" % d.getVar('PACKAGE_ARCH')) or ""}" +QEMU_OPTIONS[vardeps] += "QEMU_EXTRAOPTIONS_${PACKAGE_ARCH}" + +QEMU_EXTRAOPTIONS_ppce500v2 = " -cpu e500v2" +QEMU_EXTRAOPTIONS_ppce500mc = " -cpu e500mc" +QEMU_EXTRAOPTIONS_ppce5500 = " -cpu e500mc" +QEMU_EXTRAOPTIONS_ppc64e5500 = " -cpu e500mc" +QEMU_EXTRAOPTIONS_ppce6500 = " -cpu e500mc" +QEMU_EXTRAOPTIONS_ppc64e6500 = " -cpu e500mc" +QEMU_EXTRAOPTIONS_ppc7400 = " -cpu 7400" diff --git a/meta/classes/qemuboot.bbclass b/meta/classes/qemuboot.bbclass new file mode 100644 index 0000000000..86b306037f --- /dev/null +++ b/meta/classes/qemuboot.bbclass @@ -0,0 +1,120 @@ +# Help runqemu boot target board, "QB" means Qemu Boot, the following +# vars can be set in conf files, such as <bsp.conf> to make it can be +# boot by runqemu: +# +# QB_SYSTEM_NAME: qemu name, e.g., "qemu-system-i386" +# +# QB_OPT_APPEND: options to append to qemu, e.g., "-show-cursor" +# +# QB_DEFAULT_KERNEL: default kernel to boot, e.g., "bzImage" +# +# QB_DEFAULT_FSTYPE: default FSTYPE to boot, e.g., "ext4" +# +# QB_MEM: memory, e.g., "-m 512" +# +# QB_MACHINE: qemu machine, e.g., "-machine virt" +# +# QB_CPU: qemu cpu, e.g., "-cpu qemu32" +# +# QB_CPU_KVM: the similar to QB_CPU, but used when kvm, e.g., '-cpu kvm64', +# set it when support kvm. +# +# QB_KERNEL_CMDLINE_APPEND: options to append to kernel's -append +# option, e.g., "console=ttyS0 console=tty" +# +# QB_DTB: qemu dtb name +# +# QB_AUDIO_DRV: qemu audio driver, e.g., "alsa", set it when support audio +# +# QB_AUDIO_OPT: qemu audio option, e.g., "-soundhw ac97,es1370", used +# when QB_AUDIO_DRV is set. +# +# QB_KERNEL_ROOT: kernel's root, e.g., /dev/vda +# +# QB_NETWORK_DEVICE: network device, e.g., "-device virtio-net-pci,netdev=net0,mac=@MAC@", +# it needs work with QB_TAP_OPT and QB_SLIRP_OPT. +# Note, runqemu will replace @MAC@ with a predefined mac, you can set +# a custom one, but that may cause conflicts when multiple qemus are +# running on the same host. +# +# QB_TAP_OPT: netowrk option for 'tap' mode, e.g., +# "-netdev tap,id=net0,ifname=@TAP@,script=no,downscript=no" +# Note, runqemu will replace "@TAP@" with the one which is used, such as tap0, tap1 ... +# +# QB_SLIRP_OPT: network option for SLIRP mode, e.g., -netdev user,id=net0" +# +# QB_ROOTFS_OPT: used as rootfs, e.g., +# "-drive id=disk0,file=@ROOTFS@,if=none,format=raw -device virtio-blk-device,drive=disk0" +# Note, runqemu will replace "@ROOTFS@" with the one which is used, such as core-image-minimal-qemuarm64.ext4. +# +# QB_SERIAL_OPT: serial port, e.g., "-serial mon:stdio" +# +# QB_TCPSERIAL_OPT: tcp serial port option, e.g., +# " -device virtio-serial-device -chardev socket,id=virtcon,port=@PORT@,host=127.0.0.1 -device virtconsole,chardev=virtcon" +# Note, runqemu will replace "@PORT@" with the port number which is used. +# +# Usage: +# IMAGE_CLASSES += "qemuboot" +# See "runqemu help" for more info + +QB_MEM ?= "-m 256" +QB_SERIAL_OPT ?= "-serial mon:stdio -serial null" +QB_DEFAULT_KERNEL ?= "${KERNEL_IMAGETYPE}" +QB_DEFAULT_FSTYPE ?= "ext4" +QB_OPT_APPEND ?= "-show-cursor" +QB_NETWORK_DEVICE ?= "-device virtio-net-pci,netdev=net0,mac=@MAC@" + +# This should be kept align with ROOT_VM +QB_DRIVE_TYPE ?= "/dev/sd" + +# Create qemuboot.conf +addtask do_write_qemuboot_conf after do_rootfs before do_image +IMGDEPLOYDIR ?= "${WORKDIR}/deploy-${PN}-image-complete" + +def qemuboot_vars(d): + build_vars = ['MACHINE', 'TUNE_ARCH', 'DEPLOY_DIR_IMAGE', + 'KERNEL_IMAGETYPE', 'IMAGE_NAME', 'IMAGE_LINK_NAME', + 'STAGING_DIR_NATIVE', 'STAGING_BINDIR_NATIVE', + 'STAGING_DIR_HOST'] + return build_vars + [k for k in d.keys() if k.startswith('QB_')] + +do_write_qemuboot_conf[vardeps] += "${@' '.join(qemuboot_vars(d))}" +python do_write_qemuboot_conf() { + import configparser + + qemuboot = "%s/%s.qemuboot.conf" % (d.getVar('IMGDEPLOYDIR'), d.getVar('IMAGE_NAME')) + qemuboot_link = "%s/%s.qemuboot.conf" % (d.getVar('IMGDEPLOYDIR'), d.getVar('IMAGE_LINK_NAME')) + topdir="%s/"%(d.getVar('TOPDIR')).replace("//","/") + cf = configparser.ConfigParser() + cf.add_section('config_bsp') + for k in qemuboot_vars(d): + # qemu-helper-native sysroot is not removed by rm_work and + # contains all tools required by runqemu + if k == 'STAGING_BINDIR_NATIVE': + val = os.path.join(d.getVar('BASE_WORKDIR'), d.getVar('BUILD_SYS'), + 'qemu-helper-native/1.0-r1/recipe-sysroot-native/usr/bin/') + else: + val = d.getVar(k) + # we only want to write out relative paths so that we can relocate images + # and still run them + val=val.replace(topdir,"") + cf.set('config_bsp', k, '%s' % val) + + # QB_DEFAULT_KERNEL's value of KERNEL_IMAGETYPE is the name of a symlink + # to the kernel file, which hinders relocatability of the qb conf. + # Read the link and replace it with the full filename of the target. + kernel_link = os.path.join(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('QB_DEFAULT_KERNEL')) + kernel = os.path.realpath(kernel_link) + # we only want to write out relative paths so that we can relocate images + # and still run them + kernel=kernel.replace(topdir,"") + cf.set('config_bsp', 'QB_DEFAULT_KERNEL', kernel) + + bb.utils.mkdirhier(os.path.dirname(qemuboot)) + with open(qemuboot, 'w') as f: + cf.write(f) + + if os.path.lexists(qemuboot_link): + os.remove(qemuboot_link) + os.symlink(os.path.basename(qemuboot), qemuboot_link) +} diff --git a/meta/classes/qmake2.bbclass b/meta/classes/qmake2.bbclass deleted file mode 100644 index 6e73ad2d1e..0000000000 --- a/meta/classes/qmake2.bbclass +++ /dev/null @@ -1,27 +0,0 @@ -# -# QMake variables for Qt4 -# -inherit qmake_base - -DEPENDS_prepend = "qt4-tools-native " - -export QMAKESPEC = "${STAGING_DATADIR}/qt4/mkspecs/${TARGET_OS}-oe-g++" -export OE_QMAKE_QT_CONFIG = "${STAGING_DATADIR}/qt4/mkspecs/qconfig.pri" -export OE_QMAKE_UIC = "${STAGING_BINDIR_NATIVE}/uic4" -export OE_QMAKE_UIC3 = "${STAGING_BINDIR_NATIVE}/uic34" -export OE_QMAKE_MOC = "${STAGING_BINDIR_NATIVE}/moc4" -export OE_QMAKE_RCC = "${STAGING_BINDIR_NATIVE}/rcc4" -export OE_QMAKE_QDBUSCPP2XML = "${STAGING_BINDIR_NATIVE}/qdbuscpp2xml4" -export OE_QMAKE_QDBUSXML2CPP = "${STAGING_BINDIR_NATIVE}/qdbusxml2cpp4" -export OE_QMAKE_QMAKE = "${STAGING_BINDIR_NATIVE}/qmake2" -export OE_QMAKE_LINK = "${CXX}" -export OE_QMAKE_CXXFLAGS = "${CXXFLAGS}" -export OE_QMAKE_INCDIR_QT = "${STAGING_INCDIR}/qt4" -export OE_QMAKE_LIBDIR_QT = "${STAGING_LIBDIR}" -export OE_QMAKE_LIBS_QT = "qt" -export OE_QMAKE_LIBS_X11 = "-lXext -lX11 -lm" -export OE_QMAKE_LIBS_X11SM = "-lSM -lICE" -export OE_QMAKE_LCONVERT = "${STAGING_BINDIR_NATIVE}/lconvert4" -export OE_QMAKE_LRELEASE = "${STAGING_BINDIR_NATIVE}/lrelease4" -export OE_QMAKE_LUPDATE = "${STAGING_BINDIR_NATIVE}/lupdate4" -export OE_QMAKE_XMLPATTERNS = "${STAGING_BINDIR_NATIVE}/xmlpatterns4" diff --git a/meta/classes/qmake_base.bbclass b/meta/classes/qmake_base.bbclass deleted file mode 100644 index dc98713fd2..0000000000 --- a/meta/classes/qmake_base.bbclass +++ /dev/null @@ -1,119 +0,0 @@ -QMAKE_MKSPEC_PATH ?= "${STAGING_DATADIR_NATIVE}/qmake" - -OE_QMAKE_PLATFORM = "${TARGET_OS}-oe-g++" -QMAKESPEC := "${QMAKE_MKSPEC_PATH}/${OE_QMAKE_PLATFORM}" - -# We override this completely to eliminate the -e normally passed in -EXTRA_OEMAKE = "" - -export OE_QMAKE_CC="${CC}" -export OE_QMAKE_CFLAGS="${CFLAGS}" -export OE_QMAKE_CXX="${CXX}" -export OE_QMAKE_LDFLAGS="${LDFLAGS}" -export OE_QMAKE_AR="${AR}" -export OE_QMAKE_STRIP="echo" -export OE_QMAKE_RPATH="-Wl,-rpath-link," - -# default to qte2 via bb.conf, inherit qt3x11 to configure for qt3x11 - -oe_qmake_mkspecs () { - mkdir -p mkspecs/${OE_QMAKE_PLATFORM} - for f in ${QMAKE_MKSPEC_PATH}/${OE_QMAKE_PLATFORM}/*; do - if [ -L $f ]; then - lnk=`readlink $f` - if [ -f mkspecs/${OE_QMAKE_PLATFORM}/$lnk ]; then - ln -s $lnk mkspecs/${OE_QMAKE_PLATFORM}/`basename $f` - else - cp $f mkspecs/${OE_QMAKE_PLATFORM}/ - fi - else - cp $f mkspecs/${OE_QMAKE_PLATFORM}/ - fi - done -} - -do_generate_qt_config_file() { - export QT_CONF_PATH=${WORKDIR}/qt.conf - cat > ${WORKDIR}/qt.conf <<EOF -[Paths] -Prefix = -Binaries = ${STAGING_BINDIR_NATIVE} -Headers = ${STAGING_INCDIR}/${QT_DIR_NAME} -Plugins = ${STAGING_LIBDIR}/${QT_DIR_NAME}/plugins/ -Mkspecs = ${STAGING_DATADIR}/${QT_DIR_NAME}/mkspecs/ -EOF -} - -addtask generate_qt_config_file after do_patch before do_configure - -qmake_base_do_configure() { - case ${QMAKESPEC} in - *linux-oe-g++|*linux-uclibc-oe-g++|*linux-gnueabi-oe-g++|*linux-uclibceabi-oe-g++|*linux-gnuspe-oe-g++|*linux-uclibcspe-oe-g++|*linux-gnun32-oe-g++) - ;; - *-oe-g++) - die Unsupported target ${TARGET_OS} for oe-g++ qmake spec - ;; - *) - bbnote Searching for qmake spec file - paths="${QMAKE_MKSPEC_PATH}/qws/${TARGET_OS}-${TARGET_ARCH}-g++" - paths="${QMAKE_MKSPEC_PATH}/${TARGET_OS}-g++ $paths" - - if (echo "${TARGET_ARCH}"|grep -q 'i.86'); then - paths="${QMAKE_MKSPEC_PATH}/qws/${TARGET_OS}-x86-g++ $paths" - fi - for i in $paths; do - if test -e $i; then - export QMAKESPEC=$i - break - fi - done - ;; - esac - - bbnote "using qmake spec in ${QMAKESPEC}, using profiles '${QMAKE_PROFILES}'" - - if [ -z "${QMAKE_PROFILES}" ]; then - PROFILES="`ls *.pro`" - else - PROFILES="${QMAKE_PROFILES}" - fi - - if [ -z "$PROFILES" ]; then - die "QMAKE_PROFILES not set and no profiles found in $PWD" - fi - - if [ ! -z "${EXTRA_QMAKEVARS_POST}" ]; then - AFTER="-after" - QMAKE_VARSUBST_POST="${EXTRA_QMAKEVARS_POST}" - bbnote "qmake postvar substitution: ${EXTRA_QMAKEVARS_POST}" - fi - - if [ ! -z "${EXTRA_QMAKEVARS_PRE}" ]; then - QMAKE_VARSUBST_PRE="${EXTRA_QMAKEVARS_PRE}" - bbnote "qmake prevar substitution: ${EXTRA_QMAKEVARS_PRE}" - fi - - # Hack .pro files to use OE utilities - LCONVERT_NAME=$(basename ${OE_QMAKE_LCONVERT}) - LRELEASE_NAME=$(basename ${OE_QMAKE_LRELEASE}) - LUPDATE_NAME=$(basename ${OE_QMAKE_LUPDATE}) - XMLPATTERNS_NAME=$(basename ${OE_QMAKE_XMLPATTERNS}) - find -name '*.pro' \ - -exec sed -i -e "s|\(=\s*.*\)/$LCONVERT_NAME|\1/lconvert|g" \ - -e "s|\(=\s*.*\)/$LRELEASE_NAME|\1/lrelease|g" \ - -e "s|\(=\s*.*\)/$LUPDATE_NAME|\1/lupdate|g" \ - -e "s|\(=\s*.*\)/$XMLPATTERNS_NAME|\1/xmlpatterns|g" \ - -e "s|\(=\s*.*\)/lconvert|\1/$LCONVERT_NAME|g" \ - -e "s|\(=\s*.*\)/lrelease|\1/$LRELEASE_NAME|g" \ - -e "s|\(=\s*.*\)/lupdate|\1/$LUPDATE_NAME|g" \ - -e "s|\(=\s*.*\)/xmlpatterns|\1/$XMLPATTERNS_NAME|g" \ - '{}' ';' - -#bbnote "Calling '${OE_QMAKE_QMAKE} -makefile -spec ${QMAKESPEC} -o Makefile $QMAKE_VARSUBST_PRE $AFTER $PROFILES $QMAKE_VARSUBST_POST'" - unset QMAKESPEC || true - ${OE_QMAKE_QMAKE} -makefile -spec ${QMAKESPEC} -o Makefile $QMAKE_VARSUBST_PRE $AFTER $PROFILES $QMAKE_VARSUBST_POST || die "Error calling ${OE_QMAKE_QMAKE} on $PROFILES" -} - -EXPORT_FUNCTIONS do_configure - -addtask configure after do_unpack do_patch before do_compile diff --git a/meta/classes/qt4e.bbclass b/meta/classes/qt4e.bbclass deleted file mode 100644 index 13b1050aac..0000000000 --- a/meta/classes/qt4e.bbclass +++ /dev/null @@ -1,21 +0,0 @@ -QT4EDEPENDS ?= "qt4-embedded " -DEPENDS_prepend = "${QT4EDEPENDS}" - -inherit qmake2 - -QT_BASE_NAME = "qt4-embedded" -QT_DIR_NAME = "qtopia" -QT_LIBINFIX = "E" -# override variables set by qmake-base to compile Qt/Embedded apps -# -export QMAKESPEC = "${STAGING_DATADIR}/${QT_DIR_NAME}/mkspecs/${TARGET_OS}-oe-g++" -export OE_QMAKE_QT_CONFIG = "${STAGING_DATADIR}/${QT_DIR_NAME}/mkspecs/qconfig.pri" -export OE_QMAKE_INCDIR_QT = "${STAGING_INCDIR}/${QT_DIR_NAME}" -export OE_QMAKE_LIBDIR_QT = "${STAGING_LIBDIR}" -export OE_QMAKE_LIBS_QT = "qt" -export OE_QMAKE_LIBS_X11 = "" -export OE_QMAKE_EXTRA_MODULES = "network" -EXTRA_QMAKEVARS_PRE += " QT_LIBINFIX=${QT_LIBINFIX} " - -# Qt4 uses atomic instructions not supported in thumb mode -ARM_INSTRUCTION_SET = "arm" diff --git a/meta/classes/qt4x11.bbclass b/meta/classes/qt4x11.bbclass deleted file mode 100644 index 6f06d34755..0000000000 --- a/meta/classes/qt4x11.bbclass +++ /dev/null @@ -1,14 +0,0 @@ -QT4DEPENDS ?= "qt4-x11 " -DEPENDS_prepend = "${QT4DEPENDS}" - -# depends on qt4-x11 -REQUIRED_DISTRO_FEATURES += "x11" - -inherit qmake2 distro_features_check - -QT_BASE_NAME = "qt4" -QT_DIR_NAME = "qt4" -QT_LIBINFIX = "" - -# Qt4 uses atomic instructions not supported in thumb mode -ARM_INSTRUCTION_SET = "arm" diff --git a/meta/classes/recipe_sanity.bbclass b/meta/classes/recipe_sanity.bbclass index ee04e333db..7fa4a849ea 100644 --- a/meta/classes/recipe_sanity.bbclass +++ b/meta/classes/recipe_sanity.bbclass @@ -1,5 +1,5 @@ def __note(msg, d): - bb.note("%s: recipe_sanity: %s" % (d.getVar("P", True), msg)) + bb.note("%s: recipe_sanity: %s" % (d.getVar("P"), msg)) __recipe_sanity_badruntimevars = "RDEPENDS RPROVIDES RRECOMMENDS RCONFLICTS" def bad_runtime_vars(cfgdata, d): @@ -7,20 +7,20 @@ def bad_runtime_vars(cfgdata, d): bb.data.inherits_class("cross", d): return - for var in d.getVar("__recipe_sanity_badruntimevars", True).split(): - val = d.getVar(var, 0) + for var in d.getVar("__recipe_sanity_badruntimevars").split(): + val = d.getVar(var, False) if val and val != cfgdata.get(var): __note("%s should be %s_${PN}" % (var, var), d) __recipe_sanity_reqvars = "DESCRIPTION" __recipe_sanity_reqdiffvars = "" def req_vars(cfgdata, d): - for var in d.getVar("__recipe_sanity_reqvars", True).split(): - if not d.getVar(var, 0): + for var in d.getVar("__recipe_sanity_reqvars").split(): + if not d.getVar(var, False): __note("%s should be set" % var, d) - for var in d.getVar("__recipe_sanity_reqdiffvars", True).split(): - val = d.getVar(var, 0) + for var in d.getVar("__recipe_sanity_reqdiffvars").split(): + val = d.getVar(var, False) cfgval = cfgdata.get(var) if not val: @@ -29,7 +29,7 @@ def req_vars(cfgdata, d): __note("%s should be defined to something other than default (%s)" % (var, cfgval), d) def var_renames_overwrite(cfgdata, d): - renames = d.getVar("__recipe_sanity_renames", 0) + renames = d.getVar("__recipe_sanity_renames", False) if renames: for (key, newkey, oldvalue, newvalue) in renames: if oldvalue != newvalue and oldvalue != cfgdata.get(newkey): @@ -38,11 +38,11 @@ def var_renames_overwrite(cfgdata, d): def incorrect_nonempty_PACKAGES(cfgdata, d): if bb.data.inherits_class("native", d) or \ bb.data.inherits_class("cross", d): - if d.getVar("PACKAGES", True): + if d.getVar("PACKAGES"): return True def can_use_autotools_base(cfgdata, d): - cfg = d.getVar("do_configure", True) + cfg = d.getVar("do_configure") if not bb.data.inherits_class("autotools", d): return False @@ -50,7 +50,7 @@ def can_use_autotools_base(cfgdata, d): if cfg.find(i) != -1: return False - for clsfile in d.getVar("__inherit_cache", 0): + for clsfile in d.getVar("__inherit_cache", False): (base, _) = os.path.splitext(os.path.basename(clsfile)) if cfg.find("%s_do_configure" % base) != -1: __note("autotools_base usage needs verification, spotted %s_do_configure" % base, d) @@ -60,8 +60,8 @@ def can_use_autotools_base(cfgdata, d): def can_delete_FILESPATH(cfgdata, d): expected = cfgdata.get("FILESPATH") expectedpaths = d.expand(expected) - unexpanded = d.getVar("FILESPATH", 0) - filespath = d.getVar("FILESPATH", True).split(":") + unexpanded = d.getVar("FILESPATH", False) + filespath = d.getVar("FILESPATH").split(":") filespath = [os.path.normpath(f) for f in filespath if os.path.exists(f)] for fp in filespath: if not fp in expectedpaths: @@ -70,33 +70,17 @@ def can_delete_FILESPATH(cfgdata, d): return False return expected != unexpanded -def can_delete_FILESDIR(cfgdata, d): - expected = cfgdata.get("FILESDIR") - #expected = "${@bb.utils.which(d.getVar('FILESPATH', True), '.')}" - unexpanded = d.getVar("FILESDIR", 0) - if unexpanded is None: - return False - - expanded = os.path.normpath(d.getVar("FILESDIR", True)) - filespath = d.getVar("FILESPATH", True).split(":") - filespath = [os.path.normpath(f) for f in filespath if os.path.exists(f)] - - return unexpanded != expected and \ - os.path.exists(expanded) and \ - (expanded in filespath or - expanded == d.expand(expected)) - def can_delete_others(p, cfgdata, d): for k in ["S", "PV", "PN", "DESCRIPTION", "DEPENDS", "SECTION", "PACKAGES", "EXTRA_OECONF", "EXTRA_OEMAKE"]: #for k in cfgdata: - unexpanded = d.getVar(k, 0) + unexpanded = d.getVar(k, False) cfgunexpanded = cfgdata.get(k) if not cfgunexpanded: continue try: - expanded = d.getVar(k, True) + expanded = d.getVar(k) cfgexpanded = d.expand(cfgunexpanded) except bb.fetch.ParameterError: continue @@ -108,16 +92,15 @@ def can_delete_others(p, cfgdata, d): (p, cfgunexpanded, unexpanded, expanded)) python do_recipe_sanity () { - p = d.getVar("P", True) - p = "%s %s %s" % (d.getVar("PN", True), d.getVar("PV", True), d.getVar("PR", True)) + p = d.getVar("P") + p = "%s %s %s" % (d.getVar("PN"), d.getVar("PV"), d.getVar("PR")) sanitychecks = [ - (can_delete_FILESDIR, "candidate for removal of FILESDIR"), (can_delete_FILESPATH, "candidate for removal of FILESPATH"), #(can_use_autotools_base, "candidate for use of autotools_base"), (incorrect_nonempty_PACKAGES, "native or cross recipe with non-empty PACKAGES"), ] - cfgdata = d.getVar("__recipe_sanity_cfgdata", 0) + cfgdata = d.getVar("__recipe_sanity_cfgdata", False) for (func, msg) in sanitychecks: if func(cfgdata, d): @@ -143,9 +126,8 @@ python recipe_sanity_eh () { cfgdata = {} for k in d.keys(): - #for k in ["S", "PR", "PV", "PN", "DESCRIPTION", "LICENSE", "DEPENDS", - # "SECTION"]: - cfgdata[k] = d.getVar(k, 0) + if not isinstance(d.getVar(k, False), bb.data_smart.DataSmart): + cfgdata[k] = d.getVar(k, False) d.setVar("__recipe_sanity_cfgdata", cfgdata) #d.setVar("__recipe_sanity_cfgdata", d) diff --git a/meta/classes/relative_symlinks.bbclass b/meta/classes/relative_symlinks.bbclass new file mode 100644 index 0000000000..3157737347 --- /dev/null +++ b/meta/classes/relative_symlinks.bbclass @@ -0,0 +1,5 @@ +do_install[postfuncs] += "install_relative_symlinks" + +python install_relative_symlinks () { + oe.path.replace_absolute_symlinks(d.getVar('D'), d) +} diff --git a/meta/classes/relocatable.bbclass b/meta/classes/relocatable.bbclass index 4ca9981f44..582812c1cf 100644 --- a/meta/classes/relocatable.bbclass +++ b/meta/classes/relocatable.bbclass @@ -1,7 +1,18 @@ inherit chrpath -SYSROOT_PREPROCESS_FUNCS += "relocatable_binaries_preprocess" +SYSROOT_PREPROCESS_FUNCS += "relocatable_binaries_preprocess relocatable_native_pcfiles" python relocatable_binaries_preprocess() { rpath_replace(d.expand('${SYSROOT_DESTDIR}'), d) } + +relocatable_native_pcfiles () { + if [ -d ${SYSROOT_DESTDIR}${libdir}/pkgconfig ]; then + rel=${@os.path.relpath(d.getVar('base_prefix'), d.getVar('libdir') + "/pkgconfig")} + sed -i -e "s:${base_prefix}:\${pcfiledir}/$rel:g" ${SYSROOT_DESTDIR}${libdir}/pkgconfig/*.pc + fi + if [ -d ${SYSROOT_DESTDIR}${datadir}/pkgconfig ]; then + rel=${@os.path.relpath(d.getVar('base_prefix'), d.getVar('datadir') + "/pkgconfig")} + sed -i -e "s:${base_prefix}:\${pcfiledir}/$rel:g" ${SYSROOT_DESTDIR}${datadir}/pkgconfig/*.pc + fi +} diff --git a/meta/classes/remove-libtool.bbclass b/meta/classes/remove-libtool.bbclass new file mode 100644 index 0000000000..3fd0cd58f9 --- /dev/null +++ b/meta/classes/remove-libtool.bbclass @@ -0,0 +1,11 @@ +# This class removes libtool .la files after do_install + +REMOVE_LIBTOOL_LA ?= "1" + +remove_libtool_la() { + if [ "${REMOVE_LIBTOOL_LA}" != "0" ]; then + find "${D}" -ignore_readdir_race -name "*.la" -delete + fi +} + +do_install[postfuncs] += "remove_libtool_la" diff --git a/meta/classes/report-error.bbclass b/meta/classes/report-error.bbclass index cabd98cc32..d6fdd364ad 100644 --- a/meta/classes/report-error.bbclass +++ b/meta/classes/report-error.bbclass @@ -9,38 +9,41 @@ ERR_REPORT_DIR ?= "${LOG_DIR}/error-report" def errorreport_getdata(e): - logpath = e.data.getVar('ERR_REPORT_DIR', True) + import codecs + logpath = e.data.getVar('ERR_REPORT_DIR') datafile = os.path.join(logpath, "error-report.txt") - with open(datafile) as f: + with codecs.open(datafile, 'r', 'utf-8') as f: data = f.read() return data def errorreport_savedata(e, newdata, file): import json - logpath = e.data.getVar('ERR_REPORT_DIR', True) + import codecs + logpath = e.data.getVar('ERR_REPORT_DIR') datafile = os.path.join(logpath, file) - with open(datafile, "w") as f: + with codecs.open(datafile, 'w', 'utf-8') as f: json.dump(newdata, f, indent=4, sort_keys=True) return datafile python errorreport_handler () { import json + import codecs - logpath = e.data.getVar('ERR_REPORT_DIR', True) + logpath = e.data.getVar('ERR_REPORT_DIR') datafile = os.path.join(logpath, "error-report.txt") if isinstance(e, bb.event.BuildStarted): bb.utils.mkdirhier(logpath) data = {} - machine = e.data.getVar("MACHINE", True) + machine = e.data.getVar("MACHINE") data['machine'] = machine - data['build_sys'] = e.data.getVar("BUILD_SYS", True) - data['nativelsb'] = e.data.getVar("NATIVELSBSTRING", True) - data['distro'] = e.data.getVar("DISTRO", True) - data['target_sys'] = e.data.getVar("TARGET_SYS", True) + data['build_sys'] = e.data.getVar("BUILD_SYS") + data['nativelsb'] = e.data.getVar("NATIVELSBSTRING") + data['distro'] = e.data.getVar("DISTRO") + data['target_sys'] = e.data.getVar("TARGET_SYS") data['failures'] = [] - data['component'] = e.getPkgs()[0] - data['branch_commit'] = base_detect_branch(e.data) + ": " + base_detect_revision(e.data) + data['component'] = " ".join(e.getPkgs()) + data['branch_commit'] = str(base_detect_branch(e.data)) + ": " + str(base_detect_revision(e.data)) lock = bb.utils.lockfile(datafile + '.lock') errorreport_savedata(e, data, "error-report.txt") bb.utils.unlockfile(lock) @@ -48,19 +51,35 @@ python errorreport_handler () { elif isinstance(e, bb.build.TaskFailed): task = e.task taskdata={} - log = e.data.getVar('BB_LOGFILE', True) + log = e.data.getVar('BB_LOGFILE') taskdata['package'] = e.data.expand("${PF}") taskdata['task'] = task if log: try: - logFile = open(log, 'r') - taskdata['log'] = logFile.read().decode('utf-8') + logFile = codecs.open(log, 'r', 'utf-8') + logdata = logFile.read() + + # Replace host-specific paths so the logs are cleaner + for d in ("TOPDIR", "TMPDIR"): + s = e.data.getVar(d) + if s: + logdata = logdata.replace(s, d) + logFile.close() except: - taskdata['log'] = "Unable to read log file" + logdata = "Unable to read log file" else: - taskdata['log'] = "No Log" + logdata = "No Log" + + # server will refuse failures longer than param specified in project.settings.py + # MAX_UPLOAD_SIZE = "5242880" + # use lower value, because 650 chars can be spent in task, package, version + max_logdata_size = 5242000 + # upload last max_logdata_size characters + if len(logdata) > max_logdata_size: + logdata = "..." + logdata[-max_logdata_size:] + taskdata['log'] = logdata lock = bb.utils.lockfile(datafile + '.lock') jsondata = json.loads(errorreport_getdata(e)) jsondata['failures'].append(taskdata) @@ -73,7 +92,7 @@ python errorreport_handler () { bb.utils.unlockfile(lock) failures = jsondata['failures'] if(len(failures) > 0): - filename = "error_report_" + e.data.getVar("BUILDNAME", True)+".txt" + filename = "error_report_" + e.data.getVar("BUILDNAME")+".txt" datafile = errorreport_savedata(e, jsondata, filename) bb.note("The errors for this build are stored in %s\nYou can send the errors to a reports server by running:\n send-error-report %s [-s server]" % (datafile, datafile)) bb.note("The contents of these logs will be posted in public if you use the above command with the default server. Please ensure you remove any identifying or proprietary information when prompted before sending.") diff --git a/meta/classes/rm_work.bbclass b/meta/classes/rm_work.bbclass index 5e9efc1d06..badeaeba07 100644 --- a/meta/classes/rm_work.bbclass +++ b/meta/classes/rm_work.bbclass @@ -10,13 +10,21 @@ # # RM_WORK_EXCLUDE += "icu-native icu busybox" # +# Recipes can also configure which entries in their ${WORKDIR} +# are preserved besides temp, which already gets excluded by default +# because it contains logs: +# do_install_append () { +# echo "bar" >${WORKDIR}/foo +# } +# RM_WORK_EXCLUDE_ITEMS += "foo" +RM_WORK_EXCLUDE_ITEMS = "temp" # Use the completion scheduler by default when rm_work is active # to try and reduce disk usage BB_SCHEDULER ?= "completion" -RMWORK_ORIG_TASK := "${BB_DEFAULT_TASK}" -BB_DEFAULT_TASK = "rm_work_all" +# Run the rm_work task in the idle scheduling class +BB_TASK_IONICE_LEVEL_task-rm_work = "3.0" do_rm_work () { # If the recipe name is in the RM_WORK_EXCLUDE, skip the recipe. @@ -34,7 +42,7 @@ do_rm_work () { # failures of removing pseudo folers on NFS2/3 server. if [ $dir = 'pseudo' ]; then rm -rf $dir 2> /dev/null || true - elif [ $dir != 'temp' ]; then + elif ! echo '${RM_WORK_EXCLUDE_ITEMS}' | grep -q -w "$dir"; then rm -rf $dir fi done @@ -55,7 +63,7 @@ do_rm_work () { *do_setscene*) break ;; - *sigdata*) + *sigdata*|*sigbasedata*) i=dummy break ;; @@ -63,6 +71,10 @@ do_rm_work () { i=dummy break ;; + *do_rootfs*|*do_image*|*do_bootimg*|*do_bootdirectdisk*|*do_vmimg*|*do_write_qemuboot_conf*) + i=dummy + break + ;; *do_build*) i=dummy break @@ -90,13 +102,12 @@ do_rm_work () { rm -f $i done } -addtask rm_work after do_${RMWORK_ORIG_TASK} - do_rm_work_all () { : } do_rm_work_all[recrdeptask] = "do_rm_work" -addtask rm_work_all after do_rm_work +do_rm_work_all[noexec] = "1" +addtask rm_work_all after before do_build do_populate_sdk[postfuncs] += "rm_work_populatesdk" rm_work_populatesdk () { @@ -104,17 +115,58 @@ rm_work_populatesdk () { } rm_work_populatesdk[cleandirs] = "${WORKDIR}/sdk" -do_rootfs[postfuncs] += "rm_work_rootfs" +do_image_complete[postfuncs] += "rm_work_rootfs" rm_work_rootfs () { : } rm_work_rootfs[cleandirs] = "${WORKDIR}/rootfs" -python () { +# This task can be used instead of do_build to trigger building +# without also invoking do_rm_work. It only exists when rm_work.bbclass +# is active, otherwise do_build needs to be used. +# +# The intended usage is +# ${@ d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build'} +# in places that previously used just 'do_build'. +RM_WORK_BUILD_WITHOUT = "do_build_without_rm_work" +do_build_without_rm_work () { + : +} +do_build_without_rm_work[noexec] = "1" + +# We have to add these tasks already now, because all tasks are +# meant to be defined before the RecipeTaskPreProcess event triggers. +# The inject_rm_work event handler then merely changes task dependencies. +addtask do_rm_work +addtask do_build_without_rm_work +addhandler inject_rm_work +inject_rm_work[eventmask] = "bb.event.RecipeTaskPreProcess" +python inject_rm_work() { + if bb.data.inherits_class('kernel', d): + d.appendVar("RM_WORK_EXCLUDE", ' ' + d.getVar("PN")) # If the recipe name is in the RM_WORK_EXCLUDE, skip the recipe. - excludes = (d.getVar("RM_WORK_EXCLUDE", True) or "").split() - pn = d.getVar("PN", True) + excludes = (d.getVar("RM_WORK_EXCLUDE") or "").split() + pn = d.getVar("PN") + + # Determine what do_build depends upon, without including do_build + # itself or our own special do_rm_work_all. + deps = set(bb.build.preceedtask('do_build', True, d)) + deps.difference_update(('do_build', 'do_rm_work_all')) + if pn in excludes: d.delVarFlag('rm_work_rootfs', 'cleandirs') d.delVarFlag('rm_work_populatesdk', 'cleandirs') + else: + # Inject do_rm_work into the tasks of the current recipe such that do_build + # depends on it and that it runs after all other tasks that block do_build, + # i.e. after all work on the current recipe is done. The reason for taking + # this approach instead of making do_rm_work depend on do_build is that + # do_build inherits additional runtime dependencies on + # other recipes and thus will typically run much later than completion of + # work in the recipe itself. + # In practice, addtask() here merely updates the dependencies. + bb.build.addtask('do_rm_work', 'do_build', ' '.join(deps), d) + + # Always update do_build_without_rm_work dependencies. + bb.build.addtask('do_build_without_rm_work', '', ' '.join(deps), d) } diff --git a/meta/classes/rm_work_and_downloads.bbclass b/meta/classes/rm_work_and_downloads.bbclass new file mode 100644 index 0000000000..7c00bea597 --- /dev/null +++ b/meta/classes/rm_work_and_downloads.bbclass @@ -0,0 +1,33 @@ +# Author: Patrick Ohly <patrick.ohly@intel.com> +# Copyright: Copyright (C) 2015 Intel Corporation +# +# This file is licensed under the MIT license, see COPYING.MIT in +# this source distribution for the terms. + +# This class is used like rm_work: +# INHERIT += "rm_work_and_downloads" +# +# In addition to removing local build directories of a recipe, it also +# removes the downloaded source. This is achieved by making the DL_DIR +# recipe-specific. While reducing disk usage, it increases network usage (for +# example, compiling the same source for target and host implies downloading +# the source twice). +# +# Because the "do_fetch" task does not get re-run after removing the downloaded +# sources, this class is also not suitable for incremental builds. +# +# Where it works well is in well-connected build environments with limited +# disk space (like TravisCI). + +inherit rm_work + +# This would ensure that the existing do_rm_work() removes the downloads, +# but does not work because some recipes have a circular dependency between +# WORKDIR and DL_DIR (via ${SRCPV}?). +# DL_DIR = "${WORKDIR}/downloads" + +# Instead go up one level and remove ourself. +DL_DIR = "${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}/${PN}/downloads" +do_rm_work_append () { + rm -rf ${DL_DIR} +} diff --git a/meta/classes/rootfs-postcommands.bbclass b/meta/classes/rootfs-postcommands.bbclass new file mode 100644 index 0000000000..498174a664 --- /dev/null +++ b/meta/classes/rootfs-postcommands.bbclass @@ -0,0 +1,304 @@ + +# Zap the root password if debug-tweaks feature is not enabled +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'empty-root-password' ], "", "zap_empty_root_password ; ",d)}' + +# Allow dropbear/openssh to accept logins from accounts with an empty password string if debug-tweaks is enabled +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'allow-empty-password' ], "ssh_allow_empty_password; ", "",d)}' + +# Enable postinst logging if debug-tweaks is enabled +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'post-install-logging' ], "postinst_enable_logging; ", "",d)}' + +# Create /etc/timestamp during image construction to give a reasonably sane default time setting +ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp ; " + +# Tweak the mount options for rootfs in /etc/fstab if read-only-rootfs is enabled +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", "read_only_rootfs_hook; ", "",d)}' + +# Generates test data file with data store variables expanded in json format +ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data ; " + +# Write manifest +IMAGE_MANIFEST = "${IMGDEPLOYDIR}/${IMAGE_NAME}.rootfs.manifest" +ROOTFS_POSTUNINSTALL_COMMAND =+ "write_image_manifest ; " +# Set default postinst log file +POSTINST_LOGFILE ?= "${localstatedir}/log/postinstall.log" +# Set default target for systemd images +SYSTEMD_DEFAULT_TARGET ?= '${@bb.utils.contains("IMAGE_FEATURES", "x11-base", "graphical.target", "multi-user.target", d)}' +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("DISTRO_FEATURES", "systemd", "set_systemd_default_target; systemd_create_users;", "", d)}' + +ROOTFS_POSTPROCESS_COMMAND += 'empty_var_volatile;' + +# Disable DNS lookups, the SSH_DISABLE_DNS_LOOKUP can be overridden to allow +# distros to choose not to take this change +SSH_DISABLE_DNS_LOOKUP ?= " ssh_disable_dns_lookup ; " +ROOTFS_POSTPROCESS_COMMAND_append_qemuall = "${SSH_DISABLE_DNS_LOOKUP}" + +# Sort the user and group entries in /etc by ID in order to make the content +# deterministic. Package installs are not deterministic, causing the ordering +# of entries to change between builds. In case that this isn't desired, +# the command can be overridden. +# +# Note that useradd-staticids.bbclass has to be used to ensure that +# the numeric IDs of dynamically created entries remain stable. +# +# We want this to run as late as possible, in particular after +# systemd_sysusers_create and set_user_group. Using _append is not +# enough for that, set_user_group is added that way and would end +# up running after us. +SORT_PASSWD_POSTPROCESS_COMMAND ??= " sort_passwd; " +python () { + d.appendVar('ROOTFS_POSTPROCESS_COMMAND', '${SORT_PASSWD_POSTPROCESS_COMMAND}') +} + +systemd_create_users () { + for conffile in ${IMAGE_ROOTFS}/usr/lib/sysusers.d/systemd.conf ${IMAGE_ROOTFS}/usr/lib/sysusers.d/systemd-remote.conf; do + [ -e $conffile ] || continue + grep -v "^#" $conffile | sed -e '/^$/d' | while read type name id comment; do + if [ "$type" = "u" ]; then + useradd_params="--shell /sbin/nologin" + [ "$id" != "-" ] && useradd_params="$useradd_params --uid $id" + [ "$comment" != "-" ] && useradd_params="$useradd_params --comment $comment" + useradd_params="$useradd_params --system $name" + eval useradd --root ${IMAGE_ROOTFS} $useradd_params || true + elif [ "$type" = "g" ]; then + groupadd_params="" + [ "$id" != "-" ] && groupadd_params="$groupadd_params --gid $id" + groupadd_params="$groupadd_params --system $name" + eval groupadd --root ${IMAGE_ROOTFS} $groupadd_params || true + elif [ "$type" = "m" ]; then + group=$id + if [ ! `grep -q "^${group}:" ${IMAGE_ROOTFS}${sysconfdir}/group` ]; then + eval groupadd --root ${IMAGE_ROOTFS} --system $group + fi + if [ ! `grep -q "^${name}:" ${IMAGE_ROOTFS}${sysconfdir}/passwd` ]; then + eval useradd --root ${IMAGE_ROOTFS} --shell /sbin/nologin --system $name + fi + eval usermod --root ${IMAGE_ROOTFS} -a -G $group $name + fi + done + done +} + +# +# A hook function to support read-only-rootfs IMAGE_FEATURES +# +read_only_rootfs_hook () { + # Tweak the mount option and fs_passno for rootfs in fstab + sed -i -e '/^[#[:space:]]*\/dev\/root/{s/defaults/ro/;s/\([[:space:]]*[[:digit:]]\)\([[:space:]]*\)[[:digit:]]$/\1\20/}' ${IMAGE_ROOTFS}/etc/fstab + + # If we're using openssh and the /etc/ssh directory has no pre-generated keys, + # we should configure openssh to use the configuration file /etc/ssh/sshd_config_readonly + # and the keys under /var/run/ssh. + if [ -d ${IMAGE_ROOTFS}/etc/ssh ]; then + if [ -e ${IMAGE_ROOTFS}/etc/ssh/ssh_host_rsa_key ]; then + echo "SYSCONFDIR=/etc/ssh" >> ${IMAGE_ROOTFS}/etc/default/ssh + echo "SSHD_OPTS=" >> ${IMAGE_ROOTFS}/etc/default/ssh + else + echo "SYSCONFDIR=/var/run/ssh" >> ${IMAGE_ROOTFS}/etc/default/ssh + echo "SSHD_OPTS='-f /etc/ssh/sshd_config_readonly'" >> ${IMAGE_ROOTFS}/etc/default/ssh + fi + fi + + # Also tweak the key location for dropbear in the same way. + if [ -d ${IMAGE_ROOTFS}/etc/dropbear ]; then + if [ -e ${IMAGE_ROOTFS}/etc/dropbear/dropbear_rsa_host_key ]; then + echo "DROPBEAR_RSAKEY_DIR=/etc/dropbear" >> ${IMAGE_ROOTFS}/etc/default/dropbear + else + echo "DROPBEAR_RSAKEY_DIR=/var/lib/dropbear" >> ${IMAGE_ROOTFS}/etc/default/dropbear + fi + fi + + + if ${@bb.utils.contains("DISTRO_FEATURES", "sysvinit", "true", "false", d)}; then + # Change the value of ROOTFS_READ_ONLY in /etc/default/rcS to yes + if [ -e ${IMAGE_ROOTFS}/etc/default/rcS ]; then + sed -i 's/ROOTFS_READ_ONLY=no/ROOTFS_READ_ONLY=yes/' ${IMAGE_ROOTFS}/etc/default/rcS + fi + # Run populate-volatile.sh at rootfs time to set up basic files + # and directories to support read-only rootfs. + if [ -x ${IMAGE_ROOTFS}/etc/init.d/populate-volatile.sh ]; then + ${IMAGE_ROOTFS}/etc/init.d/populate-volatile.sh + fi + fi +} + +# +# This function is intended to disallow empty root password if 'debug-tweaks' is not in IMAGE_FEATURES. +# +zap_empty_root_password () { + if [ -e ${IMAGE_ROOTFS}/etc/shadow ]; then + sed -i 's%^root::%root:*:%' ${IMAGE_ROOTFS}/etc/shadow + fi + if [ -e ${IMAGE_ROOTFS}/etc/passwd ]; then + sed -i 's%^root::%root:*:%' ${IMAGE_ROOTFS}/etc/passwd + fi +} + +# +# allow dropbear/openssh to accept root logins and logins from accounts with an empty password string +# +ssh_allow_empty_password () { + for config in sshd_config sshd_config_readonly; do + if [ -e ${IMAGE_ROOTFS}${sysconfdir}/ssh/$config ]; then + sed -i 's/^[#[:space:]]*PermitRootLogin.*/PermitRootLogin yes/' ${IMAGE_ROOTFS}${sysconfdir}/ssh/$config + sed -i 's/^[#[:space:]]*PermitEmptyPasswords.*/PermitEmptyPasswords yes/' ${IMAGE_ROOTFS}${sysconfdir}/ssh/$config + fi + done + + if [ -e ${IMAGE_ROOTFS}${sbindir}/dropbear ] ; then + if grep -q DROPBEAR_EXTRA_ARGS ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear 2>/dev/null ; then + if ! grep -q "DROPBEAR_EXTRA_ARGS=.*-B" ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear ; then + sed -i 's/^DROPBEAR_EXTRA_ARGS="*\([^"]*\)"*/DROPBEAR_EXTRA_ARGS="\1 -B"/' ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear + fi + else + printf '\nDROPBEAR_EXTRA_ARGS="-B"\n' >> ${IMAGE_ROOTFS}${sysconfdir}/default/dropbear + fi + fi + + if [ -d ${IMAGE_ROOTFS}${sysconfdir}/pam.d ] ; then + sed -i 's/nullok_secure/nullok/' ${IMAGE_ROOTFS}${sysconfdir}/pam.d/* + fi +} + +ssh_disable_dns_lookup () { + if [ -e ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config ]; then + sed -i -e 's:#UseDNS yes:UseDNS no:' ${IMAGE_ROOTFS}${sysconfdir}/ssh/sshd_config + fi +} + +python sort_passwd () { + import rootfspostcommands + rootfspostcommands.sort_passwd(d.expand('${IMAGE_ROOTFS}${sysconfdir}')) +} + +# +# Enable postinst logging if debug-tweaks is enabled +# +postinst_enable_logging () { + mkdir -p ${IMAGE_ROOTFS}${sysconfdir}/default + echo "POSTINST_LOGGING=1" >> ${IMAGE_ROOTFS}${sysconfdir}/default/postinst + echo "LOGFILE=${POSTINST_LOGFILE}" >> ${IMAGE_ROOTFS}${sysconfdir}/default/postinst +} + +# +# Modify systemd default target +# +set_systemd_default_target () { + if [ -d ${IMAGE_ROOTFS}${sysconfdir}/systemd/system -a -e ${IMAGE_ROOTFS}${systemd_unitdir}/system/${SYSTEMD_DEFAULT_TARGET} ]; then + ln -sf ${systemd_unitdir}/system/${SYSTEMD_DEFAULT_TARGET} ${IMAGE_ROOTFS}${sysconfdir}/systemd/system/default.target + fi +} + +# If /var/volatile is not empty, we have seen problems where programs such as the +# journal make assumptions based on the contents of /var/volatile. The journal +# would then write to /var/volatile before it was mounted, thus hiding the +# items previously written. +# +# This change is to attempt to fix those types of issues in a way that doesn't +# affect users that may not be using /var/volatile. +empty_var_volatile () { + if [ -e ${IMAGE_ROOTFS}/etc/fstab ]; then + match=`awk '$1 !~ "#" && $2 ~ /\/var\/volatile/{print $2}' ${IMAGE_ROOTFS}/etc/fstab 2> /dev/null` + if [ -n "$match" ]; then + find ${IMAGE_ROOTFS}/var/volatile -mindepth 1 -delete + fi + fi +} + +# Turn any symbolic /sbin/init link into a file +remove_init_link () { + if [ -h ${IMAGE_ROOTFS}/sbin/init ]; then + LINKFILE=${IMAGE_ROOTFS}`readlink ${IMAGE_ROOTFS}/sbin/init` + rm ${IMAGE_ROOTFS}/sbin/init + cp $LINKFILE ${IMAGE_ROOTFS}/sbin/init + fi +} + +make_zimage_symlink_relative () { + if [ -L ${IMAGE_ROOTFS}/boot/zImage ]; then + (cd ${IMAGE_ROOTFS}/boot/ && for i in `ls zImage-* | sort`; do ln -sf $i zImage; done) + fi +} + +python write_image_manifest () { + from oe.rootfs import image_list_installed_packages + from oe.utils import format_pkg_list + + deploy_dir = d.getVar('IMGDEPLOYDIR') + link_name = d.getVar('IMAGE_LINK_NAME') + manifest_name = d.getVar('IMAGE_MANIFEST') + + if not manifest_name: + return + + pkgs = image_list_installed_packages(d) + with open(manifest_name, 'w+') as image_manifest: + image_manifest.write(format_pkg_list(pkgs, "ver")) + image_manifest.write("\n") + + if os.path.exists(manifest_name): + manifest_link = deploy_dir + "/" + link_name + ".manifest" + if os.path.lexists(manifest_link): + os.remove(manifest_link) + os.symlink(os.path.basename(manifest_name), manifest_link) +} + +# Can be use to create /etc/timestamp during image construction to give a reasonably +# sane default time setting +rootfs_update_timestamp () { + date -u +%4Y%2m%2d%2H%2M%2S >${IMAGE_ROOTFS}/etc/timestamp +} + +# Prevent X from being started +rootfs_no_x_startup () { + if [ -f ${IMAGE_ROOTFS}/etc/init.d/xserver-nodm ]; then + chmod a-x ${IMAGE_ROOTFS}/etc/init.d/xserver-nodm + fi +} + +rootfs_trim_schemas () { + for schema in ${IMAGE_ROOTFS}/etc/gconf/schemas/*.schemas + do + # Need this in case no files exist + if [ -e $schema ]; then + oe-trim-schemas $schema > $schema.new + mv $schema.new $schema + fi + done +} + +rootfs_check_host_user_contaminated () { + contaminated="${WORKDIR}/host-user-contaminated.txt" + HOST_USER_UID="$(PSEUDO_UNLOAD=1 id -u)" + HOST_USER_GID="$(PSEUDO_UNLOAD=1 id -g)" + + find "${IMAGE_ROOTFS}" -wholename "${IMAGE_ROOTFS}/home" -prune \ + -user "$HOST_USER_UID" -o -group "$HOST_USER_GID" >"$contaminated" + + if [ -s "$contaminated" ]; then + echo "WARNING: Paths in the rootfs are owned by the same user or group as the user running bitbake. See the logfile for the specific paths." + cat "$contaminated" | sed "s,^, ," + fi +} + +# Make any absolute links in a sysroot relative +rootfs_sysroot_relativelinks () { + sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT} +} + + +# Generated test data json file +python write_image_test_data() { + from oe.data import export2json + + testdata = "%s/%s.testdata.json" % (d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('IMAGE_NAME')) + testdata_link = "%s/%s.testdata.json" % (d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('IMAGE_LINK_NAME')) + + bb.utils.mkdirhier(os.path.dirname(testdata)) + searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/") + export2json(d, testdata,searchString=searchString,replaceString="") + + if os.path.lexists(testdata_link): + os.remove(testdata_link) + os.symlink(os.path.basename(testdata), testdata_link) +} diff --git a/meta/classes/rootfs_deb.bbclass b/meta/classes/rootfs_deb.bbclass index d51b4582d2..10af4b5ff0 100644 --- a/meta/classes/rootfs_deb.bbclass +++ b/meta/classes/rootfs_deb.bbclass @@ -3,19 +3,17 @@ # ROOTFS_PKGMANAGE = "dpkg apt" -ROOTFS_PKGMANAGE_BOOTSTRAP = "run-postinsts" do_rootfs[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot" do_populate_sdk[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot bzip2-native:do_populate_sysroot" do_rootfs[recrdeptask] += "do_package_write_deb" -rootfs_deb_do_rootfs[vardepsexclude] += "BUILDNAME" do_rootfs[vardeps] += "PACKAGE_FEED_URIS" do_rootfs[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock" do_populate_sdk[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock" python rootfs_deb_bad_recommendations() { - if d.getVar("BAD_RECOMMENDATIONS", True): + if d.getVar("BAD_RECOMMENDATIONS"): bb.warn("Debian package install does not support BAD_RECOMMENDATIONS") } do_rootfs[prefuncs] += "rootfs_deb_bad_recommendations" @@ -26,7 +24,7 @@ opkglibdir = "${localstatedir}/lib/opkg" python () { # Map TARGET_ARCH to Debian's ideas about architectures - darch = d.getVar('SDK_ARCH', True) + darch = d.getVar('SDK_ARCH') if darch in ["x86", "i486", "i586", "i686", "pentium"]: d.setVar('DEB_SDK_ARCH', 'i386') elif darch == "x86_64": diff --git a/meta/classes/rootfs_ipk.bbclass b/meta/classes/rootfs_ipk.bbclass index dd144e49ef..a57b1d34f7 100644 --- a/meta/classes/rootfs_ipk.bbclass +++ b/meta/classes/rootfs_ipk.bbclass @@ -7,13 +7,11 @@ EXTRAOPKGCONFIG ?= "" ROOTFS_PKGMANAGE = "opkg ${EXTRAOPKGCONFIG}" -ROOTFS_PKGMANAGE_BOOTSTRAP = "run-postinsts" do_rootfs[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot" do_populate_sdk[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot" do_rootfs[recrdeptask] += "do_package_write_ipk" do_rootfs[vardeps] += "PACKAGE_FEED_URIS" -rootfs_ipk_do_rootfs[vardepsexclude] += "BUILDNAME" do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock" do_populate_sdk[lockfiles] += "${WORKDIR}/ipk.lock" @@ -28,7 +26,7 @@ MULTILIBRE_ALLOW_REP = "${OPKGLIBDIR}/opkg|/usr/lib/opkg" python () { - if d.getVar('BUILD_IMAGES_FROM_FEEDS', True): + if d.getVar('BUILD_IMAGES_FROM_FEEDS'): flags = d.getVarFlag('do_rootfs', 'recrdeptask') flags = flags.replace("do_package_write_ipk", "") flags = flags.replace("do_deploy", "") diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass index d85d001a62..7f305f51ca 100644 --- a/meta/classes/rootfs_rpm.bbclass +++ b/meta/classes/rootfs_rpm.bbclass @@ -2,36 +2,32 @@ # Creates a root filesystem out of rpm packages # -ROOTFS_PKGMANAGE = "rpm smartpm" -ROOTFS_PKGMANAGE_BOOTSTRAP = "run-postinsts" +ROOTFS_PKGMANAGE = "rpm dnf" -# Add 50Meg of extra space for Smart -IMAGE_ROOTFS_EXTRA_SPACE_append = "${@bb.utils.contains("PACKAGE_INSTALL", "smartpm", " + 51200", "" ,d)}" +# dnf is using our custom distutils, and so will fail without these +export STAGING_INCDIR +export STAGING_LIBDIR -# Smart is python based, so be sure python-native is available to us. -EXTRANATIVEPATH += "python-native" +# Add 100Meg of extra space for dnf +IMAGE_ROOTFS_EXTRA_SPACE_append = "${@bb.utils.contains("PACKAGE_INSTALL", "dnf", " + 102400", "" ,d)}" + +# Dnf is python based, so be sure python3-native is available to us. +EXTRANATIVEPATH += "python3-native" # opkg is needed for update-alternatives RPMROOTFSDEPENDS = "rpm-native:do_populate_sysroot \ - rpmresolve-native:do_populate_sysroot \ - python-smartpm-native:do_populate_sysroot \ - createrepo-native:do_populate_sysroot \ + dnf-native:do_populate_sysroot \ + createrepo-c-native:do_populate_sysroot \ opkg-native:do_populate_sysroot" do_rootfs[depends] += "${RPMROOTFSDEPENDS}" do_populate_sdk[depends] += "${RPMROOTFSDEPENDS}" do_rootfs[recrdeptask] += "do_package_write_rpm" -rootfs_rpm_do_rootfs[vardepsexclude] += "BUILDNAME" do_rootfs[vardeps] += "PACKAGE_FEED_URIS" -# RPM doesn't work with multiple rootfs generation at once due to collisions in the use of files -# in ${DEPLOY_DIR_RPM}. This can be removed if package_update_index_rpm can be called concurrently -do_rootfs[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock" -do_populate_sdk[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock" - python () { - if d.getVar('BUILD_IMAGES_FROM_FEEDS', True): + if d.getVar('BUILD_IMAGES_FROM_FEEDS'): flags = d.getVarFlag('do_rootfs', 'recrdeptask') flags = flags.replace("do_package_write_rpm", "") flags = flags.replace("do_deploy", "") @@ -41,7 +37,3 @@ python () { d.setVar('RPM_POSTPROCESS_COMMANDS', '') } -# Smart is python based, so be sure python-native is available to us. -EXTRANATIVEPATH += "python-native" - -rpmlibdir = "/var/lib/rpm" diff --git a/meta/classes/sanity.bbclass b/meta/classes/sanity.bbclass index 2864318632..e8064ac483 100644 --- a/meta/classes/sanity.bbclass +++ b/meta/classes/sanity.bbclass @@ -3,10 +3,10 @@ # SANITY_REQUIRED_UTILITIES ?= "patch diffstat makeinfo git bzip2 tar \ - gzip gawk chrpath wget cpio perl" + gzip gawk chrpath wget cpio perl file which" def bblayers_conf_file(d): - return os.path.join(d.getVar('TOPDIR', True), 'conf/bblayers.conf') + return os.path.join(d.getVar('TOPDIR'), 'conf/bblayers.conf') def sanity_conf_read(fn): with open(fn, 'r') as f: @@ -20,26 +20,83 @@ def sanity_conf_find_line(pattern, lines): if re.search(pattern, line)), (None, None)) def sanity_conf_update(fn, lines, version_var_name, new_version): - index, line = sanity_conf_find_line(version_var_name, lines) + index, line = sanity_conf_find_line(r"^%s" % version_var_name, lines) lines[index] = '%s = "%d"\n' % (version_var_name, new_version) with open(fn, "w") as f: f.write(''.join(lines)) -# Functions added to this variable MUST throw an exception (or sys.exit()) unless they -# successfully changed LCONF_VERSION in bblayers.conf -BBLAYERS_CONF_UPDATE_FUNCS += "oecore_update_bblayers" +# Functions added to this variable MUST throw a NotImplementedError exception unless +# they successfully changed the config version in the config file. Exceptions +# are used since exec_func doesn't handle return values. +BBLAYERS_CONF_UPDATE_FUNCS += " \ + conf/bblayers.conf:LCONF_VERSION:LAYER_CONF_VERSION:oecore_update_bblayers \ + conf/local.conf:CONF_VERSION:LOCALCONF_VERSION:oecore_update_localconf \ + conf/site.conf:SCONF_VERSION:SITE_CONF_VERSION:oecore_update_siteconf \ +" +SANITY_DIFF_TOOL ?= "meld" + +SANITY_LOCALCONF_SAMPLE ?= "${COREBASE}/meta*/conf/local.conf.sample" +python oecore_update_localconf() { + # Check we are using a valid local.conf + current_conf = d.getVar('CONF_VERSION') + conf_version = d.getVar('LOCALCONF_VERSION') + + failmsg = """Your version of local.conf was generated from an older/newer version of +local.conf.sample and there have been updates made to this file. Please compare the two +files and merge any changes before continuing. + +Matching the version numbers will remove this message. + +\"${SANITY_DIFF_TOOL} conf/local.conf ${SANITY_LOCALCONF_SAMPLE}\" + +is a good way to visualise the changes.""" + failmsg = d.expand(failmsg) + + raise NotImplementedError(failmsg) +} + +SANITY_SITECONF_SAMPLE ?= "${COREBASE}/meta*/conf/site.conf.sample" +python oecore_update_siteconf() { + # If we have a site.conf, check it's valid + current_sconf = d.getVar('SCONF_VERSION') + sconf_version = d.getVar('SITE_CONF_VERSION') + + failmsg = """Your version of site.conf was generated from an older version of +site.conf.sample and there have been updates made to this file. Please compare the two +files and merge any changes before continuing. + +Matching the version numbers will remove this message. + +\"${SANITY_DIFF_TOOL} conf/site.conf ${SANITY_SITECONF_SAMPLE}\" + +is a good way to visualise the changes.""" + failmsg = d.expand(failmsg) + + raise NotImplementedError(failmsg) +} + +SANITY_BBLAYERCONF_SAMPLE ?= "${COREBASE}/meta*/conf/bblayers.conf.sample" python oecore_update_bblayers() { # bblayers.conf is out of date, so see if we can resolve that - current_lconf = int(d.getVar('LCONF_VERSION', True)) + current_lconf = int(d.getVar('LCONF_VERSION')) + lconf_version = int(d.getVar('LAYER_CONF_VERSION')) + + failmsg = """Your version of bblayers.conf has the wrong LCONF_VERSION (has ${LCONF_VERSION}, expecting ${LAYER_CONF_VERSION}). +Please compare your file against bblayers.conf.sample and merge any changes before continuing. +"${SANITY_DIFF_TOOL} conf/bblayers.conf ${SANITY_BBLAYERCONF_SAMPLE}" + +is a good way to visualise the changes.""" + failmsg = d.expand(failmsg) + if not current_lconf: - sys.exit() - lconf_version = int(d.getVar('LAYER_CONF_VERSION', True)) + raise NotImplementedError(failmsg) + lines = [] if current_lconf < 4: - sys.exit() + raise NotImplementedError(failmsg) bblayers_fn = bblayers_conf_file(d) lines = sanity_conf_read(bblayers_fn) @@ -58,29 +115,64 @@ python oecore_update_bblayers() { lines[index] = (bbpath_line[:start + 1] + topdir_var + ':' + bbpath_line[start + 1:]) else: - sys.exit() + raise NotImplementedError(failmsg) else: index, bbfiles_line = sanity_conf_find_line('BBFILES', lines) if bbfiles_line: lines.insert(index, 'BBPATH = "' + topdir_var + '"\n') else: - sys.exit() + raise NotImplementedError(failmsg) current_lconf += 1 sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) + bb.note("Your conf/bblayers.conf has been automatically updated.") return elif current_lconf == 5 and lconf_version > 5: # Null update, to avoid issues with people switching between poky and other distros current_lconf = 6 sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) + bb.note("Your conf/bblayers.conf has been automatically updated.") + return + + status.addresult() + + elif current_lconf == 6 and lconf_version > 6: + # Handle rename of meta-yocto -> meta-poky + # This marks the start of separate version numbers but code is needed in OE-Core + # for the migration, one last time. + layers = d.getVar('BBLAYERS').split() + layers = [ os.path.basename(path) for path in layers ] + if 'meta-yocto' in layers: + found = False + while True: + index, meta_yocto_line = sanity_conf_find_line(r'.*meta-yocto[\'"\s\n]', lines) + if meta_yocto_line: + lines[index] = meta_yocto_line.replace('meta-yocto', 'meta-poky') + found = True + else: + break + if not found: + raise NotImplementedError(failmsg) + index, meta_yocto_line = sanity_conf_find_line('LCONF_VERSION.*\n', lines) + if meta_yocto_line: + lines[index] = 'POKY_BBLAYERS_CONF_VERSION = "1"\n' + else: + raise NotImplementedError(failmsg) + with open(bblayers_fn, "w") as f: + f.write(''.join(lines)) + bb.note("Your conf/bblayers.conf has been automatically updated.") + return + current_lconf += 1 + sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) + bb.note("Your conf/bblayers.conf has been automatically updated.") return - sys.exit() + raise NotImplementedError(failmsg) } def raise_sanity_error(msg, d, network_error=False): - if d.getVar("SANITY_USE_EVENTS", True) == "1": + if d.getVar("SANITY_USE_EVENTS") == "1": try: bb.event.fire(bb.event.SanityCheckFailed(msg, network_error), d) except TypeError: @@ -106,8 +198,8 @@ def check_toolchain_tune_args(data, tune, multilib, errs): return found_errors def check_toolchain_args_present(data, tune, multilib, tune_errors, which): - args_set = (data.getVar("TUNE_%s" % which, True) or "").split() - args_wanted = (data.getVar("TUNEABI_REQUIRED_%s_tune-%s" % (which, tune), True) or "").split() + args_set = (data.getVar("TUNE_%s" % which) or "").split() + args_wanted = (data.getVar("TUNEABI_REQUIRED_%s_tune-%s" % (which, tune)) or "").split() args_missing = [] # If no args are listed/required, we are done. @@ -134,9 +226,8 @@ def check_toolchain_tune(data, tune, multilib): # Apply the overrides so we can look at the details. overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + multilib localdata.setVar("OVERRIDES", overrides) - bb.data.update_data(localdata) bb.debug(2, "Sanity-checking tuning '%s' (%s) features:" % (tune, multilib)) - features = (localdata.getVar("TUNE_FEATURES_tune-%s" % tune, True) or "").split() + features = (localdata.getVar("TUNE_FEATURES_tune-%s" % tune) or "").split() if not features: return "Tuning '%s' has no defined features, and cannot be used." % tune valid_tunes = localdata.getVarFlags('TUNEVALID') or {} @@ -156,9 +247,9 @@ def check_toolchain_tune(data, tune, multilib): bb.debug(2, " %s: %s" % (feature, valid_tunes[feature])) else: tune_errors.append("Feature '%s' is not defined." % feature) - whitelist = localdata.getVar("TUNEABI_WHITELIST", True) + whitelist = localdata.getVar("TUNEABI_WHITELIST") if whitelist: - tuneabi = localdata.getVar("TUNEABI_tune-%s" % tune, True) + tuneabi = localdata.getVar("TUNEABI_tune-%s" % tune) if not tuneabi: tuneabi = tune if True not in [x in whitelist.split() for x in tuneabi.split()]: @@ -172,13 +263,13 @@ def check_toolchain_tune(data, tune, multilib): def check_toolchain(data): tune_error_set = [] - deftune = data.getVar("DEFAULTTUNE", True) + deftune = data.getVar("DEFAULTTUNE") tune_errors = check_toolchain_tune(data, deftune, 'default') if tune_errors: tune_error_set.append(tune_errors) - multilibs = (data.getVar("MULTILIB_VARIANTS", True) or "").split() - global_multilibs = (data.getVar("MULTILIB_GLOBAL_VARIANTS", True) or "").split() + multilibs = (data.getVar("MULTILIB_VARIANTS") or "").split() + global_multilibs = (data.getVar("MULTILIB_GLOBAL_VARIANTS") or "").split() if multilibs: seen_libs = [] @@ -190,7 +281,7 @@ def check_toolchain(data): seen_libs.append(lib) if not lib in global_multilibs: tune_error_set.append("Multilib %s is not present in MULTILIB_GLOBAL_VARIANTS" % lib) - tune = data.getVar("DEFAULTTUNE_virtclass-multilib-%s" % lib, True) + tune = data.getVar("DEFAULTTUNE_virtclass-multilib-%s" % lib) if tune in seen_tunes: tune_error_set.append("The tuning '%s' appears in more than one multilib." % tune) else: @@ -241,7 +332,7 @@ def check_create_long_filename(filepath, pathname): def check_path_length(filepath, pathname, limit): if len(filepath) > limit: - return "The length of %s is longer than 410, this would cause unexpected errors, please use a shorter path.\n" % pathname + return "The length of %s is longer than %s, this would cause unexpected errors, please use a shorter path.\n" % (pathname, limit) return "" def get_filesystem_id(path): @@ -268,27 +359,34 @@ def check_connectivity(d): # URI's to check can be set in the CONNECTIVITY_CHECK_URIS variable # using the same syntax as for SRC_URI. If the variable is not set # the check is skipped - test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS', True) or "").split() + test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS') or "").split() retval = "" + bbn = d.getVar('BB_NO_NETWORK') + if bbn not in (None, '0', '1'): + return 'BB_NO_NETWORK should be "0" or "1", but it is "%s"' % bbn + # Only check connectivity if network enabled and the # CONNECTIVITY_CHECK_URIS are set - network_enabled = not d.getVar('BB_NO_NETWORK', True) + network_enabled = not (bbn == '1') check_enabled = len(test_uris) - # Take a copy of the data store and unset MIRRORS and PREMIRRORS - data = bb.data.createCopy(d) - data.delVar('PREMIRRORS') - data.delVar('MIRRORS') if check_enabled and network_enabled: + # Take a copy of the data store and unset MIRRORS and PREMIRRORS + data = bb.data.createCopy(d) + data.delVar('PREMIRRORS') + data.delVar('MIRRORS') try: fetcher = bb.fetch2.Fetch(test_uris, data) fetcher.checkstatus() except Exception as err: # Allow the message to be configured so that users can be # pointed to a support mechanism. - msg = data.getVar('CONNECTIVITY_CHECK_MSG', True) or "" + msg = data.getVar('CONNECTIVITY_CHECK_MSG') or "" if len(msg) == 0: - msg = "%s. Please ensure your network is configured correctly.\n" % err + msg = "%s.\n" % err + msg += " Please ensure your host's network is configured correctly,\n" + msg += " or set BB_NO_NETWORK = \"1\" to disable network access if\n" + msg += " all required sources are on local disk.\n" retval = msg return retval @@ -296,7 +394,7 @@ def check_connectivity(d): def check_supported_distro(sanity_data): from fnmatch import fnmatch - tested_distros = sanity_data.getVar('SANITY_TESTED_DISTROS', True) + tested_distros = sanity_data.getVar('SANITY_TESTED_DISTROS') if not tested_distros: return @@ -319,16 +417,17 @@ def check_sanity_validmachine(sanity_data): messages = "" # Check TUNE_ARCH is set - if sanity_data.getVar('TUNE_ARCH', True) == 'INVALID': + if sanity_data.getVar('TUNE_ARCH') == 'INVALID': messages = messages + 'TUNE_ARCH is unset. Please ensure your MACHINE configuration includes a valid tune configuration file which will set this correctly.\n' # Check TARGET_OS is set - if sanity_data.getVar('TARGET_OS', True) == 'INVALID': + if sanity_data.getVar('TARGET_OS') == 'INVALID': messages = messages + 'Please set TARGET_OS directly, or choose a MACHINE or DISTRO that does so.\n' # Check that we don't have duplicate entries in PACKAGE_ARCHS & that TUNE_PKGARCH is in PACKAGE_ARCHS - pkgarchs = sanity_data.getVar('PACKAGE_ARCHS', True) - tunepkg = sanity_data.getVar('TUNE_PKGARCH', True) + pkgarchs = sanity_data.getVar('PACKAGE_ARCHS') + tunepkg = sanity_data.getVar('TUNE_PKGARCH') + defaulttune = sanity_data.getVar('DEFAULTTUNE') tunefound = False seen = {} dups = [] @@ -345,7 +444,7 @@ def check_sanity_validmachine(sanity_data): messages = messages + "Error, the PACKAGE_ARCHS variable contains duplicates. The following archs are listed more than once: %s" % " ".join(dups) if tunefound == False: - messages = messages + "Error, the PACKAGE_ARCHS variable does not contain TUNE_PKGARCH (%s)." % tunepkg + messages = messages + "Error, the PACKAGE_ARCHS variable (%s) for DEFAULTTUNE (%s) does not contain TUNE_PKGARCH (%s)." % (pkgarchs, defaulttune, tunepkg) return messages @@ -355,7 +454,7 @@ def check_gcc_march(sanity_data): message = "" # Check if -march not in BUILD_CFLAGS - if sanity_data.getVar("BUILD_CFLAGS",True).find("-march") < 0: + if sanity_data.getVar("BUILD_CFLAGS").find("-march") < 0: result = False # Construct a test file @@ -365,19 +464,19 @@ def check_gcc_march(sanity_data): # Check if GCC could work without march if not result: - status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc gcc_test.c -o gcc_test") + status,res = oe.utils.getstatusoutput(sanity_data.expand("${BUILD_CC} gcc_test.c -o gcc_test")) if status == 0: result = True; if not result: - status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc -march=native gcc_test.c -o gcc_test") + status,res = oe.utils.getstatusoutput(sanity_data.expand("${BUILD_CC} -march=native gcc_test.c -o gcc_test")) if status == 0: message = "BUILD_CFLAGS_append = \" -march=native\"" result = True; if not result: - build_arch = sanity_data.getVar('BUILD_ARCH', True) - status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc -march=%s gcc_test.c -o gcc_test" % build_arch) + build_arch = sanity_data.getVar('BUILD_ARCH') + status,res = oe.utils.getstatusoutput(sanity_data.expand("${BUILD_CC} -march=%s gcc_test.c -o gcc_test" % build_arch)) if status == 0: message = "BUILD_CFLAGS_append = \" -march=%s\"" % build_arch result = True; @@ -437,60 +536,43 @@ def check_tar_version(sanity_data): return None # We use git parameters and functionality only found in 1.7.8 or later +# The kernel tools assume git >= 1.8.3.1 (verified needed > 1.7.9.5) see #6162 +# The git fetcher also had workarounds for git < 1.7.9.2 which we've dropped def check_git_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("git --version 2> /dev/null") if status != 0: return "Unable to execute git --version, exit code %s\n" % status version = result.split()[2] - if LooseVersion(version) < LooseVersion("1.7.8"): - return "Your version of git is older than 1.7.8 and has bugs which will break builds. Please install a newer version of git.\n" + if LooseVersion(version) < LooseVersion("1.8.3.1"): + return "Your version of git is older than 1.8.3.1 and has bugs which will break builds. Please install a newer version of git.\n" return None # Check the required perl modules which may not be installed by default def check_perl_modules(sanity_data): ret = "" modules = ( "Text::ParseWords", "Thread::Queue", "Data::Dumper" ) + errresult = '' for m in modules: - status, result = oe.utils.getstatusoutput("perl -e 'use %s' 2> /dev/null" % m) + status, result = oe.utils.getstatusoutput("perl -e 'use %s'" % m) if status != 0: + errresult += result ret += "%s " % m if ret: - return "Required perl module(s) not found: %s\n" % ret + return "Required perl module(s) not found: %s\n\n%s\n" % (ret, errresult) return None -def sanity_check_conffiles(status, d): - # Check we are using a valid local.conf - current_conf = d.getVar('CONF_VERSION', True) - conf_version = d.getVar('LOCALCONF_VERSION', True) - - if current_conf != conf_version: - status.addresult("Your version of local.conf was generated from an older/newer version of local.conf.sample and there have been updates made to this file. Please compare the two files and merge any changes before continuing.\nMatching the version numbers will remove this message.\n\"meld conf/local.conf ${COREBASE}/meta*/conf/local.conf.sample\" is a good way to visualise the changes.\n") - - # Check bblayers.conf is valid - current_lconf = d.getVar('LCONF_VERSION', True) - lconf_version = d.getVar('LAYER_CONF_VERSION', True) - if current_lconf != lconf_version: - funcs = d.getVar('BBLAYERS_CONF_UPDATE_FUNCS', True).split() - for func in funcs: - success = True +def sanity_check_conffiles(d): + funcs = d.getVar('BBLAYERS_CONF_UPDATE_FUNCS').split() + for func in funcs: + conffile, current_version, required_version, func = func.split(":") + if check_conf_exists(conffile, d) and d.getVar(current_version) is not None and \ + d.getVar(current_version) != d.getVar(required_version): try: - bb.build.exec_func(func, d) - except Exception: - success = False - if success: - bb.note("Your conf/bblayers.conf has been automatically updated.") - status.reparse = True - if not status.reparse: - status.addresult("Your version of bblayers.conf has the wrong LCONF_VERSION (has %s, expecting %s).\nPlease compare the your file against bblayers.conf.sample and merge any changes before continuing.\n\"meld conf/bblayers.conf ${COREBASE}/meta*/conf/bblayers.conf.sample\" is a good way to visualise the changes.\n" % (current_lconf, lconf_version)) - - # If we have a site.conf, check it's valid - if check_conf_exists("conf/site.conf", d): - current_sconf = d.getVar('SCONF_VERSION', True) - sconf_version = d.getVar('SITE_CONF_VERSION', True) - if current_sconf != sconf_version: - status.addresult("Your version of site.conf was generated from an older version of site.conf.sample and there have been updates made to this file. Please compare the two files and merge any changes before continuing.\nMatching the version numbers will remove this message.\n\"meld conf/site.conf ${COREBASE}/meta*/conf/site.conf.sample\" is a good way to visualise the changes.\n") - + bb.build.exec_func(func, d, pythonexception=True) + except NotImplementedError as e: + bb.fatal(str(e)) + d.setVar("BB_INVALIDCONF", True) def sanity_handle_abichanges(status, d): # @@ -498,55 +580,16 @@ def sanity_handle_abichanges(status, d): # import subprocess - current_abi = d.getVar('OELAYOUT_ABI', True) - abifile = d.getVar('SANITY_ABIFILE', True) + current_abi = d.getVar('OELAYOUT_ABI') + abifile = d.getVar('SANITY_ABIFILE') if os.path.exists(abifile): with open(abifile, "r") as f: abi = f.read().strip() if not abi.isdigit(): with open(abifile, "w") as f: f.write(current_abi) - elif abi == "2" and current_abi == "3": - bb.note("Converting staging from layout version 2 to layout version 3") - subprocess.call(d.expand("mv ${TMPDIR}/staging ${TMPDIR}/sysroots"), shell=True) - subprocess.call(d.expand("ln -s sysroots ${TMPDIR}/staging"), shell=True) - subprocess.call(d.expand("cd ${TMPDIR}/stamps; for i in */*do_populate_staging; do new=`echo $i | sed -e 's/do_populate_staging/do_populate_sysroot/'`; mv $i $new; done"), shell=True) - with open(abifile, "w") as f: - f.write(current_abi) - elif abi == "3" and current_abi == "4": - bb.note("Converting staging layout from version 3 to layout version 4") - if os.path.exists(d.expand("${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}")): - subprocess.call(d.expand("mv ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS} ${STAGING_BINDIR_CROSS}"), shell=True) - subprocess.call(d.expand("ln -s ${STAGING_BINDIR_CROSS} ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}"), shell=True) - with open(abifile, "w") as f: - f.write(current_abi) - elif abi == "4": - status.addresult("Staging layout has changed. The cross directory has been deprecated and cross packages are now built under the native sysroot.\nThis requires a rebuild.\n") - elif abi == "5" and current_abi == "6": - bb.note("Converting staging layout from version 5 to layout version 6") - subprocess.call(d.expand("mv ${TMPDIR}/pstagelogs ${SSTATE_MANIFESTS}"), shell=True) - with open(abifile, "w") as f: - f.write(current_abi) - elif abi == "7" and current_abi == "8": - status.addresult("Your configuration is using stamp files including the sstate hash but your build directory was built with stamp files that do not include this.\nTo continue, either rebuild or switch back to the OEBasic signature handler with BB_SIGNATURE_HANDLER = 'OEBasic'.\n") - elif (abi != current_abi and current_abi == "9"): - status.addresult("The layout of the TMPDIR STAMPS directory has changed. Please clean out TMPDIR and rebuild (sstate will be still be valid and reused)\n") - elif (abi != current_abi and current_abi == "10" and (abi == "8" or abi == "9")): - bb.note("Converting staging layout from version 8/9 to layout version 10") - cmd = d.expand("grep -r -l sysroot-providers/virtual_kernel ${SSTATE_MANIFESTS}") - ret, result = oe.utils.getstatusoutput(cmd) - result = result.split() - for f in result: - bb.note("Uninstalling manifest file %s" % f) - sstate_clean_manifest(f, d) - with open(abifile, "w") as f: - f.write(current_abi) - elif abi == "10" and current_abi == "11": - bb.note("Converting staging layout from version 10 to layout version 11") - # Files in xf86-video-modesetting moved to xserver-xorg and bitbake can't currently handle that: - subprocess.call(d.expand("rm ${TMPDIR}/sysroots/*/usr/lib/xorg/modules/drivers/modesetting_drv.so ${TMPDIR}/sysroots/*/pkgdata/runtime/xf86-video-modesetting* ${TMPDIR}/sysroots/*/pkgdata/runtime-reverse/xf86-video-modesetting* ${TMPDIR}/sysroots/*/pkgdata/shlibs2/xf86-video-modesetting*"), shell=True) - with open(abifile, "w") as f: - f.write(current_abi) + elif int(abi) <= 11 and current_abi == "12": + status.addresult("The layout of TMPDIR changed for Recipe Specific Sysroots.\nConversion doesn't make sense and this change will rebuild everything so please delete TMPDIR (%s).\n" % d.getVar("TMPDIR")) elif (abi != current_abi): # Code to convert from one ABI to another could go here if possible. status.addresult("Error, TMPDIR has changed its layout version number (%s to %s) and you need to either rebuild, revert or adjust it at your own risk.\n" % (abi, current_abi)) @@ -571,9 +614,9 @@ def check_sanity_sstate_dir_change(sstate_dir, data): return testmsg def check_sanity_version_change(status, d): - # Sanity checks to be done when SANITY_VERSION changes + # Sanity checks to be done when SANITY_VERSION or NATIVELSBSTRING changes # In other words, these tests run once in a given build directory and then - # never again until the sanity version changes. + # never again until the sanity version or host distrubution id/version changes. # Check the python install is complete. glib-2.0-natives requries # xml.parsers.expat @@ -593,13 +636,13 @@ def check_sanity_version_change(status, d): if not check_app_exists("${MAKE}", d): missing = missing + "GNU make," - if not check_app_exists('${BUILD_PREFIX}gcc', d): - missing = missing + "C Compiler (%sgcc)," % d.getVar("BUILD_PREFIX", True) + if not check_app_exists('${BUILD_CC}', d): + missing = missing + "C Compiler (%s)," % d.getVar("BUILD_CC") - if not check_app_exists('${BUILD_PREFIX}g++', d): - missing = missing + "C++ Compiler (%sg++)," % d.getVar("BUILD_PREFIX", True) + if not check_app_exists('${BUILD_CXX}', d): + missing = missing + "C++ Compiler (%s)," % d.getVar("BUILD_CXX") - required_utilities = d.getVar('SANITY_REQUIRED_UTILITIES', True) + required_utilities = d.getVar('SANITY_REQUIRED_UTILITIES') for util in required_utilities.split(): if not check_app_exists(util, d): @@ -609,7 +652,7 @@ def check_sanity_version_change(status, d): missing = missing.rstrip(',') status.addresult("Please install the following missing utilities: %s\n" % missing) - assume_provided = d.getVar('ASSUME_PROVIDED', True).split() + assume_provided = d.getVar('ASSUME_PROVIDED').split() # Check user doesn't have ASSUME_PROVIDED = instead of += in local.conf if "diffstat-native" not in assume_provided: status.addresult('Please use ASSUME_PROVIDED +=, not ASSUME_PROVIDED = in your local.conf\n') @@ -632,7 +675,7 @@ def check_sanity_version_change(status, d): status.addresult(" __sync_bool_compare_and_swap (&atomic, 2, 3);\n") # Check that TMPDIR isn't on a filesystem with limited filename length (eg. eCryptFS) - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') status.addresult(check_create_long_filename(tmpdir, "TMPDIR")) tmpdirmode = os.stat(tmpdir).st_mode if (tmpdirmode & stat.S_ISGID): @@ -656,7 +699,7 @@ def check_sanity_version_change(status, d): if netcheck: status.network_error = True - nolibs = d.getVar('NO32LIBS', True) + nolibs = d.getVar('NO32LIBS') if not nolibs: lib32path = '/lib' if os.path.exists('/lib64') and ( os.path.islink('/lib64') or os.path.islink('/lib') ): @@ -665,8 +708,8 @@ def check_sanity_version_change(status, d): if os.path.exists('%s/libc.so.6' % lib32path) and not os.path.exists('/usr/include/gnu/stubs-32.h'): status.addresult("You have a 32-bit libc, but no 32-bit headers. You must install the 32-bit libc headers.\n") - bbpaths = d.getVar('BBPATH', True).split(":") - if ("." in bbpaths or "./" in bbpaths or "" in bbpaths) and not status.reparse: + bbpaths = d.getVar('BBPATH').split(":") + if ("." in bbpaths or "./" in bbpaths or "" in bbpaths): status.addresult("BBPATH references the current directory, either through " \ "an empty entry, a './' or a '.'.\n\t This is unsafe and means your "\ "layer configuration is adding empty elements to BBPATH.\n\t "\ @@ -675,7 +718,7 @@ def check_sanity_version_change(status, d): "references.\n" \ "Parsed BBPATH is" + str(bbpaths)); - oes_bb_conf = d.getVar( 'OES_BITBAKE_CONF', True) + oes_bb_conf = d.getVar( 'OES_BITBAKE_CONF') if not oes_bb_conf: status.addresult('You are not using the OpenEmbedded version of conf/bitbake.conf. This means your environment is misconfigured, in particular check BBPATH.\n') @@ -685,6 +728,16 @@ def check_sanity_version_change(status, d): # Check that TMPDIR isn't located on nfs status.addresult(check_not_nfs(tmpdir, "TMPDIR")) +def sanity_check_locale(d): + """ + Currently bitbake switches locale to en_US.UTF-8 so check that this locale actually exists. + """ + import locale + try: + locale.setlocale(locale.LC_ALL, "en_US.UTF-8") + except locale.Error: + raise_sanity_error("You system needs to support the en_US.UTF-8 locale.", d) + def check_sanity_everybuild(status, d): import os, stat # Sanity tests which test the users environment so need to run at each build (or are so cheap @@ -700,26 +753,26 @@ def check_sanity_everybuild(status, d): # Check the bitbake version meets minimum requirements from distutils.version import LooseVersion - minversion = d.getVar('BB_MIN_VERSION', True) + minversion = d.getVar('BB_MIN_VERSION') if (LooseVersion(bb.__version__) < LooseVersion(minversion)): status.addresult('Bitbake version %s is required and version %s was found\n' % (minversion, bb.__version__)) - sanity_check_conffiles(status, d) + sanity_check_locale(d) - paths = d.getVar('PATH', True).split(":") + paths = d.getVar('PATH').split(":") if "." in paths or "./" in paths or "" in paths: status.addresult("PATH contains '.', './' or '' (empty element), which will break the build, please remove this.\nParsed PATH is " + str(paths) + "\n") # Check that the DISTRO is valid, if set # need to take into account DISTRO renaming DISTRO - distro = d.getVar('DISTRO', True) + distro = d.getVar('DISTRO') if distro and distro != "nodistro": if not ( check_conf_exists("conf/distro/${DISTRO}.conf", d) or check_conf_exists("conf/distro/include/${DISTRO}.inc", d) ): - status.addresult("DISTRO '%s' not found. Please set a valid DISTRO in your local.conf\n" % d.getVar("DISTRO", True)) + status.addresult("DISTRO '%s' not found. Please set a valid DISTRO in your local.conf\n" % d.getVar("DISTRO")) # Check that DL_DIR is set, exists and is writable. In theory, we should never even hit the check if DL_DIR isn't # set, since so much relies on it being set. - dldir = d.getVar('DL_DIR', True) + dldir = d.getVar('DL_DIR') if not dldir: status.addresult("DL_DIR is not set. Your environment is misconfigured, check that DL_DIR is set, and if the directory exists, that it is writable. \n") if os.path.exists(dldir) and not os.access(dldir, os.W_OK): @@ -728,9 +781,9 @@ def check_sanity_everybuild(status, d): # Check that the MACHINE is valid, if it is set machinevalid = True - if d.getVar('MACHINE', True): + if d.getVar('MACHINE'): if not check_conf_exists("conf/machine/${MACHINE}.conf", d): - status.addresult('Please set a valid MACHINE in your local.conf or environment\n') + status.addresult('MACHINE=%s is invalid. Please set a valid MACHINE in your local.conf, environment or other configuration file.\n' % (d.getVar('MACHINE'))) machinevalid = False else: status.addresult(check_sanity_validmachine(d)) @@ -741,7 +794,7 @@ def check_sanity_everybuild(status, d): status.addresult(check_toolchain(d)) # Check that the SDKMACHINE is valid, if it is set - if d.getVar('SDKMACHINE', True): + if d.getVar('SDKMACHINE'): if not check_conf_exists("conf/machine-sdk/${SDKMACHINE}.conf", d): status.addresult('Specified SDKMACHINE value is not valid\n') elif d.getVar('SDK_ARCH', False) == "${BUILD_ARCH}": @@ -749,20 +802,12 @@ def check_sanity_everybuild(status, d): check_supported_distro(d) - # Check if DISPLAY is set if TEST_IMAGE is set - if d.getVar('TEST_IMAGE', True) == '1' or d.getVar('DEFAULT_TEST_SUITES', True): - testtarget = d.getVar('TEST_TARGET', True) - if testtarget == 'qemu' or testtarget == 'QemuTarget': - display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True) - if not display: - status.addresult('testimage needs an X desktop to start qemu, please set DISPLAY correctly (e.g. DISPLAY=:1.0)\n') - - omask = os.umask(022) - if omask & 0755: + omask = os.umask(0o022) + if omask & 0o755: status.addresult("Please use a umask which allows a+rx and u+rwx\n") os.umask(omask) - if d.getVar('TARGET_ARCH', True) == "arm": + if d.getVar('TARGET_ARCH') == "arm": # This path is no longer user-readable in modern (very recent) Linux try: if os.path.exists("/proc/sys/vm/mmap_min_addr"): @@ -775,7 +820,7 @@ def check_sanity_everybuild(status, d): except: pass - oeroot = d.getVar('COREBASE', True) + oeroot = d.getVar('COREBASE') if oeroot.find('+') != -1: status.addresult("Error, you have an invalid character (+) in your COREBASE directory path. Please move the installation to a directory which doesn't include any + characters.") if oeroot.find('@') != -1: @@ -788,20 +833,18 @@ def check_sanity_everybuild(status, d): mirror_vars = ['MIRRORS', 'PREMIRRORS', 'SSTATE_MIRRORS'] protocols = ['http', 'ftp', 'file', 'https', \ 'git', 'gitsm', 'hg', 'osc', 'p4', 'svn', \ - 'bzr', 'cvs'] + 'bzr', 'cvs', 'npm', 'sftp', 'ssh', 's3' ] for mirror_var in mirror_vars: - mirrors = (d.getVar(mirror_var, True) or '').replace('\\n', '\n').split('\n') - for mirror_entry in mirrors: - mirror_entry = mirror_entry.strip() - if not mirror_entry: - # ignore blank lines - continue + mirrors = (d.getVar(mirror_var) or '').replace('\\n', ' ').split() - try: - pattern, mirror = mirror_entry.split() - except ValueError: - bb.warn('Invalid %s: %s, should be 2 members.' % (mirror_var, mirror_entry.strip())) - continue + # Split into pairs + if len(mirrors) % 2 != 0: + bb.warn('Invalid mirror variable value for %s: %s, should contain paired members.' % (mirror_var, mirrors.strip())) + continue + mirrors = list(zip(*[iter(mirrors)]*2)) + + for mirror_entry in mirrors: + pattern, mirror = mirror_entry decoded = bb.fetch2.decodeurl(pattern) try: @@ -819,19 +862,17 @@ def check_sanity_everybuild(status, d): continue if mirror.startswith('file://'): - if not mirror.startswith('file:///'): - bb.warn('Invalid file url in %s: %s, must be absolute path (file:///)' % (mirror_var, mirror_entry)) - import urlparse - check_symlink(urlparse.urlparse(mirror).path, d) + import urllib + check_symlink(urllib.parse.urlparse(mirror).path, d) # SSTATE_MIRROR ends with a /PATH string if mirror.endswith('/PATH'): # remove /PATH$ from SSTATE_MIRROR to get a working # base directory path - mirror_base = urlparse.urlparse(mirror[:-1*len('/PATH')]).path + mirror_base = urllib.parse.urlparse(mirror[:-1*len('/PATH')]).path check_symlink(mirror_base, d) # Check that TMPDIR hasn't changed location since the last time we were run - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') checkfile = os.path.join(tmpdir, "saved_tmpdir") if os.path.exists(checkfile): with open(checkfile, "r") as f: @@ -841,35 +882,28 @@ def check_sanity_everybuild(status, d): else: bb.utils.mkdirhier(tmpdir) # Remove setuid, setgid and sticky bits from TMPDIR - os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISUID) - os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISGID) - os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISVTX) + try: + os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISUID) + os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISGID) + os.chmod(tmpdir, os.stat(tmpdir).st_mode & ~ stat.S_ISVTX) + except OSError as exc: + bb.warn("Unable to chmod TMPDIR: %s" % exc) with open(checkfile, "w") as f: f.write(tmpdir) - # Check vmdk and live can't be built together. - if 'vmdk' in d.getVar('IMAGE_FSTYPES', True) and 'live' in d.getVar('IMAGE_FSTYPES', True): - status.addresult("Error, IMAGE_FSTYPES vmdk and live can't be built together\n") - - # Check vdi and live can't be built together. - if 'vdi' in d.getVar('IMAGE_FSTYPES', True) and 'live' in d.getVar('IMAGE_FSTYPES', True): - status.addresult("Error, IMAGE_FSTYPES vdi and live can't be built together\n") - - # Check qcow2 and live can't be built together. - if 'qcow2' in d.getVar('IMAGE_FSTYPES', True) and 'live' in d.getVar('IMAGE_FSTYPES', True): - status.addresult("Error, IMAGE_FSTYPES qcow2 and live can't be built together\n") - - # Check /bin/sh links to dash or bash - real_sh = os.path.realpath('/bin/sh') - if not real_sh.endswith('/dash') and not real_sh.endswith('/bash'): - status.addresult("Error, /bin/sh links to %s, must be dash or bash\n" % real_sh) + # If /bin/sh is a symlink, check that it points to dash or bash + if os.path.islink('/bin/sh'): + real_sh = os.path.realpath('/bin/sh') + # Due to update-alternatives, the shell name may take various + # forms, such as /bin/dash, bin/bash, /bin/bash.bash ... + if '/dash' not in real_sh and '/bash' not in real_sh: + status.addresult("Error, /bin/sh links to %s, must be dash or bash\n" % real_sh) def check_sanity(sanity_data): class SanityStatus(object): def __init__(self): self.messages = "" self.network_error = False - self.reparse = False def addresult(self, message): if message: @@ -877,8 +911,8 @@ def check_sanity(sanity_data): status = SanityStatus() - tmpdir = sanity_data.getVar('TMPDIR', True) - sstate_dir = sanity_data.getVar('SSTATE_DIR', True) + tmpdir = sanity_data.getVar('TMPDIR') + sstate_dir = sanity_data.getVar('SSTATE_DIR') check_symlink(sstate_dir, sanity_data) @@ -886,6 +920,7 @@ def check_sanity(sanity_data): last_sanity_version = 0 last_tmpdir = "" last_sstate_dir = "" + last_nativelsbstr = "" sanityverfile = sanity_data.expand("${TOPDIR}/conf/sanity_info") if os.path.exists(sanityverfile): with open(sanityverfile, 'r') as f: @@ -896,12 +931,17 @@ def check_sanity(sanity_data): last_tmpdir = line.split()[1] if line.startswith('SSTATE_DIR'): last_sstate_dir = line.split()[1] + if line.startswith('NATIVELSBSTRING'): + last_nativelsbstr = line.split()[1] check_sanity_everybuild(status, sanity_data) - sanity_version = int(sanity_data.getVar('SANITY_VERSION', True) or 1) + sanity_version = int(sanity_data.getVar('SANITY_VERSION') or 1) network_error = False - if last_sanity_version < sanity_version: + # NATIVELSBSTRING var may have been overridden with "universal", so + # get actual host distribution id and version + nativelsbstr = lsb_distro_identifier(sanity_data) + if last_sanity_version < sanity_version or last_nativelsbstr != nativelsbstr: check_sanity_version_change(status, sanity_data) status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) else: @@ -913,12 +953,12 @@ def check_sanity(sanity_data): f.write("SANITY_VERSION %s\n" % sanity_version) f.write("TMPDIR %s\n" % tmpdir) f.write("SSTATE_DIR %s\n" % sstate_dir) + f.write("NATIVELSBSTRING %s\n" % nativelsbstr) sanity_handle_abichanges(status, sanity_data) if status.messages != "": raise_sanity_error(sanity_data.expand(status.messages), sanity_data, status.network_error) - return status.reparse # Create a copy of the datastore and finalise it to ensure appends and # overrides are set - the datastore has yet to be finalised at ConfigParsed @@ -927,15 +967,20 @@ def copy_data(e): sanity_data.finalize() return sanity_data +addhandler config_reparse_eventhandler +config_reparse_eventhandler[eventmask] = "bb.event.ConfigParsed" +python config_reparse_eventhandler() { + sanity_check_conffiles(e.data) +} + addhandler check_sanity_eventhandler check_sanity_eventhandler[eventmask] = "bb.event.SanityCheck bb.event.NetworkTest" python check_sanity_eventhandler() { if bb.event.getName(e) == "SanityCheck": sanity_data = copy_data(e) + check_sanity(sanity_data) if e.generateevents: sanity_data.setVar("SANITY_USE_EVENTS", "1") - reparse = check_sanity(sanity_data) - e.data.setVar("BB_INVALIDCONF", reparse) bb.event.fire(bb.event.SanityCheckPassed(), e.data) elif bb.event.getName(e) == "NetworkTest": sanity_data = copy_data(e) diff --git a/meta/classes/scons.bbclass b/meta/classes/scons.bbclass index b8de822ff7..b9ae19d582 100644 --- a/meta/classes/scons.bbclass +++ b/meta/classes/scons.bbclass @@ -2,13 +2,15 @@ DEPENDS += "python-scons-native" EXTRA_OESCONS ?= "" +do_configure[noexec] = "1" + scons_do_compile() { ${STAGING_BINDIR_NATIVE}/scons ${PARALLEL_MAKE} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} || \ die "scons build execution failed." } scons_do_install() { - ${STAGING_BINDIR_NATIVE}/scons PREFIX=${D}${prefix} prefix=${D}${prefix} install ${EXTRA_OESCONS}|| \ + ${STAGING_BINDIR_NATIVE}/scons install_root=${D}${prefix} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} install || \ die "scons install execution failed." } diff --git a/meta/classes/sdl.bbclass b/meta/classes/sdl.bbclass deleted file mode 100644 index cc31288f61..0000000000 --- a/meta/classes/sdl.bbclass +++ /dev/null @@ -1,6 +0,0 @@ -# -# (C) Michael 'Mickey' Lauer <mickey@Vanille.de> -# - -DEPENDS += "virtual/libsdl libsdl-mixer libsdl-image" -SECTION = "x11/games" diff --git a/meta/classes/sign_ipk.bbclass b/meta/classes/sign_ipk.bbclass new file mode 100644 index 0000000000..e5057b7799 --- /dev/null +++ b/meta/classes/sign_ipk.bbclass @@ -0,0 +1,52 @@ +# Class for generating signed IPK packages. +# +# Configuration variables used by this class: +# IPK_GPG_PASSPHRASE_FILE +# Path to a file containing the passphrase of the signing key. +# IPK_GPG_NAME +# Name of the key to sign with. +# IPK_GPG_BACKEND +# Optional variable for specifying the backend to use for signing. +# Currently the only available option is 'local', i.e. local signing +# on the build host. +# IPK_GPG_SIGNATURE_TYPE +# Optional variable for specifying the type of gpg signatures, can be: +# 1. Ascii armored (ASC), default if not set +# 2. Binary (BIN) +# GPG_BIN +# Optional variable for specifying the gpg binary/wrapper to use for +# signing. +# GPG_PATH +# Optional variable for specifying the gnupg "home" directory: +# + +inherit sanity + +IPK_SIGN_PACKAGES = '1' +IPK_GPG_BACKEND ?= 'local' +IPK_GPG_SIGNATURE_TYPE ?= 'ASC' + +python () { + # Check configuration + for var in ('IPK_GPG_NAME', 'IPK_GPG_PASSPHRASE_FILE'): + if not d.getVar(var): + raise_sanity_error("You need to define %s in the config" % var, d) + + sigtype = d.getVar("IPK_GPG_SIGNATURE_TYPE") + if sigtype.upper() != "ASC" and sigtype.upper() != "BIN": + raise_sanity_error("Bad value for IPK_GPG_SIGNATURE_TYPE (%s), use either ASC or BIN" % sigtype) +} + +def sign_ipk(d, ipk_to_sign): + from oe.gpg_sign import get_signer + + bb.debug(1, 'Signing ipk: %s' % ipk_to_sign) + + signer = get_signer(d, d.getVar('IPK_GPG_BACKEND')) + sig_type = d.getVar('IPK_GPG_SIGNATURE_TYPE') + is_ascii_sig = (sig_type.upper() != "BIN") + + signer.detach_sign(ipk_to_sign, + d.getVar('IPK_GPG_NAME'), + d.getVar('IPK_GPG_PASSPHRASE_FILE'), + armor=is_ascii_sig) diff --git a/meta/classes/sign_package_feed.bbclass b/meta/classes/sign_package_feed.bbclass new file mode 100644 index 0000000000..71df03bab3 --- /dev/null +++ b/meta/classes/sign_package_feed.bbclass @@ -0,0 +1,43 @@ +# Class for signing package feeds +# +# Related configuration variables that will be used after this class is +# iherited: +# PACKAGE_FEED_PASSPHRASE_FILE +# Path to a file containing the passphrase of the signing key. +# PACKAGE_FEED_GPG_NAME +# Name of the key to sign with. May be key id or key name. +# PACKAGE_FEED_GPG_BACKEND +# Optional variable for specifying the backend to use for signing. +# Currently the only available option is 'local', i.e. local signing +# on the build host. +# PACKAGE_FEED_GPG_SIGNATURE_TYPE +# Optional variable for specifying the type of gpg signature, can be: +# 1. Ascii armored (ASC), default if not set +# 2. Binary (BIN) +# This variable is only available for IPK feeds. It is ignored on +# other packaging backends. +# GPG_BIN +# Optional variable for specifying the gpg binary/wrapper to use for +# signing. +# GPG_PATH +# Optional variable for specifying the gnupg "home" directory: +# +inherit sanity + +PACKAGE_FEED_SIGN = '1' +PACKAGE_FEED_GPG_BACKEND ?= 'local' +PACKAGE_FEED_GPG_SIGNATURE_TYPE ?= 'ASC' + +python () { + # Check sanity of configuration + for var in ('PACKAGE_FEED_GPG_NAME', 'PACKAGE_FEED_GPG_PASSPHRASE_FILE'): + if not d.getVar(var): + raise_sanity_error("You need to define %s in the config" % var, d) + + sigtype = d.getVar("PACKAGE_FEED_GPG_SIGNATURE_TYPE") + if sigtype.upper() != "ASC" and sigtype.upper() != "BIN": + raise_sanity_error("Bad value for PACKAGE_FEED_GPG_SIGNATURE_TYPE (%s), use either ASC or BIN" % sigtype) +} + +do_package_index[depends] += "signing-keys:do_deploy" +do_rootfs[depends] += "signing-keys:do_populate_sysroot" diff --git a/meta/classes/sign_rpm.bbclass b/meta/classes/sign_rpm.bbclass index 0aa4cd841c..bc2e947107 100644 --- a/meta/classes/sign_rpm.bbclass +++ b/meta/classes/sign_rpm.bbclass @@ -1,75 +1,46 @@ # Class for generating signed RPM packages. # # Configuration variables used by this class: -# RPM_GPG_PASSPHRASE_FILE -# Path to a file containing the passphrase of the signing key. +# RPM_GPG_PASSPHRASE +# The passphrase of the signing key. # RPM_GPG_NAME -# Name of the key to sign with. Alternatively you can define -# %_gpg_name macro in your ~/.oerpmmacros file. -# RPM_GPG_PUBKEY -# Path to a file containing the public key (in "armor" format) -# corresponding the signing key. +# Name of the key to sign with. May be key id or key name. +# RPM_GPG_BACKEND +# Optional variable for specifying the backend to use for signing. +# Currently the only available option is 'local', i.e. local signing +# on the build host. # GPG_BIN # Optional variable for specifying the gpg binary/wrapper to use for # signing. +# GPG_PATH +# Optional variable for specifying the gnupg "home" directory: # inherit sanity RPM_SIGN_PACKAGES='1' +RPM_GPG_BACKEND ?= 'local' -_check_gpg_name () { - macrodef=`rpm -E '%_gpg_name'` - [ "$macrodef" == "%_gpg_name" ] && return 1 || return 0 +python () { + if d.getVar('RPM_GPG_PASSPHRASE_FILE'): + raise_sanity_error('RPM_GPG_PASSPHRASE_FILE is replaced by RPM_GPG_PASSPHRASE', d) + # Check configuration + for var in ('RPM_GPG_NAME', 'RPM_GPG_PASSPHRASE'): + if not d.getVar(var): + raise_sanity_error("You need to define %s in the config" % var, d) } - -def rpmsign_wrapper(d, files, passphrase, gpg_name=None): - import pexpect - - # Find the correct rpm binary - rpm_bin_path = d.getVar('STAGING_BINDIR_NATIVE', True) + '/rpm' - cmd = rpm_bin_path + " --addsign " - if gpg_name: - cmd += "--define '%%_gpg_name %s' " % gpg_name - else: - try: - bb.build.exec_func('_check_gpg_name', d) - except bb.build.FuncFailed: - raise_sanity_error("You need to define RPM_GPG_NAME in bitbake " - "config or the %_gpg_name RPM macro defined " - "(e.g. in ~/.oerpmmacros", d) - if d.getVar('GPG_BIN', True): - cmd += "--define '%%__gpg %s' " % d.getVar('GPG_BIN', True) - cmd += ' '.join(files) - - # Need to use pexpect for feeding the passphrase - proc = pexpect.spawn(cmd) - try: - proc.expect_exact('Enter pass phrase:', timeout=15) - proc.sendline(passphrase) - proc.expect(pexpect.EOF, timeout=900) - proc.close() - except pexpect.TIMEOUT as err: - bb.debug('rpmsign timeout: %s' % err) - proc.terminate() - return proc.exitstatus - - python sign_rpm () { import glob + from oe.gpg_sign import get_signer - rpm_gpg_pass_file = (d.getVar("RPM_GPG_PASSPHRASE_FILE", True) or "") - if rpm_gpg_pass_file: - with open(rpm_gpg_pass_file) as fobj: - rpm_gpg_passphrase = fobj.readlines()[0].rstrip('\n') - else: - raise_sanity_error("You need to define RPM_GPG_PASSPHRASE_FILE in the config", d) - - rpm_gpg_name = (d.getVar("RPM_GPG_NAME", True) or "") + signer = get_signer(d, d.getVar('RPM_GPG_BACKEND')) + rpms = glob.glob(d.getVar('RPM_PKGWRITEDIR') + '/*') - rpms = glob.glob(d.getVar('RPM_PKGWRITEDIR', True) + '/*') - - if rpmsign_wrapper(d, rpms, rpm_gpg_passphrase, rpm_gpg_name) != 0: - raise bb.build.FuncFailed("RPM signing failed") + signer.sign_rpms(rpms, + d.getVar('RPM_GPG_NAME'), + d.getVar('RPM_GPG_PASSPHRASE')) } + +do_package_index[depends] += "signing-keys:do_deploy" +do_rootfs[depends] += "signing-keys:do_populate_sysroot" diff --git a/meta/classes/sip.bbclass b/meta/classes/sip.bbclass deleted file mode 100644 index 6ed2a13bda..0000000000 --- a/meta/classes/sip.bbclass +++ /dev/null @@ -1,61 +0,0 @@ -# Build Class for Sip based Python Bindings -# (C) Michael 'Mickey' Lauer <mickey@Vanille.de> -# -STAGING_SIPDIR ?= "${STAGING_DATADIR_NATIVE}/sip" - -DEPENDS =+ "sip-native" -RDEPENDS_${PN} += "python-sip" - -# default stuff, do not uncomment -# EXTRA_SIPTAGS = "-tWS_X11 -tQt_4_3_0" - -# do_generate is before do_configure so ensure that sip_native is populated in sysroot before executing it -do_generate[depends] += "sip-native:do_populate_sysroot" - -sip_do_generate() { - if [ -z "${SIP_MODULES}" ]; then - MODULES="`ls sip/*mod.sip`" - else - MODULES="${SIP_MODULES}" - fi - - if [ -z "$MODULES" ]; then - die "SIP_MODULES not set and no modules found in $PWD" - else - bbnote "using modules '${SIP_MODULES}' and tags '${EXTRA_SIPTAGS}'" - fi - - if [ -z "${EXTRA_SIPTAGS}" ]; then - die "EXTRA_SIPTAGS needs to be set!" - else - SIPTAGS="${EXTRA_SIPTAGS}" - fi - - if [ ! -z "${SIP_FEATURES}" ]; then - FEATURES="-z ${SIP_FEATURES}" - bbnote "sip feature file: ${SIP_FEATURES}" - fi - - for module in $MODULES - do - install -d ${module}/ - echo "calling 'sip4 -I sip -I ${STAGING_SIPDIR} ${SIPTAGS} ${FEATURES} -c ${module} -b ${module}/${module}.pro.in sip/${module}/${module}mod.sip'" - sip4 -I ${STAGING_SIPDIR} -I sip ${SIPTAGS} ${FEATURES} -c ${module} -b ${module}/${module}.sbf \ - sip/${module}/${module}mod.sip || die "Error calling sip on ${module}" - sed -e 's,target,TARGET,' -e 's,sources,SOURCES,' -e 's,headers,HEADERS,' \ - ${module}/${module}.sbf | sed s,"moc_HEADERS =","HEADERS +=", \ - >${module}/${module}.pro - echo "TEMPLATE=lib" >>${module}/${module}.pro - [ "${module}" = "qt" ] && echo "" >>${module}/${module}.pro - [ "${module}" = "qtcanvas" ] && echo "" >>${module}/${module}.pro - [ "${module}" = "qttable" ] && echo "" >>${module}/${module}.pro - [ "${module}" = "qwt" ] && echo "" >>${module}/${module}.pro - [ "${module}" = "qtpe" ] && echo "" >>${module}/${module}.pro - [ "${module}" = "qtpe" ] && echo "LIBS+=-lqpe" >>${module}/${module}.pro - true - done -} - -EXPORT_FUNCTIONS do_generate - -addtask generate after do_unpack do_patch before do_configure diff --git a/meta/classes/siteconfig.bbclass b/meta/classes/siteconfig.bbclass index 45dce489de..bb491d2994 100644 --- a/meta/classes/siteconfig.bbclass +++ b/meta/classes/siteconfig.bbclass @@ -2,12 +2,12 @@ python siteconfig_do_siteconfig () { shared_state = sstate_state_fromvars(d) if shared_state['task'] != 'populate_sysroot': return - if not os.path.isdir(os.path.join(d.getVar('FILE_DIRNAME', True), 'site_config')): + if not os.path.isdir(os.path.join(d.getVar('FILE_DIRNAME'), 'site_config')): bb.debug(1, "No site_config directory, skipping do_siteconfig") return + sstate_install(shared_state, d) bb.build.exec_func('do_siteconfig_gencache', d) sstate_clean(shared_state, d) - sstate_install(shared_state, d) } EXTRASITECONFIG ?= "" diff --git a/meta/classes/siteinfo.bbclass b/meta/classes/siteinfo.bbclass index 9f6eac4e1e..2c33732be3 100644 --- a/meta/classes/siteinfo.bbclass +++ b/meta/classes/siteinfo.bbclass @@ -8,7 +8,7 @@ # # 'what' can be one of # * target: Returns the target name ("<arch>-<os>") -# * endianess: Return "be" for big endian targets, "le" for little endian +# * endianness: Return "be" for big endian targets, "le" for little endian # * bits: Returns the bit size of the target, either "32" or "64" # * libc: Returns the name of the c library used by the target # @@ -36,12 +36,17 @@ def siteinfo_data(d): "mips": "endian-big bit-32 mips-common", "mips64": "endian-big bit-64 mips-common", "mips64el": "endian-little bit-64 mips-common", + "mipsisa64r6": "endian-big bit-64 mips-common", + "mipsisa64r6el": "endian-little bit-64 mips-common", "mipsel": "endian-little bit-32 mips-common", + "mipsisa32r6": "endian-big bit-32 mips-common", + "mipsisa32r6el": "endian-little bit-32 mips-common", "powerpc": "endian-big bit-32 powerpc-common", "nios2": "endian-little bit-32 nios2-common", "powerpc64": "endian-big bit-64 powerpc-common", "ppc": "endian-big bit-32 powerpc-common", "ppc64": "endian-big bit-64 powerpc-common", + "ppc64le" : "endian-little bit-64 powerpc-common", "sh3": "endian-little bit-32 sh-common", "sh4": "endian-little bit-32 sh-common", "sparc": "endian-big bit-32", @@ -70,6 +75,8 @@ def siteinfo_data(d): targetinfo = { "aarch64-linux-gnu": "aarch64-linux", "aarch64_be-linux-gnu": "aarch64_be-linux", + "aarch64-linux-musl": "aarch64-linux", + "aarch64_be-linux-musl": "aarch64_be-linux", "arm-linux-gnueabi": "arm-linux", "arm-linux-musleabi": "arm-linux", "arm-linux-uclibceabi": "arm-linux-uclibc", @@ -78,10 +85,12 @@ def siteinfo_data(d): "armeb-linux-musleabi": "armeb-linux", "mips-linux-musl": "mips-linux", "mipsel-linux-musl": "mipsel-linux", - "mips64-linux-musl": "mips-linux", - "mips64el-linux-musl": "mipsel-linux", + "mips64-linux-musl": "mips64-linux", + "mips64el-linux-musl": "mips64el-linux", "mips64-linux-gnun32": "mips-linux bit-32", "mips64el-linux-gnun32": "mipsel-linux bit-32", + "mipsisa64r6-linux-gnun32": "mipsisa32r6-linux bit-32", + "mipsisa64r6el-linux-gnun32": "mipsisa32r6el-linux bit-32", "powerpc-linux": "powerpc32-linux", "powerpc-linux-musl": "powerpc-linux powerpc32-linux", "powerpc-linux-uclibc": "powerpc-linux powerpc32-linux", @@ -91,6 +100,7 @@ def siteinfo_data(d): "powerpc64-linux-gnuspe": "powerpc-linux powerpc64-linux", "powerpc64-linux-muslspe": "powerpc-linux powerpc64-linux", "powerpc64-linux": "powerpc-linux", + "powerpc64-linux-musl": "powerpc-linux", "x86_64-cygwin": "bit-64", "x86_64-darwin": "bit-64", "x86_64-darwin9": "bit-64", @@ -103,8 +113,16 @@ def siteinfo_data(d): "x86_64-mingw32": "bit-64", } - hostarch = d.getVar("HOST_ARCH", True) - hostos = d.getVar("HOST_OS", True) + # Add in any extra user supplied data which may come from a BSP layer, removing the + # need to always change this class directly + extra_siteinfo = (d.getVar("SITEINFO_EXTRA_DATAFUNCS") or "").split() + for m in extra_siteinfo: + call = m + "(archinfo, osinfo, targetinfo, d)" + locs = { "archinfo" : archinfo, "osinfo" : osinfo, "targetinfo" : targetinfo, "d" : d} + archinfo, osinfo, targetinfo = bb.utils.better_eval(call, locs) + + hostarch = d.getVar("HOST_ARCH") + hostos = d.getVar("HOST_OS") target = "%s-%s" % (hostarch, hostos) sitedata = [] @@ -128,7 +146,7 @@ python () { d.setVar("SITEINFO_ENDIANNESS", "be") else: bb.error("Unable to determine endianness for architecture '%s'" % - d.getVar("HOST_ARCH", True)) + d.getVar("HOST_ARCH")) bb.fatal("Please add your architecture to siteinfo.bbclass") if "bit-32" in sitedata: @@ -137,32 +155,37 @@ python () { d.setVar("SITEINFO_BITS", "64") else: bb.error("Unable to determine bit size for architecture '%s'" % - d.getVar("HOST_ARCH", True)) + d.getVar("HOST_ARCH")) bb.fatal("Please add your architecture to siteinfo.bbclass") } -def siteinfo_get_files(d, no_cache = False): +def siteinfo_get_files(d, aclocalcache = False): sitedata = siteinfo_data(d) sitefiles = "" - for path in d.getVar("BBPATH", True).split(":"): + for path in d.getVar("BBPATH").split(":"): for element in sitedata: filename = os.path.join(path, "site", element) if os.path.exists(filename): sitefiles += filename + " " - if no_cache: return sitefiles + if not aclocalcache: + return sitefiles - # Now check for siteconfig cache files - # Use the files copied to the aclocal cache generated by autotools.bbclass - # to avoid races - path_siteconfig = d.getVar('ACLOCALDIR', True) + # Now check for siteconfig cache files in the directory setup by autotools.bbclass to + # avoid races. + # + # ACLOCALDIR may or may not exist so cache should only be set to True from autotools.bbclass + # after files have been copied into this location. To do otherwise risks parsing/signature + # issues and the directory being created/removed whilst this code executes. This can happen + # when a multilib recipe is parsed along with its base variant which may be running at the time + # causing rare but nasty failures + path_siteconfig = d.getVar('ACLOCALDIR') if path_siteconfig and os.path.isdir(path_siteconfig): for i in os.listdir(path_siteconfig): if not i.endswith("_config"): continue filename = os.path.join(path_siteconfig, i) sitefiles += filename + " " - return sitefiles # diff --git a/meta/classes/spdx.bbclass b/meta/classes/spdx.bbclass index 454c53e96f..c5f544d2a4 100644 --- a/meta/classes/spdx.bbclass +++ b/meta/classes/spdx.bbclass @@ -26,20 +26,20 @@ python do_spdx () { import json, shutil info = {} - info['workdir'] = d.getVar('WORKDIR', True) - info['sourcedir'] = d.getVar('SPDX_S', True) - info['pn'] = d.getVar('PN', True) - info['pv'] = d.getVar('PV', True) - info['spdx_version'] = d.getVar('SPDX_VERSION', True) - info['data_license'] = d.getVar('DATA_LICENSE', True) - - sstatedir = d.getVar('SPDXSSTATEDIR', True) + info['workdir'] = d.getVar('WORKDIR') + info['sourcedir'] = d.getVar('SPDX_S') + info['pn'] = d.getVar('PN') + info['pv'] = d.getVar('PV') + info['spdx_version'] = d.getVar('SPDX_VERSION') + info['data_license'] = d.getVar('DATA_LICENSE') + + sstatedir = d.getVar('SPDXSSTATEDIR') sstatefile = os.path.join(sstatedir, info['pn'] + info['pv'] + ".spdx") - manifest_dir = d.getVar('SPDX_MANIFEST_DIR', True) + manifest_dir = d.getVar('SPDX_MANIFEST_DIR') info['outfile'] = os.path.join(manifest_dir, info['pn'] + ".spdx" ) - info['spdx_temp_dir'] = d.getVar('SPDX_TEMP_DIR', True) + info['spdx_temp_dir'] = d.getVar('SPDX_TEMP_DIR') info['tar_file'] = os.path.join(info['workdir'], info['pn'] + ".tar.gz" ) # Make sure important dirs exist @@ -74,9 +74,9 @@ python do_spdx () { foss_license_info = cached_spdx['Licenses'] else: ## setup fossology command - foss_server = d.getVar('FOSS_SERVER', True) - foss_flags = d.getVar('FOSS_WGET_FLAGS', True) - foss_full_spdx = d.getVar('FOSS_FULL_SPDX', True) == "true" or False + foss_server = d.getVar('FOSS_SERVER') + foss_flags = d.getVar('FOSS_WGET_FLAGS') + foss_full_spdx = d.getVar('FOSS_FULL_SPDX') == "true" or False foss_command = "wget %s --post-file=%s %s"\ % (foss_flags, info['tar_file'], foss_server) @@ -219,20 +219,19 @@ def hash_string(data): def run_fossology(foss_command, full_spdx): import string, re import subprocess - - p = subprocess.Popen(foss_command.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - foss_output, foss_error = p.communicate() - if p.returncode != 0: + + try: + foss_output = subprocess.check_output(foss_command.split(), + stderr=subprocess.STDOUT).decode('utf-8') + except subprocess.CalledProcessError as e: return None - foss_output = unicode(foss_output, "utf-8") foss_output = string.replace(foss_output, '\r', '') # Package info package_info = {} if full_spdx: - # All mandatory, only one occurance + # All mandatory, only one occurrence package_info['PackageCopyrightText'] = re.findall('PackageCopyrightText: (.*?</text>)', foss_output, re.S)[0] package_info['PackageLicenseDeclared'] = re.findall('PackageLicenseDeclared: (.*)', foss_output)[0] package_info['PackageLicenseConcluded'] = re.findall('PackageLicenseConcluded: (.*)', foss_output)[0] diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index 4e6afef541..ddc442cdf9 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass @@ -10,28 +10,32 @@ def generate_sstatefn(spec, hash, d): SSTATE_PKGARCH = "${PACKAGE_ARCH}" SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" -SSTATE_SWSPEC = "sstate:${BPN}::${PV}:${PR}::${SSTATE_VERSION}:" -SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), d.getVar('BB_TASKHASH', True), d)}" +SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:" +SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_TASKHASH'), d)}" SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" SSTATE_EXTRAPATH = "" SSTATE_EXTRAPATHWILDCARD = "" SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}" +# explicitly make PV to depend on evaluated value of PV variable +PV[vardepvalue] = "${PV}" + # We don't want the sstate to depend on things like the distro string # of the system, we let the sstate paths take care of this. SSTATE_EXTRAPATH[vardepvalue] = "" # For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical) -SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/ ${DEPLOY_DIR_RPM}/all/" +SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/ ${DEPLOY_DIR_RPM}/noarch/" # Avoid docbook/sgml catalog warnings for now SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml" # Archive the sources for many architectures in one deploy folder SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}" -SSTATE_SCAN_FILES ?= "*.la *-config *_config" -SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f' +SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*" +SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f' +SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} ${SSTATE_BUILDDIR}' -BB_HASHFILENAME = "${SSTATE_EXTRAPATH} ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}" +BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}" SSTATE_ARCHS = " \ ${BUILD_ARCH} \ @@ -50,19 +54,23 @@ SSTATEPOSTCREATEFUNCS = "" SSTATEPREINSTFUNCS = "" SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack" SSTATEPOSTINSTFUNCS = "" -EXTRA_STAGING_FIXMES ?= "" +EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR" +SSTATECLEANFUNCS = "" -SIGGEN_LOCKEDSIGS_CHECK_LEVEL ?= 'error' +# Check whether sstate exists for tasks that support sstate and are in the +# locked signatures file. +SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error' -# Specify dirs in which the shell function is executed and don't use ${B} -# as default dirs to avoid possible race about ${B} with other task. -sstate_create_package[dirs] = "${SSTATE_BUILDDIR}" -sstate_unpack_package[dirs] = "${SSTATE_INSTDIR}" +# Check whether the task's computed hash matches the task's hash in the +# locked signatures file. +SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error" -# Do not run sstate_hardcode_path() in ${B}: -# the ${B} maybe removed by cmake_do_configure() while -# sstate_hardcode_path() running. -sstate_hardcode_path[dirs] = "${SSTATE_BUILDDIR}" +# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to +# not sign) +SSTATE_SIG_KEY ?= "" +SSTATE_SIG_PASSPHRASE ?= "" +# Whether to verify the GnUPG signatures when extracting sstate archives +SSTATE_VERIFY_SIG ?= "0" python () { if bb.data.inherits_class('native', d): @@ -75,22 +83,17 @@ python () { d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}")) elif bb.data.inherits_class('cross-canadian', d): d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}")) - elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH", True) == "all": + elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all": d.setVar('SSTATE_PKGARCH', "allarch") else: d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}")) if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d): d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") + d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}") d.setVar('SSTATE_EXTRAPATHWILDCARD', "*/") - # These classes encode staging paths into their scripts data so can only be - # reused if we manipulate the paths - if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d): - scan_cmd = "grep -Irl ${STAGING_DIR} ${SSTATE_BUILDDIR}" - d.setVar('SSTATE_SCAN_CMD', scan_cmd) - - unique_tasks = set((d.getVar('SSTATETASKS', True) or "").split()) + unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split())) d.setVar('SSTATETASKS', " ".join(unique_tasks)) for task in unique_tasks: d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ") @@ -108,25 +111,27 @@ def sstate_init(task, d): def sstate_state_fromvars(d, task = None): if task is None: - task = d.getVar('BB_CURRENTTASK', True) + task = d.getVar('BB_CURRENTTASK') if not task: bb.fatal("sstate code running without task context?!") task = task.replace("_setscene", "") if task.startswith("do_"): task = task[3:] - inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs', True) or "").split() - outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs', True) or "").split() - plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs', True) or "").split() - lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile', True) or "").split() - lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared', True) or "").split() - interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs', True) or "").split() + inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split() + outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split() + plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split() + lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split() + lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split() + interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split() + fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or "" if not task or len(inputs) != len(outputs): bb.fatal("sstate variables not setup correctly?!") if task == "populate_lic": d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") d.setVar("SSTATE_EXTRAPATH", "") + d.setVar('SSTATE_EXTRAPATHWILDCARD', "") ss = sstate_init(task, d) for i in range(len(inputs)): @@ -135,6 +140,7 @@ def sstate_state_fromvars(d, task = None): ss['lockfiles-shared'] = lockfilesshared ss['plaindirs'] = plaindirs ss['interceptfuncs'] = interceptfuncs + ss['fixmedir'] = fixmedir return ss def sstate_add(ss, source, dest, d): @@ -150,21 +156,22 @@ def sstate_add(ss, source, dest, d): def sstate_install(ss, d): import oe.path + import oe.sstatesig import subprocess sharedfiles = [] shareddirs = [] bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) - d2 = d.createCopy() - extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info', True) - if extrainf: - d2.setVar("SSTATE_MANMACH", extrainf) - manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task']) + sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) + + manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d) if os.access(manifest, os.R_OK): bb.fatal("Package already staged (%s)?!" % manifest) + d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm") + locks = [] for lock in ss['lockfiles-shared']: locks.append(bb.utils.lockfile(lock, True)) @@ -183,42 +190,48 @@ def sstate_install(ss, d): srcdir = os.path.join(walkroot, dir) dstdir = srcdir.replace(state[1], state[2]) #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir)) + if os.path.islink(srcdir): + sharedfiles.append(dstdir) + continue if not dstdir.endswith("/"): dstdir = dstdir + "/" shareddirs.append(dstdir) # Check the file list for conflicts against files which already exist - whitelist = (d.getVar("SSTATE_DUPWHITELIST", True) or "").split() + whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split() match = [] for f in sharedfiles: - if os.path.exists(f): + if os.path.exists(f) and not os.path.islink(f): f = os.path.normpath(f) realmatch = True for w in whitelist: + w = os.path.normpath(w) if f.startswith(w): realmatch = False break if realmatch: match.append(f) - sstate_search_cmd = "grep -rl '%s' %s --exclude=master.list | sed -e 's:^.*/::' -e 's:\.populate-sysroot::'" % (f, d.expand("${SSTATE_MANIFESTS}")) + sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}")) search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] - if search_output != "": - match.append("Matched in %s" % search_output.rstrip()) + if search_output: + match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip()) + else: + match.append(" (not matched to any task)") if match: bb.error("The recipe %s is trying to install files into a shared " \ "area when those files already exist. Those files and their manifest " \ - "location are:\n %s\nPlease verify which recipe should provide the " \ - "above files.\nThe build has stopped as continuing in this scenario WILL " \ - "break things, if not now, possibly in the future (we've seen builds fail " \ + "location are:\n %s\nPlease verify which recipe should provide the " \ + "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \ + "break things - if not now, possibly in the future (we've seen builds fail " \ "several months later). If the system knew how to recover from this " \ - "automatically it would however there are several different scenarios " \ + "automatically it would, however there are several different scenarios " \ "which can result in this and we don't know which one this is. It may be " \ "you have switched providers of something like virtual/kernel (e.g. from " \ "linux-yocto to linux-yocto-dev), in that case you need to execute the " \ "clean task for both recipes and it will resolve this error. It may be " \ "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \ - "those recipes should again resolve this error however switching " \ - "DISTRO_FEATURES on an existing build directory is not supported, you " \ + "those recipes should again resolve this error, however switching " \ + "DISTRO_FEATURES on an existing build directory is not supported - you " \ "should really clean out tmp and rebuild (reusing sstate should be safe). " \ "It could be the overlapping files detected are harmless in which case " \ "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \ @@ -226,9 +239,13 @@ def sstate_install(ss, d): "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \ "be to resolve the conflict. If in doubt, please ask on the mailing list, " \ "sharing the error and filelist above." % \ - (d.getVar('PN', True), "\n ".join(match))) + (d.getVar('PN'), "\n ".join(match))) bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.") + if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"): + sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd") + sharedfiles.append(ss['fixmedir'] + "/fixmepath") + # Write out the manifest f = open(manifest, "w") for file in sharedfiles: @@ -247,7 +264,7 @@ def sstate_install(ss, d): i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}") l = bb.utils.lockfile(i + ".lock") - filedata = d.getVar("STAMP", True) + " " + d2.getVar("SSTATE_MANFILEPREFIX", True) + " " + d.getVar("WORKDIR", True) + "\n" + filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n" manifests = [] if os.path.exists(i): with open(i, "r") as f: @@ -262,8 +279,9 @@ def sstate_install(ss, d): if os.path.exists(state[1]): oe.path.copyhardlinktree(state[1], state[2]) - for postinst in (d.getVar('SSTATEPOSTINSTFUNCS', True) or '').split(): - bb.build.exec_func(postinst, d) + for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split(): + # All hooks should run in the SSTATE_INSTDIR + bb.build.exec_func(postinst, d, (sstateinst,)) for lock in locks: bb.utils.unlockfile(lock) @@ -272,19 +290,11 @@ sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANF sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}" def sstate_installpkg(ss, d): - import oe.path - import subprocess - - def prepdir(dir): - # remove dir if it exists, ensure any parent directories do exist - if os.path.exists(dir): - oe.path.remove(dir) - bb.utils.mkdirhier(dir) - oe.path.remove(dir) + from oe.gpg_sign import get_signer sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) - sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['task'] + ".tgz" - sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['task'] + ".tgz" + sstatefetch = d.getVar('SSTATE_PKGNAME') + '_' + ss['task'] + ".tgz" + sstatepkg = d.getVar('SSTATE_PKG') + '_' + ss['task'] + ".tgz" if not os.path.exists(sstatepkg): pstaging_fetch(sstatefetch, sstatepkg, d) @@ -298,16 +308,52 @@ def sstate_installpkg(ss, d): d.setVar('SSTATE_INSTDIR', sstateinst) d.setVar('SSTATE_PKG', sstatepkg) - for f in (d.getVar('SSTATEPREINSTFUNCS', True) or '').split() + ['sstate_unpack_package'] + (d.getVar('SSTATEPOSTUNPACKFUNCS', True) or '').split(): - bb.build.exec_func(f, d) + if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False): + signer = get_signer(d, 'local') + if not signer.verify(sstatepkg + '.sig'): + bb.warn("Cannot verify signature on sstate package %s" % sstatepkg) + + # Empty sstateinst directory, ensure its clean + if os.path.exists(sstateinst): + oe.path.remove(sstateinst) + bb.utils.mkdirhier(sstateinst) + + sstateinst = d.getVar("SSTATE_INSTDIR") + d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) + + for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']: + # All hooks should run in the SSTATE_INSTDIR + bb.build.exec_func(f, d, (sstateinst,)) + + return sstate_installpkgdir(ss, d) + +def sstate_installpkgdir(ss, d): + import oe.path + import subprocess + + sstateinst = d.getVar("SSTATE_INSTDIR") + d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) + + for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split(): + # All hooks should run in the SSTATE_INSTDIR + bb.build.exec_func(f, d, (sstateinst,)) + + def prepdir(dir): + # remove dir if it exists, ensure any parent directories do exist + if os.path.exists(dir): + oe.path.remove(dir) + bb.utils.mkdirhier(dir) + oe.path.remove(dir) for state in ss['dirs']: + if d.getVar('SSTATE_SKIP_CREATION') == '1': + continue prepdir(state[1]) os.rename(sstateinst + state[0], state[1]) sstate_install(ss, d) for plain in ss['plaindirs']: - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') src = sstateinst + "/" + plain.replace(workdir, '') dest = plain bb.utils.mkdirhier(src) @@ -323,28 +369,40 @@ python sstate_hardcode_path_unpack () { # sstate_hardcode_path(d) import subprocess - sstateinst = d.getVar('SSTATE_INSTDIR', True) - fixmefn = sstateinst + "fixmepath" + sstateinst = d.getVar('SSTATE_INSTDIR') + sstatefixmedir = d.getVar('SSTATE_FIXMEDIR') + fixmefn = sstateinst + "fixmepath" if os.path.isfile(fixmefn): - staging = d.getVar('STAGING_DIR', True) - staging_target = d.getVar('STAGING_DIR_TARGET', True) - staging_host = d.getVar('STAGING_DIR_HOST', True) - - if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): - sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIR:%s:g'" % (staging) - elif bb.data.inherits_class('cross', d): - sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIR:%s:g'" % (staging_target, staging) - else: + staging_target = d.getVar('RECIPE_SYSROOT') + staging_host = d.getVar('RECIPE_SYSROOT_NATIVE') + + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d): sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host) + elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d): + sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host) + else: + sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target) - extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES', True) or '' + extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or '' for fixmevar in extra_staging_fixmes.split(): - fixme_path = d.getVar(fixmevar, True) + fixme_path = d.getVar(fixmevar) sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd) + # Defer do_populate_sysroot relocation command + if sstatefixmedir: + bb.utils.mkdirhier(sstatefixmedir) + with open(sstatefixmedir + "/fixmepath.cmd", "w") as f: + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath") + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST") + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST") + sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET") + f.write(sstate_hardcode_cmd) + bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath") + return + bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd)) subprocess.call(sstate_hardcode_cmd, shell=True) @@ -356,17 +414,17 @@ python sstate_hardcode_path_unpack () { def sstate_clean_cachefile(ss, d): import oe.path - sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['task'] + ".tgz*" + sstatepkgfile = d.getVar('SSTATE_PATHSPEC') + "*_" + ss['task'] + ".tgz*" bb.note("Removing %s" % sstatepkgfile) oe.path.remove(sstatepkgfile) def sstate_clean_cachefiles(d): - for task in (d.getVar('SSTATETASKS', True) or "").split(): + for task in (d.getVar('SSTATETASKS') or "").split(): ld = d.createCopy() ss = sstate_state_fromvars(ld, task) sstate_clean_cachefile(ss, ld) -def sstate_clean_manifest(manifest, d): +def sstate_clean_manifest(manifest, d, prefix=None): import oe.path mfile = open(manifest) @@ -375,6 +433,8 @@ def sstate_clean_manifest(manifest, d): for entry in entries: entry = entry.strip() + if prefix and not entry.startswith("/"): + entry = prefix + "/" + entry bb.debug(2, "Removing manifest: %s" % entry) # We can race against another package populating directories as we're removing them # so we ignore errors here. @@ -385,10 +445,17 @@ def sstate_clean_manifest(manifest, d): elif os.path.exists(entry) and len(os.listdir(entry)) == 0: os.rmdir(entry[:-1]) else: - oe.path.remove(entry) + os.remove(entry) except OSError: pass + postrm = manifest + ".postrm" + if os.path.exists(manifest + ".postrm"): + import subprocess + os.chmod(postrm, 0o755) + subprocess.call(postrm, shell=True) + oe.path.remove(postrm) + oe.path.remove(manifest) def sstate_clean(ss, d): @@ -396,8 +463,8 @@ def sstate_clean(ss, d): import glob d2 = d.createCopy() - stamp_clean = d.getVar("STAMPCLEAN", True) - extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info', True) + stamp_clean = d.getVar("STAMPCLEAN") + extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info') if extrainf: d2.setVar("SSTATE_MANMACH", extrainf) wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf) @@ -429,7 +496,7 @@ def sstate_clean(ss, d): rm_nohash = ".do_%s" % ss['task'] for stfile in glob.glob(wildcard_stfile): # Keep the sigdata - if ".sigdata." in stfile: + if ".sigdata." in stfile or ".sigbasedata." in stfile: continue # Preserve taint files in the stamps directory if stfile.endswith('.taint'): @@ -438,18 +505,22 @@ def sstate_clean(ss, d): stfile.endswith(rm_nohash): oe.path.remove(stfile) + # Removes the users/groups created by the package + for cleanfunc in (d.getVar('SSTATECLEANFUNCS') or '').split(): + bb.build.exec_func(cleanfunc, d) + sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX" CLEANFUNCS += "sstate_cleanall" python sstate_cleanall() { - bb.note("Removing shared state for package %s" % d.getVar('PN', True)) + bb.note("Removing shared state for package %s" % d.getVar('PN')) - manifest_dir = d.getVar('SSTATE_MANIFESTS', True) + manifest_dir = d.getVar('SSTATE_MANIFESTS') if not os.path.exists(manifest_dir): return - tasks = d.getVar('SSTATETASKS', True).split() + tasks = d.getVar('SSTATETASKS').split() for name in tasks: ld = d.createCopy() shared_state = sstate_state_fromvars(ld, name) @@ -465,29 +536,29 @@ python sstate_hardcode_path () { # Note: the logic in this function needs to match the reverse logic # in sstate_installpkg(ss, d) - staging = d.getVar('STAGING_DIR', True) - staging_target = d.getVar('STAGING_DIR_TARGET', True) - staging_host = d.getVar('STAGING_DIR_HOST', True) - sstate_builddir = d.getVar('SSTATE_BUILDDIR', True) + staging_target = d.getVar('RECIPE_SYSROOT') + staging_host = d.getVar('RECIPE_SYSROOT_NATIVE') + sstate_builddir = d.getVar('SSTATE_BUILDDIR') - if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): - sstate_grep_cmd = "grep -l -e '%s'" % (staging) - sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIR:g'" % (staging) - elif bb.data.inherits_class('cross', d): - sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging) - sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g; s:%s:FIXMESTAGINGDIR:g'" % (staging_target, staging) - else: + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d): sstate_grep_cmd = "grep -l -e '%s'" % (staging_host) sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % (staging_host) + elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d): + sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host) + sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g; s:%s:FIXMESTAGINGDIRHOST:g'" % (staging_target, staging_host) + else: + sstate_grep_cmd = "grep -l -e '%s'" % (staging_target) + sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % (staging_target) - extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES', True) or '' + extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or '' for fixmevar in extra_staging_fixmes.split(): - fixme_path = d.getVar(fixmevar, True) + fixme_path = d.getVar(fixmevar) sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar) + sstate_grep_cmd += " -e '%s'" % (fixme_path) fixmefn = sstate_builddir + "fixmepath" - sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD', True) + sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD') sstate_filelist_cmd = "tee %s" % (fixmefn) # fixmepath file needs relative paths, drop sstate_builddir prefix @@ -502,92 +573,81 @@ python sstate_hardcode_path () { sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd) bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd)) - subprocess.call(sstate_hardcode_cmd, shell=True) + subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir) # If the fixmefn is empty, remove it.. if os.stat(fixmefn).st_size == 0: os.remove(fixmefn) else: bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd)) - subprocess.call(sstate_filelist_relative_cmd, shell=True) + subprocess.check_output(sstate_filelist_relative_cmd, shell=True) } def sstate_package(ss, d): import oe.path - def make_relative_symlink(path, outputpath, d): - # Replace out absolute TMPDIR paths in symlinks with relative ones - if not os.path.islink(path): - return - link = os.readlink(path) - if not os.path.isabs(link): - return - if not link.startswith(tmpdir): - return - - depth = outputpath.rpartition(tmpdir)[2].count('/') - base = link.partition(tmpdir)[2].strip() - while depth > 1: - base = "/.." + base - depth -= 1 - base = "." + base - - bb.debug(2, "Replacing absolute path %s with relative path %s for %s" % (link, base, outputpath)) - os.remove(path) - os.symlink(base, path) - - tmpdir = d.getVar('TMPDIR', True) + tmpdir = d.getVar('TMPDIR') sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task']) - sstatepkg = d.getVar('SSTATE_PKG', True) + '_'+ ss['task'] + ".tgz" + sstatepkg = d.getVar('SSTATE_PKG') + '_'+ ss['task'] + ".tgz" bb.utils.remove(sstatebuild, recurse=True) bb.utils.mkdirhier(sstatebuild) bb.utils.mkdirhier(os.path.dirname(sstatepkg)) for state in ss['dirs']: if not os.path.exists(state[1]): continue + if d.getVar('SSTATE_SKIP_CREATION') == '1': + continue srcbase = state[0].rstrip("/").rsplit('/', 1)[0] + # Find and error for absolute symlinks. We could attempt to relocate but its not + # clear where the symlink is relative to in this context. We could add that markup + # to sstate tasks but there aren't many of these so better just avoid them entirely. for walkroot, dirs, files in os.walk(state[1]): - for file in files: + for file in files + dirs: srcpath = os.path.join(walkroot, file) - dstpath = srcpath.replace(state[1], state[2]) - make_relative_symlink(srcpath, dstpath, d) - for dir in dirs: - srcpath = os.path.join(walkroot, dir) - dstpath = srcpath.replace(state[1], state[2]) - make_relative_symlink(srcpath, dstpath, d) + if not os.path.islink(srcpath): + continue + link = os.readlink(srcpath) + if not os.path.isabs(link): + continue + if not link.startswith(tmpdir): + continue + bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link)) bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0])) - oe.path.copyhardlinktree(state[1], sstatebuild + state[0]) + os.rename(state[1], sstatebuild + state[0]) - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') for plain in ss['plaindirs']: pdir = plain.replace(workdir, sstatebuild) bb.utils.mkdirhier(plain) bb.utils.mkdirhier(pdir) - oe.path.copyhardlinktree(plain, pdir) + os.rename(plain, pdir) d.setVar('SSTATE_BUILDDIR', sstatebuild) d.setVar('SSTATE_PKG', sstatepkg) - for f in (d.getVar('SSTATECREATEFUNCS', True) or '').split() + ['sstate_create_package'] + \ - (d.getVar('SSTATEPOSTCREATEFUNCS', True) or '').split(): - bb.build.exec_func(f, d) - + for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \ + ['sstate_create_package', 'sstate_sign_package'] + \ + (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split(): + # All hooks should run in SSTATE_BUILDDIR. + bb.build.exec_func(f, d, (sstatebuild,)) + bb.siggen.dump_this_task(sstatepkg + ".siginfo", d) + d.setVar('SSTATE_INSTDIR', sstatebuild) + return def pstaging_fetch(sstatefetch, sstatepkg, d): import bb.fetch2 # Only try and fetch if the user has configured a mirror - mirrors = d.getVar('SSTATE_MIRRORS', True) + mirrors = d.getVar('SSTATE_MIRRORS') if not mirrors: return # Copy the data object and override DL_DIR and SRC_URI localdata = bb.data.createCopy(d) - bb.data.update_data(localdata) dldir = localdata.expand("${SSTATE_DIR}") bb.utils.mkdirhier(dldir) @@ -599,24 +659,22 @@ def pstaging_fetch(sstatefetch, sstatepkg, d): # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, # we'll want to allow network access for the current set of fetches. - if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": + if localdata.getVar('BB_NO_NETWORK') == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK') == "1": localdata.delVar('BB_NO_NETWORK') # Try a fetch from the sstate mirror, if it fails just return and # we will build the package - for srcuri in ['file://{0}'.format(sstatefetch), - 'file://{0}.siginfo'.format(sstatefetch)]: + uris = ['file://{0};downloadfilename={0}'.format(sstatefetch), + 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)] + if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False): + uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)] + + for srcuri in uris: localdata.setVar('SRC_URI', srcuri) try: fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) fetcher.download() - # Need to optimise this, if using file:// urls, the fetcher just changes the local path - # For now work around by symlinking - localpath = bb.data.expand(fetcher.localpath(srcuri), localdata) - if localpath != sstatepkg and os.path.exists(localpath) and not os.path.exists(sstatepkg): - os.symlink(localpath, sstatepkg) - except bb.fetch2.BBFetchException: break @@ -624,25 +682,35 @@ def sstate_setscene(d): shared_state = sstate_state_fromvars(d) accelerate = sstate_installpkg(shared_state, d) if not accelerate: - raise bb.build.FuncFailed("No suitable staging package found") + bb.fatal("No suitable staging package found") python sstate_task_prefunc () { shared_state = sstate_state_fromvars(d) sstate_clean(shared_state, d) } +sstate_task_prefunc[dirs] = "${WORKDIR}" python sstate_task_postfunc () { shared_state = sstate_state_fromvars(d) - sstate_install(shared_state, d) + for intercept in shared_state['interceptfuncs']: - bb.build.exec_func(intercept, d) - omask = os.umask(002) - if omask != 002: - bb.note("Using umask 002 (not %0o) for sstate packaging" % omask) + bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),)) + + omask = os.umask(0o002) + if omask != 0o002: + bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask) sstate_package(shared_state, d) os.umask(omask) + + sstateinst = d.getVar("SSTATE_INSTDIR") + d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir']) + + sstate_installpkgdir(shared_state, d) + + bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True) } - +sstate_task_postfunc[dirs] = "${WORKDIR}" + # # Shell function to generate a sstate package from a directory @@ -662,11 +730,20 @@ sstate_create_package () { else tar -cz --file=$TFILE --files-from=/dev/null fi - chmod 0664 $TFILE + chmod 0664 $TFILE mv -f $TFILE ${SSTATE_PKG} +} - cd ${WORKDIR} - rm -rf ${SSTATE_BUILDDIR} +python sstate_sign_package () { + from oe.gpg_sign import get_signer + + if d.getVar('SSTATE_SIG_KEY'): + signer = get_signer(d, 'local') + sstate_pkg = d.getVar('SSTATE_PKG') + if os.path.exists(sstate_pkg + '.sig'): + os.unlink(sstate_pkg + '.sig') + signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None, + d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) } # @@ -674,9 +751,13 @@ sstate_create_package () { # Will be run from within SSTATE_INSTDIR. # sstate_unpack_package () { - tar -xmvzf ${SSTATE_PKG} + tar -xvzf ${SSTATE_PKG} + # update .siginfo atime on local/NFS mirror + [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo # Use "! -w ||" to return true for read only files [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG} + [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig + [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo } BB_HASHCHECK_FUNCTION = "sstate_checkhashes" @@ -693,7 +774,10 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): # Magic data from BB_HASHFILENAME splithashfn = sq_hashfn[task].split(" ") spec = splithashfn[1] - extrapath = splithashfn[0] + if splithashfn[0] == "True": + extrapath = d.getVar("NATIVELSBSTRING") + "/" + else: + extrapath = "" tname = sq_task[task][3:] @@ -718,11 +802,10 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): missed.append(task) bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) - mirrors = d.getVar("SSTATE_MIRRORS", True) + mirrors = d.getVar("SSTATE_MIRRORS") if mirrors: # Copy the data object and override DL_DIR and SRC_URI localdata = bb.data.createCopy(d) - bb.data.update_data(localdata) dldir = localdata.expand("${SSTATE_DIR}") localdata.delVar('MIRRORS') @@ -734,7 +817,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, # we'll want to allow network access for the current set of fetches. - if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": + if localdata.getVar('BB_NO_NETWORK') == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK') == "1": localdata.delVar('BB_NO_NETWORK') from bb.fetch2 import FetchConnectionCache @@ -763,7 +846,8 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): except: missed.append(task) bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri) - pass + pass + bb.event.fire(bb.event.ProcessProgress("Checking sstate mirror object availability", len(tasklist) - thread_worker.tasks.qsize()), d) tasklist = [] for task in range(len(sq_fn)): @@ -774,18 +858,23 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): tasklist.append((task, sstatefile)) if tasklist: - bb.note("Checking sstate mirror object availability (for %s objects)" % len(tasklist)) + bb.event.fire(bb.event.ProcessStarted("Checking sstate mirror object availability", len(tasklist)), d) + import multiprocessing nproc = min(multiprocessing.cpu_count(), len(tasklist)) + bb.event.enable_threadlock() pool = oe.utils.ThreadedPool(nproc, len(tasklist), worker_init=checkstatus_init, worker_end=checkstatus_end) for t in tasklist: pool.add_task(checkstatus, t) pool.start() pool.wait_completion() + bb.event.disable_threadlock() + + bb.event.fire(bb.event.ProcessFinished("Checking sstate mirror object availability"), d) - inheritlist = d.getVar("INHERIT", True) + inheritlist = d.getVar("INHERIT") if "toaster" in inheritlist: evdata = {'missed': [], 'found': []}; for task in missed: @@ -805,26 +894,41 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): BB_SETSCENE_DEPVALID = "setscene_depvalid" -def setscene_depvalid(task, taskdependees, notneeded, d): +def setscene_depvalid(task, taskdependees, notneeded, d, log=None): # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME] # task is included in taskdependees too + # Return - False - We need this dependency + # - True - We can skip this dependency + + def logit(msg, log): + if log is not None: + log.append(msg) + else: + bb.debug(2, msg) - bb.debug(2, "Considering setscene task: %s" % (str(taskdependees[task]))) + logit("Considering setscene task: %s" % (str(taskdependees[task])), log) def isNativeCross(x): - return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x - - def isPostInstDep(x): - if x in ["qemu-native", "gdk-pixbuf-native", "qemuwrapper-cross", "depmodwrapper-cross", "systemd-systemctl-native", "gtk-icon-utils-native"]: - return True - return False + return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross") # We only need to trigger populate_lic through direct dependencies if taskdependees[task][1] == "do_populate_lic": return True + # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects + if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir": + return True + + # We only need to trigger packagedata through direct dependencies + # but need to preserve packagedata on packagedata links + if taskdependees[task][1] == "do_packagedata": + for dep in taskdependees: + if taskdependees[dep][1] == "do_packagedata": + return False + return True + for dep in taskdependees: - bb.debug(2, " considering dependency: %s" % (str(taskdependees[dep]))) + logit(" considering dependency: %s" % (str(taskdependees[dep])), log) if task == dep: continue if dep in notneeded: @@ -832,15 +936,21 @@ def setscene_depvalid(task, taskdependees, notneeded, d): # do_package_write_* and do_package doesn't need do_package if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']: continue - # do_package_write_* and do_package doesn't need do_populate_sysroot, unless is a postinstall dependency - if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']: - if isPostInstDep(taskdependees[task][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']: - return False + # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies + if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']: + return False + # do_package/packagedata/package_qa don't need do_populate_sysroot + if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']: continue # Native/Cross packages don't exist and are noexec anyway if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']: continue + # This is due to the [depends] in useradd.bbclass complicating matters + # The logic *is* reversed here due to the way hard setscene dependencies are injected + if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot': + continue + # Consider sysroot depending on sysroot tasks if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot': # base-passwd/shadow-sysroot don't need their dependencies @@ -849,6 +959,9 @@ def setscene_depvalid(task, taskdependees, notneeded, d): # Nothing need depend on libc-initial/gcc-cross-initial if "-initial" in taskdependees[task][0]: continue + # For meta-extsdk-toolchain we want all sysroot dependencies + if taskdependees[dep][0] == 'meta-extsdk-toolchain': + return False # Native/Cross populate_sysroot need their dependencies if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]): return False @@ -864,13 +977,12 @@ def setscene_depvalid(task, taskdependees, notneeded, d): if taskdependees[task][1] == 'do_shared_workdir': continue - # This is due to the [depends] in useradd.bbclass complicating matters - # The logic *is* reversed here due to the way hard setscene dependencies are injected - if taskdependees[task][1] == 'do_package' and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot': + if taskdependees[dep][1] == "do_populate_lic": continue + # Safe fallthrough default - bb.debug(2, " Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep]))) + logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log) return False return True @@ -879,15 +991,15 @@ sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded" python sstate_eventhandler() { d = e.data # When we write an sstate package we rewrite the SSTATE_PKG - spkg = d.getVar('SSTATE_PKG', True) + spkg = d.getVar('SSTATE_PKG') if not spkg.endswith(".tgz"): - taskname = d.getVar("BB_RUNTASK", True)[3:] - spec = d.getVar('SSTATE_PKGSPEC', True) - swspec = d.getVar('SSTATE_SWSPEC', True) + taskname = d.getVar("BB_RUNTASK")[3:] + spec = d.getVar('SSTATE_PKGSPEC') + swspec = d.getVar('SSTATE_SWSPEC') if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec: d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") d.setVar("SSTATE_EXTRAPATH", "") - sstatepkg = d.getVar('SSTATE_PKG', True) + sstatepkg = d.getVar('SSTATE_PKG') bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d) } @@ -906,7 +1018,7 @@ python sstate_eventhandler2() { stamps = e.stamps.values() removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1") seen = [] - for a in d.getVar("SSTATE_ARCHS", True).split(): + for a in d.getVar("SSTATE_ARCHS").split(): toremove = [] i = d.expand("${SSTATE_MANIFESTS}/index-" + a) if not os.path.exists(i): @@ -918,11 +1030,17 @@ python sstate_eventhandler2() { if stamp not in stamps: toremove.append(l) if stamp not in seen: - bb.note("Stamp %s is not reachable, removing related manifests" % stamp) + bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp) seen.append(stamp) + + if toremove: + bb.note("There are %d recipes to be removed from sysroot %s, removing..." % (len(toremove), a)) + for r in toremove: (stamp, manifest, workdir) = r.split() for m in glob.glob(manifest + ".*"): + if m.endswith(".postrm"): + continue sstate_clean_manifest(m, d) bb.utils.remove(stamp + "*") if removeworkdir: diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass index 967eddd605..4015dd754c 100644 --- a/meta/classes/staging.bbclass +++ b/meta/classes/staging.bbclass @@ -1,3 +1,38 @@ +# These directories will be staged in the sysroot +SYSROOT_DIRS = " \ + ${includedir} \ + ${libdir} \ + ${base_libdir} \ + ${nonarch_base_libdir} \ + ${datadir} \ +" + +# These directories are also staged in the sysroot when they contain files that +# are usable on the build system +SYSROOT_DIRS_NATIVE = " \ + ${bindir} \ + ${sbindir} \ + ${base_bindir} \ + ${base_sbindir} \ + ${libexecdir} \ + ${sysconfdir} \ + ${localstatedir} \ +" +SYSROOT_DIRS_append_class-native = " ${SYSROOT_DIRS_NATIVE}" +SYSROOT_DIRS_append_class-cross = " ${SYSROOT_DIRS_NATIVE}" +SYSROOT_DIRS_append_class-crosssdk = " ${SYSROOT_DIRS_NATIVE}" + +# These directories will not be staged in the sysroot +SYSROOT_DIRS_BLACKLIST = " \ + ${mandir} \ + ${docdir} \ + ${infodir} \ + ${datadir}/locale \ + ${datadir}/applications \ + ${datadir}/fonts \ + ${datadir}/pixmaps \ + ${libdir}/${PN}/ptest \ +" sysroot_stage_dir() { src="$1" @@ -14,43 +49,18 @@ sysroot_stage_dir() { ) } -sysroot_stage_libdir() { - src="$1" - dest="$2" - - sysroot_stage_dir $src $dest -} - sysroot_stage_dirs() { from="$1" to="$2" - sysroot_stage_dir $from${includedir} $to${includedir} - if [ "${BUILD_SYS}" = "${HOST_SYS}" ]; then - sysroot_stage_dir $from${bindir} $to${bindir} - sysroot_stage_dir $from${sbindir} $to${sbindir} - sysroot_stage_dir $from${base_bindir} $to${base_bindir} - sysroot_stage_dir $from${base_sbindir} $to${base_sbindir} - sysroot_stage_dir $from${libexecdir} $to${libexecdir} - sysroot_stage_dir $from${sysconfdir} $to${sysconfdir} - sysroot_stage_dir $from${localstatedir} $to${localstatedir} - fi - if [ -d $from${libdir} ] - then - sysroot_stage_libdir $from${libdir} $to${libdir} - fi - if [ -d $from${base_libdir} ] - then - sysroot_stage_libdir $from${base_libdir} $to${base_libdir} - fi - if [ -d $from${nonarch_base_libdir} ] - then - sysroot_stage_libdir $from${nonarch_base_libdir} $to${nonarch_base_libdir} - fi - sysroot_stage_dir $from${datadir} $to${datadir} - # We don't care about docs/info/manpages/locales - rm -rf $to${mandir}/ $to${docdir}/ $to${infodir}/ ${to}${datadir}/locale/ - rm -rf $to${datadir}/applications/ $to${datadir}/fonts/ $to${datadir}/pixmaps/ + for dir in ${SYSROOT_DIRS}; do + sysroot_stage_dir "$from$dir" "$to$dir" + done + + # Remove directories we do not care about + for dir in ${SYSROOT_DIRS_BLACKLIST}; do + rm -rf "$to$dir" + done } sysroot_stage_all() { @@ -60,8 +70,8 @@ sysroot_stage_all() { python sysroot_strip () { import stat, errno - dvar = d.getVar('SYSROOT_DESTDIR', True) - pn = d.getVar('PN', True) + dvar = d.getVar('SYSROOT_DESTDIR') + pn = d.getVar('PN') os.chdir(dvar) @@ -94,9 +104,9 @@ python sysroot_strip () { elffiles = {} inodes = {} - libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True)) - baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True)) - if (d.getVar('INHIBIT_SYSROOT_STRIP', True) != '1'): + libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir")) + baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir")) + if (d.getVar('INHIBIT_SYSROOT_STRIP') != '1'): # # First lets figure out all of the files we may have to process # @@ -127,7 +137,10 @@ python sysroot_strip () { elf_file = isELF(file) if elf_file & 1: if elf_file & 2: - bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)) + if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split(): + bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) + else: + bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)) continue if s.st_ino in inodes: @@ -142,7 +155,7 @@ python sysroot_strip () { # # Now strip them (in parallel) # - strip = d.getVar("STRIP", True) + strip = d.getVar("STRIP") sfiles = [] for file in elffiles: elf_file = int(elffiles[file]) @@ -158,42 +171,18 @@ do_populate_sysroot[umask] = "022" addtask populate_sysroot after do_install SYSROOT_PREPROCESS_FUNCS ?= "" -SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir/" -SYSROOT_LOCK = "${STAGING_DIR}/staging.lock" - -# We clean out any existing sstate from the sysroot if we rerun configure -python sysroot_cleansstate () { - ss = sstate_state_fromvars(d, "populate_sysroot") - sstate_clean(ss, d) -} -do_configure[prefuncs] += "sysroot_cleansstate" - - -BB_SETSCENE_VERIFY_FUNCTION = "sysroot_checkhashes" - -def sysroot_checkhashes(covered, tasknames, fnids, fns, d, invalidtasks = None): - problems = set() - configurefnids = set() - if not invalidtasks: - invalidtasks = xrange(len(tasknames)) - for task in invalidtasks: - if tasknames[task] == "do_configure" and task not in covered: - configurefnids.add(fnids[task]) - for task in covered: - if tasknames[task] == "do_populate_sysroot" and fnids[task] in configurefnids: - problems.add(task) - return problems +SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir" python do_populate_sysroot () { bb.build.exec_func("sysroot_stage_all", d) bb.build.exec_func("sysroot_strip", d) - for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS', True) or '').split(): + for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS') or '').split(): bb.build.exec_func(f, d) - pn = d.getVar("PN", True) - multiprov = d.getVar("MULTI_PROVIDER_WHITELIST", True).split() + pn = d.getVar("PN") + multiprov = d.getVar("MULTI_PROVIDER_WHITELIST").split() provdir = d.expand("${SYSROOT_DESTDIR}${base_prefix}/sysroot-providers/") bb.utils.mkdirhier(provdir) - for p in d.getVar("PROVIDES", True).split(): + for p in d.getVar("PROVIDES").split(): if p in multiprov: continue p = p.replace("/", "_") @@ -204,15 +193,466 @@ python do_populate_sysroot () { do_populate_sysroot[vardeps] += "${SYSROOT_PREPROCESS_FUNCS}" do_populate_sysroot[vardepsexclude] += "MULTI_PROVIDER_WHITELIST" +POPULATESYSROOTDEPS = "" +POPULATESYSROOTDEPS_class-target = "virtual/${MLPREFIX}${TARGET_PREFIX}binutils:do_populate_sysroot" +POPULATESYSROOTDEPS_class-nativesdk = "virtual/${TARGET_PREFIX}binutils-crosssdk:do_populate_sysroot" +do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}" + SSTATETASKS += "do_populate_sysroot" do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}" do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}" -do_populate_sysroot[sstate-outputdirs] = "${STAGING_DIR_HOST}/" -do_populate_sysroot[stamp-extra-info] = "${MACHINE}" +do_populate_sysroot[sstate-outputdirs] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/${PN}" +do_populate_sysroot[sstate-fixmedir] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/${PN}" python do_populate_sysroot_setscene () { sstate_setscene(d) } addtask do_populate_sysroot_setscene +def staging_copyfile(c, target, dest, postinsts, seendirs): + import errno + + destdir = os.path.dirname(dest) + if destdir not in seendirs: + bb.utils.mkdirhier(destdir) + seendirs.add(destdir) + if "/usr/bin/postinst-" in c: + postinsts.append(dest) + if os.path.islink(c): + linkto = os.readlink(c) + if os.path.lexists(dest): + if not os.path.islink(dest): + raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest) + if os.readlink(dest) == linkto: + return dest + raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest) + os.symlink(linkto, dest) + #bb.warn(c) + else: + try: + os.link(c, dest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(c, dest) + else: + raise + return dest + +def staging_copydir(c, target, dest, seendirs): + if dest not in seendirs: + bb.utils.mkdirhier(dest) + seendirs.add(dest) + +def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d): + import subprocess + + if not fixme: + return + cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative) + for fixmevar in ['HOSTTOOLS_DIR', 'PKGDATA_DIR']: + fixme_path = d.getVar(fixmevar) + cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) + bb.note(cmd) + subprocess.check_output(cmd, shell=True) + + +def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d): + import glob + import subprocess + + fixme = [] + postinsts = [] + seendirs = set() + stagingdir = d.getVar("STAGING_DIR") + if native: + pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*'] + targetdir = nativesysroot + else: + pkgarchs = ['${MACHINE_ARCH}'] + pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split())) + pkgarchs.append('allarch') + targetdir = targetsysroot + + bb.utils.mkdirhier(targetdir) + for pkgarch in pkgarchs: + for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)): + if manifest.endswith("-initial.populate_sysroot"): + # skip glibc-initial and libgcc-initial due to file overlap + continue + tmanifest = targetdir + "/" + os.path.basename(manifest) + if os.path.exists(tmanifest): + continue + try: + os.link(manifest, tmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(manifest, tmanifest) + else: + raise + with open(manifest, "r") as f: + for l in f: + l = l.strip() + if l.endswith("/fixmepath"): + fixme.append(l) + continue + if l.endswith("/fixmepath.cmd"): + continue + dest = l.replace(stagingdir, "") + dest = targetdir + "/" + "/".join(dest.split("/")[3:]) + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + try: + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + except FileExistsError: + continue + + staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d) + for p in postinsts: + subprocess.check_output(p, shell=True) + +# +# Manifests here are complicated. The main sysroot area has the unpacked sstate +# which us unrelocated and tracked by the main sstate manifests. Each recipe +# specific sysroot has manifests for each dependency that is installed there. +# The task hash is used to tell whether the data needs to be reinstalled. We +# use a symlink to point to the currently installed hash. There is also a +# "complete" stamp file which is used to mark if installation completed. If +# something fails (e.g. a postinst), this won't get written and we would +# remove and reinstall the dependency. This also means partially installed +# dependencies should get cleaned up correctly. +# + +python extend_recipe_sysroot() { + import copy + import subprocess + import errno + import collections + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + mytaskname = d.getVar("BB_RUNTASK") + workdir = d.getVar("WORKDIR") + #bb.warn(str(taskdepdata)) + pn = d.getVar("PN") + + if mytaskname.endswith("_setscene"): + mytaskname = mytaskname.replace("_setscene", "") + + start = None + configuredeps = [] + for dep in taskdepdata: + data = taskdepdata[dep] + if data[1] == mytaskname and data[0] == pn: + start = dep + break + if start is None: + bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") + + # We need to figure out which sysroot files we need to expose to this task. + # This needs to match what would get restored from sstate, which is controlled + # ultimately by calls from bitbake to setscene_depvalid(). + # That function expects a setscene dependency tree. We build a dependency tree + # condensed to inter-sstate task dependencies, similar to that used by setscene + # tasks. We can then call into setscene_depvalid() and decide + # which dependencies we can "see" and should expose in the recipe specific sysroot. + setscenedeps = copy.deepcopy(taskdepdata) + + start = set([start]) + + sstatetasks = d.getVar("SSTATETASKS").split() + + def print_dep_tree(deptree): + data = "" + for dep in deptree: + deps = " " + "\n ".join(deptree[dep][3]) + "\n" + data = "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5]) + return data + + #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata)) + + #bb.note(" start2 is %s" % str(start)) + + # If start is an sstate task (like do_package) we need to add in its direct dependencies + # else the code below won't recurse into them. + for dep in set(start): + for dep2 in setscenedeps[dep][3]: + start.add(dep2) + start.remove(dep) + + #bb.note(" start3 is %s" % str(start)) + + # Create collapsed do_populate_sysroot -> do_populate_sysroot tree + for dep in taskdepdata: + data = setscenedeps[dep] + if data[1] not in sstatetasks: + for dep2 in setscenedeps: + data2 = setscenedeps[dep2] + if dep in data2[3]: + data2[3].update(setscenedeps[dep][3]) + data2[3].remove(dep) + if dep in start: + start.update(setscenedeps[dep][3]) + start.remove(dep) + del setscenedeps[dep] + + # Remove circular references + for dep in setscenedeps: + if dep in setscenedeps[dep][3]: + setscenedeps[dep][3].remove(dep) + + #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps)) + #bb.note(" start is %s" % str(start)) + + # Direct dependencies should be present and can be depended upon + for dep in set(start): + if setscenedeps[dep][1] == "do_populate_sysroot": + if dep not in configuredeps: + configuredeps.append(dep) + bb.note("Direct dependencies are %s" % str(configuredeps)) + #bb.note(" or %s" % str(start)) + + msgbuf = [] + # Call into setscene_depvalid for each sub-dependency and only copy sysroot files + # for ones that would be restored from sstate. + done = list(start) + next = list(start) + while next: + new = [] + for dep in next: + data = setscenedeps[dep] + for datadep in data[3]: + if datadep in done: + continue + taskdeps = {} + taskdeps[dep] = setscenedeps[dep][:2] + taskdeps[datadep] = setscenedeps[datadep][:2] + retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf) + if retval: + msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep) + continue + done.append(datadep) + new.append(datadep) + if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot": + configuredeps.append(datadep) + msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0]) + else: + msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0]) + next = new + + bb.note("\n".join(msgbuf)) + + stagingdir = d.getVar("STAGING_DIR") + sharedmanifests = stagingdir + "-components/manifests" + recipesysroot = d.getVar("RECIPE_SYSROOT") + recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE") + current_variant = d.getVar("BBEXTENDVARIANT") + + # Detect bitbake -b usage + nodeps = d.getVar("BB_LIMITEDDEPS") or False + if nodeps: + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d) + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d) + bb.utils.unlockfile(lock) + + depdir = recipesysrootnative + "/installeddeps" + bb.utils.mkdirhier(depdir) + bb.utils.mkdirhier(sharedmanifests) + + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + + fixme = {} + fixme[''] = [] + fixme['native'] = [] + seendirs = set() + postinsts = [] + multilibs = {} + manifests = {} + + installed = [] + + for f in os.listdir(depdir): + if not f.endswith(".complete"): + continue + f = depdir + "/" + f + if os.path.islink(f) and not os.path.exists(f): + bb.note("%s no longer exists, removing from sysroot" % f) + lnk = os.readlink(f.replace(".complete", "")) + sstate_clean_manifest(depdir + "/" + lnk, d, workdir) + os.unlink(f) + os.unlink(f.replace(".complete", "")) + + for dep in configuredeps: + c = setscenedeps[dep][0] + taskhash = setscenedeps[dep][5] + taskmanifest = depdir + "/" + c + "." + taskhash + if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"): + bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c) + continue + + installed.append(c) + + if os.path.exists(depdir + "/" + c): + lnk = os.readlink(depdir + "/" + c) + if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"): + bb.note("%s exists in sysroot, skipping" % c) + continue + else: + bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash)) + sstate_clean_manifest(depdir + "/" + lnk, d, workdir) + os.unlink(depdir + "/" + c) + if os.path.lexists(depdir + "/" + c + ".complete"): + os.unlink(depdir + "/" + c + ".complete") + elif os.path.lexists(depdir + "/" + c): + os.unlink(depdir + "/" + c) + + os.symlink(c + "." + taskhash, depdir + "/" + c) + + d2 = d + destsysroot = recipesysroot + variant = '' + if setscenedeps[dep][2].startswith("virtual:multilib"): + variant = setscenedeps[dep][2].split(":")[2] + if variant != current_variant: + if variant not in multilibs: + multilibs[variant] = get_multilib_datastore(variant, d) + d2 = multilibs[variant] + destsysroot = d2.getVar("RECIPE_SYSROOT") + + native = False + if c.endswith("-native"): + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}-%s.populate_sysroot" % c) + native = True + elif c.startswith("nativesdk-"): + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${SDK_ARCH}_${SDK_OS}-%s.populate_sysroot" % c) + elif "-cross-" in c: + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}_${TARGET_ARCH}-%s.populate_sysroot" % c) + native = True + elif "-crosssdk" in c: + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}-%s.populate_sysroot" % c) + native = True + else: + pkgarchs = ['${MACHINE_ARCH}'] + pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split())) + pkgarchs.append('allarch') + for pkgarch in pkgarchs: + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.populate_sysroot" % (pkgarch, c)) + if os.path.exists(manifest): + break + if not os.path.exists(manifest): + bb.warn("Manifest %s not found?" % manifest) + else: + newmanifest = collections.OrderedDict() + if native: + fm = fixme['native'] + targetdir = recipesysrootnative + else: + fm = fixme[''] + targetdir = destsysroot + with open(manifest, "r") as f: + manifests[dep] = manifest + for l in f: + l = l.strip() + if l.endswith("/fixmepath"): + fm.append(l) + continue + if l.endswith("/fixmepath.cmd"): + continue + dest = l.replace(stagingdir, "") + dest = targetdir + "/" + "/".join(dest.split("/")[3:]) + newmanifest[l] = dest + # Having multiple identical manifests in each sysroot eats diskspace so + # create a shared pool of them and hardlink if we can. + # We create the manifest in advance so that if something fails during installation, + # or the build is interrupted, subsequent exeuction can cleanup. + sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest) + if not os.path.exists(sharedm): + smlock = bb.utils.lockfile(sharedm + ".lock") + # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other + # but python can lose file handles so we need to do this under a lock. + if not os.path.exists(sharedm): + with open(sharedm, 'w') as m: + for l in newmanifest: + dest = newmanifest[l] + m.write(dest.replace(workdir + "/", "") + "\n") + bb.utils.unlockfile(smlock) + try: + os.link(sharedm, taskmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(sharedm, taskmanifest) + else: + raise + # Finally actually install the files + for l in newmanifest: + dest = newmanifest[l] + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + + for f in fixme: + if f == '': + staging_processfixme(fixme[f], recipesysroot, recipesysroot, recipesysrootnative, d) + elif f == 'native': + staging_processfixme(fixme[f], recipesysrootnative, recipesysroot, recipesysrootnative, d) + else: + staging_processfixme(fixme[f], multilibs[f].getVar("RECIPE_SYSROOT"), recipesysroot, recipesysrootnative, d) + + for p in postinsts: + subprocess.check_output(p, shell=True) + + for dep in manifests: + c = setscenedeps[dep][0] + os.symlink(manifests[dep], depdir + "/" + c + ".complete") + + # We want to remove anything which this task previously installed but is no longer a dependency + # This could potentially race against another task which also installed it but still requires it + # but the alternative is not doing anything at all and that race window should be small enough + # to be insignificant + taskindex = depdir + "/" + "index." + mytaskname + if os.path.exists(taskindex): + with open(taskindex, "r") as f: + for l in f: + l = l.strip() + if l not in installed: + l = depdir + "/" + l + if not os.path.exists(l): + # Was likely already uninstalled + continue + bb.note("Task %s no longer depends on %s, removing from sysroot" % (mytaskname, l)) + lnk = os.readlink(l) + sstate_clean_manifest(depdir + "/" + lnk, d, workdir) + os.unlink(l) + os.unlink(l + ".complete") + with open(taskindex, "w") as f: + for l in sorted(installed): + f.write(l + "\n") + + bb.utils.unlockfile(lock) +} +extend_recipe_sysroot[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA" + +python do_prepare_recipe_sysroot () { + bb.build.exec_func("extend_recipe_sysroot", d) +} +addtask do_prepare_recipe_sysroot before do_configure after do_fetch + +# Clean out the recipe specific sysroots before do_fetch +# (use a prefunc so we can order before extend_recipe_sysroot if it gets added) +python clean_recipe_sysroot() { + return +} +clean_recipe_sysroot[cleandirs] += "${RECIPE_SYSROOT} ${RECIPE_SYSROOT_NATIVE}" +do_fetch[prefuncs] += "clean_recipe_sysroot" + +python staging_taskhandler() { + bbtasks = e.tasklist + for task in bbtasks: + deps = d.getVarFlag(task, "depends") + if deps and "populate_sysroot" in deps: + d.appendVarFlag(task, "prefuncs", " extend_recipe_sysroot") +} +staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess" +addhandler staging_taskhandler diff --git a/meta/classes/syslinux.bbclass b/meta/classes/syslinux.bbclass index d6498d98bb..d6f882420e 100644 --- a/meta/classes/syslinux.bbclass +++ b/meta/classes/syslinux.bbclass @@ -20,19 +20,24 @@ do_bootimg[depends] += "${MLPREFIX}syslinux:do_populate_sysroot \ syslinux-native:do_populate_sysroot" -SYSLINUXCFG = "${S}/syslinux.cfg" - -ISOLINUXDIR = "/isolinux" +ISOLINUXDIR ?= "/isolinux" SYSLINUXDIR = "/" # The kernel has an internal default console, which you can override with # a console=...some_tty... SYSLINUX_DEFAULT_CONSOLE ?= "" SYSLINUX_SERIAL ?= "0 115200" SYSLINUX_SERIAL_TTY ?= "console=ttyS0,115200" -ISO_BOOTIMG = "isolinux/isolinux.bin" -ISO_BOOTCAT = "isolinux/boot.cat" -MKISOFS_OPTIONS = "-no-emul-boot -boot-load-size 4 -boot-info-table" -APPEND_prepend = " ${SYSLINUX_ROOT} " +SYSLINUX_PROMPT ?= "0" +SYSLINUX_TIMEOUT ?= "50" +AUTO_SYSLINUXMENU ?= "1" +SYSLINUX_ALLOWOPTIONS ?= "1" +SYSLINUX_ROOT ?= "${ROOT}" +SYSLINUX_CFG_VM ?= "${S}/syslinux_vm.cfg" +SYSLINUX_CFG_LIVE ?= "${S}/syslinux_live.cfg" +APPEND ?= "" + +# Need UUID utility code. +inherit fs-uuid syslinux_populate() { DEST=$1 @@ -42,7 +47,7 @@ syslinux_populate() { install -d ${DEST}${BOOTDIR} # Install the config files - install -m 0644 ${SYSLINUXCFG} ${DEST}${BOOTDIR}/${CFGNAME} + install -m 0644 ${SYSLINUX_CFG} ${DEST}${BOOTDIR}/${CFGNAME} if [ "${AUTO_SYSLINUXMENU}" = 1 ] ; then install -m 0644 ${STAGING_DATADIR}/syslinux/vesamenu.c32 ${DEST}${BOOTDIR}/vesamenu.c32 install -m 0444 ${STAGING_DATADIR}/syslinux/libcom32.c32 ${DEST}${BOOTDIR}/libcom32.c32 @@ -67,7 +72,7 @@ syslinux_hddimg_populate() { } syslinux_hddimg_install() { - syslinux ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.hddimg + syslinux ${IMGDEPLOYDIR}/${IMAGE_NAME}.hddimg } syslinux_hdddirect_install() { @@ -79,12 +84,12 @@ python build_syslinux_cfg () { import copy import sys - workdir = d.getVar('WORKDIR', True) + workdir = d.getVar('WORKDIR') if not workdir: bb.error("WORKDIR not defined, unable to package") return - labels = d.getVar('LABELS', True) + labels = d.getVar('LABELS') if not labels: bb.debug(1, "LABELS not defined, nothing to do") return @@ -93,45 +98,50 @@ python build_syslinux_cfg () { bb.debug(1, "No labels, nothing to do") return - cfile = d.getVar('SYSLINUXCFG', True) + cfile = d.getVar('SYSLINUX_CFG') if not cfile: - raise bb.build.FuncFailed('Unable to read SYSLINUXCFG') + bb.fatal('Unable to read SYSLINUX_CFG') try: - cfgfile = file(cfile, 'w') + cfgfile = open(cfile, 'w') except OSError: - raise bb.build.funcFailed('Unable to open %s' % (cfile)) + bb.fatal('Unable to open %s' % cfile) cfgfile.write('# Automatically created by OE\n') - opts = d.getVar('SYSLINUX_OPTS', True) + opts = d.getVar('SYSLINUX_OPTS') if opts: for opt in opts.split(';'): cfgfile.write('%s\n' % opt) - cfgfile.write('ALLOWOPTIONS 1\n'); - syslinux_default_console = d.getVar('SYSLINUX_DEFAULT_CONSOLE', True) - syslinux_serial_tty = d.getVar('SYSLINUX_SERIAL_TTY', True) - syslinux_serial = d.getVar('SYSLINUX_SERIAL', True) + allowoptions = d.getVar('SYSLINUX_ALLOWOPTIONS') + if allowoptions: + cfgfile.write('ALLOWOPTIONS %s\n' % allowoptions) + else: + cfgfile.write('ALLOWOPTIONS 1\n') + + syslinux_default_console = d.getVar('SYSLINUX_DEFAULT_CONSOLE') + syslinux_serial_tty = d.getVar('SYSLINUX_SERIAL_TTY') + syslinux_serial = d.getVar('SYSLINUX_SERIAL') if syslinux_serial: cfgfile.write('SERIAL %s\n' % syslinux_serial) - menu = d.getVar('AUTO_SYSLINUXMENU', True) + menu = (d.getVar('AUTO_SYSLINUXMENU') == "1") if menu and syslinux_serial: cfgfile.write('DEFAULT Graphics console %s\n' % (labels.split()[0])) else: cfgfile.write('DEFAULT %s\n' % (labels.split()[0])) - timeout = d.getVar('SYSLINUX_TIMEOUT', True) + timeout = d.getVar('SYSLINUX_TIMEOUT') if timeout: cfgfile.write('TIMEOUT %s\n' % timeout) else: cfgfile.write('TIMEOUT 50\n') - prompt = d.getVar('SYSLINUX_PROMPT', True) + prompt = d.getVar('SYSLINUX_PROMPT') if prompt: cfgfile.write('PROMPT %s\n' % prompt) else: @@ -141,47 +151,48 @@ python build_syslinux_cfg () { cfgfile.write('ui vesamenu.c32\n') cfgfile.write('menu title Select kernel options and boot kernel\n') cfgfile.write('menu tabmsg Press [Tab] to edit, [Return] to select\n') - splash = d.getVar('SYSLINUX_SPLASH', True) + splash = d.getVar('SYSLINUX_SPLASH') if splash: cfgfile.write('menu background splash.lss\n') for label in labels.split(): localdata = bb.data.createCopy(d) - overrides = localdata.getVar('OVERRIDES', True) + overrides = localdata.getVar('OVERRIDES') if not overrides: - raise bb.build.FuncFailed('OVERRIDES not defined') + bb.fatal('OVERRIDES not defined') localdata.setVar('OVERRIDES', label + ':' + overrides) - bb.data.update_data(localdata) btypes = [ [ "", syslinux_default_console ] ] if menu and syslinux_serial: btypes = [ [ "Graphics console ", syslinux_default_console ], [ "Serial console ", syslinux_serial_tty ] ] + root= d.getVar('SYSLINUX_ROOT') + if not root: + bb.fatal('SYSLINUX_ROOT not defined') + for btype in btypes: cfgfile.write('LABEL %s%s\nKERNEL /vmlinuz\n' % (btype[0], label)) - exargs = d.getVar('SYSLINUX_KERNEL_ARGS', True) + exargs = d.getVar('SYSLINUX_KERNEL_ARGS') if exargs: btype[1] += " " + exargs - append = localdata.getVar('APPEND', True) - initrd = localdata.getVar('INITRD', True) - - if append: - cfgfile.write('APPEND ') + append = localdata.getVar('APPEND') + initrd = localdata.getVar('INITRD') - if initrd: - cfgfile.write('initrd=/initrd ') + append = root + " " + append + cfgfile.write('APPEND ') - cfgfile.write('LABEL=%s '% (label)) + if initrd: + cfgfile.write('initrd=/initrd ') - cfgfile.write('%s %s\n' % (append, btype[1])) - else: - cfgfile.write('APPEND %s\n' % btype[1]) + cfgfile.write('LABEL=%s '% (label)) + append = replace_rootfs_uuid(d, append) + cfgfile.write('%s %s\n' % (append, btype[1])) cfgfile.close() } -build_syslinux_cfg[vardeps] += "APPEND" +build_syslinux_cfg[dirs] = "${S}" diff --git a/meta/classes/gummiboot.bbclass b/meta/classes/systemd-boot.bbclass index dae19775c3..4e69a2c6b5 100644 --- a/meta/classes/gummiboot.bbclass +++ b/meta/classes/systemd-boot.bbclass @@ -1,38 +1,46 @@ -# Copyright (C) 2014 Intel Corporation +# Copyright (C) 2016 Intel Corporation # # Released under the MIT license (see COPYING.MIT) -# gummiboot.bbclass - equivalent of grub-efi.bbclass -# Set EFI_PROVIDER = "gummiboot" to use gummiboot on your live images instead of grub-efi -# (images built by bootimage.bbclass or boot-directdisk.bbclass) +# systemd-boot.bbclass - The "systemd-boot" is essentially the gummiboot merged into systemd. +# The original standalone gummiboot project is dead without any more +# maintenance. +# +# Set EFI_PROVIDER = "systemd-boot" to use systemd-boot on your live images instead of grub-efi +# (images built by image-live.bbclass or image-vm.bbclass) -do_bootimg[depends] += "${MLPREFIX}gummiboot:do_deploy" -do_bootdirectdisk[depends] += "${MLPREFIX}gummiboot:do_deploy" +do_bootimg[depends] += "${MLPREFIX}systemd-boot:do_deploy" +do_bootdirectdisk[depends] += "${MLPREFIX}systemd-boot:do_deploy" EFIDIR = "/EFI/BOOT" -GUMMIBOOT_CFG ?= "${S}/loader.conf" -GUMMIBOOT_ENTRIES ?= "" -GUMMIBOOT_TIMEOUT ?= "10" +SYSTEMD_BOOT_CFG ?= "${S}/loader.conf" +SYSTEMD_BOOT_ENTRIES ?= "" +SYSTEMD_BOOT_TIMEOUT ?= "10" + +# Need UUID utility code. +inherit fs-uuid efi_populate() { DEST=$1 - EFI_IMAGE="gummibootia32.efi" + EFI_IMAGE="systemd-bootia32.efi" DEST_EFI_IMAGE="bootia32.efi" if [ "${TARGET_ARCH}" = "x86_64" ]; then - EFI_IMAGE="gummibootx64.efi" + EFI_IMAGE="systemd-bootx64.efi" DEST_EFI_IMAGE="bootx64.efi" fi install -d ${DEST}${EFIDIR} - # gummiboot requires these paths for configuration files + # systemd-boot requires these paths for configuration files # they are not customizable so no point in new vars install -d ${DEST}/loader install -d ${DEST}/loader/entries install -m 0644 ${DEPLOY_DIR_IMAGE}/${EFI_IMAGE} ${DEST}${EFIDIR}/${DEST_EFI_IMAGE} - install -m 0644 ${GUMMIBOOT_CFG} ${DEST}/loader/loader.conf - for i in ${GUMMIBOOT_ENTRIES}; do + EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g') + printf 'fs0:%s\%s\n' "$EFIPATH" "$DEST_EFI_IMAGE" >${DEST}/startup.nsh + install -m 0644 ${SYSTEMD_BOOT_CFG} ${DEST}/loader/loader.conf + for i in ${SYSTEMD_BOOT_ENTRIES}; do install -m 0644 ${i} ${DEST}/loader/entries done } @@ -43,7 +51,8 @@ efi_iso_populate() { mkdir -p ${EFIIMGDIR}/${EFIDIR} cp $iso_dir/${EFIDIR}/* ${EFIIMGDIR}${EFIDIR} cp $iso_dir/vmlinuz ${EFIIMGDIR} - echo "${DEST_EFI_IMAGE}" > ${EFIIMGDIR}/startup.nsh + EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g') + echo "fs0:${EFIPATH}\\${DEST_EFI_IMAGE}" > ${EFIIMGDIR}/startup.nsh if [ -f "$iso_dir/initrd" ] ; then cp $iso_dir/initrd ${EFIIMGDIR} fi @@ -54,8 +63,8 @@ efi_hddimg_populate() { } python build_efi_cfg() { - s = d.getVar("S", True) - labels = d.getVar('LABELS', True) + s = d.getVar("S") + labels = d.getVar('LABELS') if not labels: bb.debug(1, "LABELS not defined, nothing to do") return @@ -64,15 +73,18 @@ python build_efi_cfg() { bb.debug(1, "No labels, nothing to do") return - cfile = d.getVar('GUMMIBOOT_CFG', True) + cfile = d.getVar('SYSTEMD_BOOT_CFG') + cdir = os.path.dirname(cfile) + if not os.path.exists(cdir): + os.makedirs(cdir) try: cfgfile = open(cfile, 'w') except OSError: - raise bb.build.funcFailed('Unable to open %s' % (cfile)) + bb.fatal('Unable to open %s' % cfile) cfgfile.write('# Automatically created by OE\n') cfgfile.write('default %s\n' % (labels.split()[0])) - timeout = d.getVar('GUMMIBOOT_TIMEOUT', True) + timeout = d.getVar('SYSTEMD_BOOT_TIMEOUT') if timeout: cfgfile.write('timeout %s\n' % timeout) else: @@ -82,24 +94,23 @@ python build_efi_cfg() { for label in labels.split(): localdata = d.createCopy() - overrides = localdata.getVar('OVERRIDES', True) + overrides = localdata.getVar('OVERRIDES') if not overrides: - raise bb.build.FuncFailed('OVERRIDES not defined') + bb.fatal('OVERRIDES not defined') entryfile = "%s/%s.conf" % (s, label) - d.appendVar("GUMMIBOOT_ENTRIES", " " + entryfile) + d.appendVar("SYSTEMD_BOOT_ENTRIES", " " + entryfile) try: entrycfg = open(entryfile, "w") except OSError: - raise bb.build.funcFailed('Unable to open %s' % (entryfile)) + bb.fatal('Unable to open %s' % entryfile) localdata.setVar('OVERRIDES', label + ':' + overrides) - bb.data.update_data(localdata) entrycfg.write('title %s\n' % label) entrycfg.write('linux /vmlinuz\n') - append = localdata.getVar('APPEND', True) - initrd = localdata.getVar('INITRD', True) + append = localdata.getVar('APPEND') + initrd = localdata.getVar('INITRD') if initrd: entrycfg.write('initrd /initrd\n') @@ -108,6 +119,7 @@ python build_efi_cfg() { lb = "install-efi" entrycfg.write('options LABEL=%s ' % lb) if append: + append = replace_rootfs_uuid(d, append) entrycfg.write('%s' % append) entrycfg.write('\n') entrycfg.close() diff --git a/meta/classes/systemd.bbclass b/meta/classes/systemd.bbclass index 46e72c7a4a..c4b4bb9b70 100644 --- a/meta/classes/systemd.bbclass +++ b/meta/classes/systemd.bbclass @@ -17,6 +17,7 @@ python __anonymous() { # files. if bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d): d.appendVar("DEPENDS", " systemd-systemctl-native") + d.appendVar("PACKAGE_WRITE_DEPS", " systemd-systemctl-native") if not bb.utils.contains('DISTRO_FEATURES', 'sysvinit', True, False, d): d.setVar("INHIBIT_UPDATERCD_BBCLASS", "1") } @@ -29,10 +30,14 @@ if [ -n "$D" ]; then fi if type systemctl >/dev/null 2>/dev/null; then + if [ -z "$D" ]; then + systemctl daemon-reload + fi + systemctl $OPTS ${SYSTEMD_AUTO_ENABLE} ${SYSTEMD_SERVICE} if [ -z "$D" -a "${SYSTEMD_AUTO_ENABLE}" = "enable" ]; then - systemctl restart ${SYSTEMD_SERVICE} + systemctl --no-block restart ${SYSTEMD_SERVICE} fi fi } @@ -59,18 +64,20 @@ systemd_populate_packages[vardepsexclude] += "OVERRIDES" python systemd_populate_packages() { + import re + if not bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d): return def get_package_var(d, var, pkg): - val = (d.getVar('%s_%s' % (var, pkg), True) or "").strip() + val = (d.getVar('%s_%s' % (var, pkg)) or "").strip() if val == "": - val = (d.getVar(var, True) or "").strip() + val = (d.getVar(var) or "").strip() return val # Check if systemd-packages already included in PACKAGES def systemd_check_package(pkg_systemd): - packages = d.getVar('PACKAGES', True) + packages = d.getVar('PACKAGES') if not pkg_systemd in packages.split(): bb.error('%s does not appear in package list, please add it' % pkg_systemd) @@ -82,25 +89,24 @@ python systemd_populate_packages() { # variable. localdata = d.createCopy() localdata.prependVar("OVERRIDES", pkg + ":") - bb.data.update_data(localdata) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += localdata.getVar('systemd_postinst', True) + postinst += localdata.getVar('systemd_postinst') d.setVar('pkg_postinst_%s' % pkg, postinst) - prerm = d.getVar('pkg_prerm_%s' % pkg, True) + prerm = d.getVar('pkg_prerm_%s' % pkg) if not prerm: prerm = '#!/bin/sh\n' - prerm += localdata.getVar('systemd_prerm', True) + prerm += localdata.getVar('systemd_prerm') d.setVar('pkg_prerm_%s' % pkg, prerm) # Add files to FILES_*-systemd if existent and not already done def systemd_append_file(pkg_systemd, file_append): appended = False - if os.path.exists(oe.path.join(d.getVar("D", True), file_append)): + if os.path.exists(oe.path.join(d.getVar("D"), file_append)): var_name = "FILES_" + pkg_systemd files = d.getVar(var_name, False) or "" if file_append not in files.split(): @@ -112,7 +118,7 @@ python systemd_populate_packages() { def systemd_add_files_and_parse(pkg_systemd, path, service, keys): # avoid infinite recursion if systemd_append_file(pkg_systemd, oe.path.join(path, service)): - fullpath = oe.path.join(d.getVar("D", True), path, service) + fullpath = oe.path.join(d.getVar("D"), path, service) if service.find('.service') != -1: # for *.service add *@.service service_base = service.replace('.service', '') @@ -135,30 +141,41 @@ python systemd_populate_packages() { # Check service-files and call systemd_add_files_and_parse for each entry def systemd_check_services(): - searchpaths = [oe.path.join(d.getVar("sysconfdir", True), "systemd", "system"),] - searchpaths.append(d.getVar("systemd_system_unitdir", True)) - systemd_packages = d.getVar('SYSTEMD_PACKAGES', True) + searchpaths = [oe.path.join(d.getVar("sysconfdir"), "systemd", "system"),] + searchpaths.append(d.getVar("systemd_system_unitdir")) + systemd_packages = d.getVar('SYSTEMD_PACKAGES') keys = 'Also' # scan for all in SYSTEMD_SERVICE[] for pkg_systemd in systemd_packages.split(): for service in get_package_var(d, 'SYSTEMD_SERVICE', pkg_systemd).split(): path_found = '' + + # Deal with adding, for example, 'ifplugd@eth0.service' from + # 'ifplugd@.service' + base = None + if service.find('@') != -1: + base = re.sub('@[^.]+.', '@.', service) + for path in searchpaths: - if os.path.exists(oe.path.join(d.getVar("D", True), path, service)): + if os.path.exists(oe.path.join(d.getVar("D"), path, service)): path_found = path break + elif base is not None: + if os.path.exists(oe.path.join(d.getVar("D"), path, base)): + path_found = path + break + if path_found != '': systemd_add_files_and_parse(pkg_systemd, path_found, service, keys) else: - raise bb.build.FuncFailed("SYSTEMD_SERVICE_%s value %s does not exist" % \ - (pkg_systemd, service)) + bb.fatal("SYSTEMD_SERVICE_%s value %s does not exist" % (pkg_systemd, service)) # Run all modifications once when creating package - if os.path.exists(d.getVar("D", True)): - for pkg in d.getVar('SYSTEMD_PACKAGES', True).split(): + if os.path.exists(d.getVar("D")): + for pkg in d.getVar('SYSTEMD_PACKAGES').split(): systemd_check_package(pkg) - if d.getVar('SYSTEMD_SERVICE_' + pkg, True): + if d.getVar('SYSTEMD_SERVICE_' + pkg): systemd_generate_package_scripts(pkg) systemd_check_services() } @@ -168,7 +185,7 @@ PACKAGESPLITFUNCS_prepend = "systemd_populate_packages " python rm_systemd_unitdir (){ import shutil if not bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d): - systemd_unitdir = oe.path.join(d.getVar("D", True), d.getVar('systemd_unitdir', True)) + systemd_unitdir = oe.path.join(d.getVar("D"), d.getVar('systemd_unitdir')) if os.path.exists(systemd_unitdir): shutil.rmtree(systemd_unitdir) systemd_libdir = os.path.dirname(systemd_unitdir) @@ -179,12 +196,12 @@ do_install[postfuncs] += "rm_systemd_unitdir " python rm_sysvinit_initddir (){ import shutil - sysv_initddir = oe.path.join(d.getVar("D", True), (d.getVar('INIT_D_DIR', True) or "/etc/init.d")) + sysv_initddir = oe.path.join(d.getVar("D"), (d.getVar('INIT_D_DIR') or "/etc/init.d")) if bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d) and \ not bb.utils.contains('DISTRO_FEATURES', 'sysvinit', True, False, d) and \ os.path.exists(sysv_initddir): - systemd_system_unitdir = oe.path.join(d.getVar("D", True), d.getVar('systemd_system_unitdir', True)) + systemd_system_unitdir = oe.path.join(d.getVar("D"), d.getVar('systemd_system_unitdir')) # If systemd_system_unitdir contains anything, delete sysv_initddir if (os.path.exists(systemd_system_unitdir) and os.listdir(systemd_system_unitdir)): diff --git a/meta/classes/terminal.bbclass b/meta/classes/terminal.bbclass index e577c6d594..a27e10c6ec 100644 --- a/meta/classes/terminal.bbclass +++ b/meta/classes/terminal.bbclass @@ -1,25 +1,27 @@ OE_TERMINAL ?= 'auto' OE_TERMINAL[type] = 'choice' OE_TERMINAL[choices] = 'auto none \ - ${@" ".join(o.name \ - for o in oe.terminal.prioritized())}' + ${@oe_terminal_prioritized()}' -OE_TERMINAL_EXPORTS += 'EXTRA_OEMAKE' +OE_TERMINAL_EXPORTS += 'EXTRA_OEMAKE CACHED_CONFIGUREVARS CONFIGUREOPTS EXTRA_OECONF' OE_TERMINAL_EXPORTS[type] = 'list' XAUTHORITY ?= "${HOME}/.Xauthority" SHELL ?= "bash" +def oe_terminal_prioritized(): + import oe.terminal + return " ".join(o.name for o in oe.terminal.prioritized()) def emit_terminal_func(command, envdata, d): cmd_func = 'do_terminal' envdata.setVar(cmd_func, 'exec ' + command) - envdata.setVarFlag(cmd_func, 'func', 1) + envdata.setVarFlag(cmd_func, 'func', '1') - runfmt = d.getVar('BB_RUNFMT', True) or "run.{func}.{pid}" + runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" runfile = runfmt.format(func=cmd_func, task=cmd_func, taskfunc=cmd_func, pid=os.getpid()) - runfile = os.path.join(d.getVar('T', True), runfile) + runfile = os.path.join(d.getVar('T'), runfile) bb.utils.mkdirhier(os.path.dirname(runfile)) with open(runfile, 'w') as script: @@ -27,7 +29,7 @@ def emit_terminal_func(command, envdata, d): bb.data.emit_func(cmd_func, script, envdata) script.write(cmd_func) script.write("\n") - os.chmod(runfile, 0755) + os.chmod(runfile, 0o755) return runfile @@ -39,14 +41,14 @@ def oe_terminal(command, title, d): for v in os.environ: envdata.setVar(v, os.environ[v]) - envdata.setVarFlag(v, 'export', 1) + envdata.setVarFlag(v, 'export', '1') for export in oe.data.typed_value('OE_TERMINAL_EXPORTS', d): - value = d.getVar(export, True) + value = d.getVar(export) if value is not None: os.environ[export] = str(value) envdata.setVar(export, str(value)) - envdata.setVarFlag(export, 'export', 1) + envdata.setVarFlag(export, 'export', '1') if export == "PSEUDO_DISABLED": if "PSEUDO_UNLOAD" in os.environ: del os.environ["PSEUDO_UNLOAD"] @@ -58,11 +60,16 @@ def oe_terminal(command, title, d): for key in origbbenv: if key in envdata: continue - value = origbbenv.getVar(key, True) + value = origbbenv.getVar(key) if value is not None: os.environ[key] = str(value) envdata.setVar(key, str(value)) - envdata.setVarFlag(key, 'export', 1) + envdata.setVarFlag(key, 'export', '1') + + # Use original PATH as a fallback + path = d.getVar('PATH') + ":" + origbbenv.getVar('PATH') + os.environ['PATH'] = path + envdata.setVar('PATH', path) # A complex PS1 might need more escaping of chars. # Lets not export PS1 instead. @@ -86,8 +93,12 @@ def oe_terminal(command, title, d): try: oe.terminal.spawn_preferred(command, title, None, d) - except oe.terminal.NoSupportedTerminals: - bb.fatal('No valid terminal found, unable to open devshell') + except oe.terminal.NoSupportedTerminals as nosup: + nosup.terms.remove("false") + cmds = '\n\t'.join(nosup.terms).replace("{command}", + "do_terminal").replace("{title}", title) + bb.fatal('No valid terminal found, unable to open devshell.\n' + + 'Tried the following commands:\n\t%s' % cmds) except oe.terminal.ExecutionError as exc: bb.fatal('Unable to spawn terminal %s: %s' % (terminal, exc)) diff --git a/meta/classes/testexport.bbclass b/meta/classes/testexport.bbclass new file mode 100644 index 0000000000..56edda9943 --- /dev/null +++ b/meta/classes/testexport.bbclass @@ -0,0 +1,177 @@ +# Copyright (C) 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +# +# testexport.bbclass allows to execute runtime test outside OE environment. +# Most of the tests are commands run on target image over ssh. +# To use it add testexport to global inherit and call your target image with -c testexport +# You can try it out like this: +# - First build an image. i.e. core-image-sato +# - Add INHERIT += "testexport" in local.conf +# - Then bitbake core-image-sato -c testexport. That will generate the directory structure +# to execute the runtime tests using runexported.py. +# +# For more information on TEST_SUITES check testimage class. + +TEST_LOG_DIR ?= "${WORKDIR}/testexport" +TEST_EXPORT_DIR ?= "${TMPDIR}/testexport/${PN}" +TEST_EXPORT_PACKAGED_DIR ?= "packages/packaged" +TEST_EXPORT_EXTRACTED_DIR ?= "packages/extracted" + +TEST_TARGET ?= "simpleremote" +TEST_TARGET_IP ?= "" +TEST_SERVER_IP ?= "" + +TEST_EXPORT_SDK_PACKAGES ?= "" +TEST_EXPORT_SDK_ENABLED ?= "0" +TEST_EXPORT_SDK_NAME ?= "testexport-tools-nativesdk" +TEST_EXPORT_SDK_DIR ?= "sdk" + +TEST_EXPORT_DEPENDS = "" +TEST_EXPORT_DEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}" +TEST_EXPORT_DEPENDS += "${@bb.utils.contains('TEST_EXPORT_SDK_ENABLED', '1', 'testexport-tarball:do_populate_sdk', '', d)}" +TEST_EXPORT_LOCK = "${TMPDIR}/testimage.lock" + +addtask testexport +do_testexport[nostamp] = "1" +do_testexport[depends] += "${TEST_EXPORT_DEPENDS} ${TESTIMAGEDEPENDS}" +do_testexport[lockfiles] += "${TEST_EXPORT_LOCK}" + +python do_testexport() { + testexport_main(d) +} + +def testexport_main(d): + import json + import logging + + from oeqa.runtime.context import OERuntimeTestContext + from oeqa.runtime.context import OERuntimeTestContextExecutor + + image_name = ("%s/%s" % (d.getVar('DEPLOY_DIR_IMAGE'), + d.getVar('IMAGE_LINK_NAME'))) + + tdname = "%s.testdata.json" % image_name + td = json.load(open(tdname, "r")) + + logger = logging.getLogger("BitBake") + + target = OERuntimeTestContextExecutor.getTarget( + d.getVar("TEST_TARGET"), None, d.getVar("TEST_TARGET_IP"), + d.getVar("TEST_SERVER_IP")) + + host_dumper = OERuntimeTestContextExecutor.getHostDumper( + d.getVar("testimage_dump_host"), d.getVar("TESTIMAGE_DUMP_DIR")) + + image_manifest = "%s.manifest" % image_name + image_packages = OERuntimeTestContextExecutor.readPackagesManifest(image_manifest) + + extract_dir = d.getVar("TEST_EXTRACTED_DIR") + + tc = OERuntimeTestContext(td, logger, target, host_dumper, + image_packages, extract_dir) + + copy_needed_files(d, tc) + +def copy_needed_files(d, tc): + import shutil + import oe.path + + from oeqa.utils.package_manager import _get_json_file + from oeqa.core.utils.test import getSuiteCasesFiles + + export_path = d.getVar('TEST_EXPORT_DIR') + corebase_path = d.getVar('COREBASE') + + # Clean everything before starting + oe.path.remove(export_path) + bb.utils.mkdirhier(os.path.join(export_path, 'lib', 'oeqa')) + + # The source of files to copy are relative to 'COREBASE' directory + # The destination is relative to 'TEST_EXPORT_DIR' + # Because we are squashing the libraries, we need to remove + # the layer/script directory + files_to_copy = [ os.path.join('meta', 'lib', 'oeqa', 'core'), + os.path.join('meta', 'lib', 'oeqa', 'runtime'), + os.path.join('meta', 'lib', 'oeqa', 'files'), + os.path.join('meta', 'lib', 'oeqa', 'utils'), + os.path.join('scripts', 'oe-test'), + os.path.join('scripts', 'lib', 'argparse_oe.py'), + os.path.join('scripts', 'lib', 'scriptutils.py'), ] + + for f in files_to_copy: + src = os.path.join(corebase_path, f) + dst = os.path.join(export_path, f.split('/', 1)[-1]) + if os.path.isdir(src): + oe.path.copytree(src, dst) + else: + shutil.copy2(src, dst) + + # Remove cases and just copy the ones specified + cases_path = os.path.join(export_path, 'lib', 'oeqa', 'runtime', 'cases') + oe.path.remove(cases_path) + bb.utils.mkdirhier(cases_path) + test_paths = get_runtime_paths(d) + test_modules = d.getVar('TEST_SUITES') + tc.loadTests(test_paths, modules=test_modules) + for f in getSuiteCasesFiles(tc.suites): + shutil.copy2(f, cases_path) + json_file = _get_json_file(f) + if json_file: + shutil.copy2(json_file, cases_path) + + # Copy test data + image_name = ("%s/%s" % (d.getVar('DEPLOY_DIR_IMAGE'), + d.getVar('IMAGE_LINK_NAME'))) + image_manifest = "%s.manifest" % image_name + tdname = "%s.testdata.json" % image_name + test_data_path = os.path.join(export_path, 'data') + bb.utils.mkdirhier(test_data_path) + shutil.copy2(image_manifest, os.path.join(test_data_path, 'manifest')) + shutil.copy2(tdname, os.path.join(test_data_path, 'testdata.json')) + + # Create tar file for common parts of testexport + create_tarball(d, "testexport.tar.gz", d.getVar("TEST_EXPORT_DIR")) + + # Copy packages needed for runtime testing + package_extraction(d, tc.suites) + test_pkg_dir = d.getVar("TEST_NEEDED_PACKAGES_DIR") + if os.path.isdir(test_pkg_dir) and os.listdir(test_pkg_dir): + export_pkg_dir = os.path.join(d.getVar("TEST_EXPORT_DIR"), "packages") + oe.path.copytree(test_pkg_dir, export_pkg_dir) + # Create tar file for packages needed by the DUT + create_tarball(d, "testexport_packages_%s.tar.gz" % d.getVar("MACHINE"), export_pkg_dir) + + # Copy SDK + if d.getVar("TEST_EXPORT_SDK_ENABLED") == "1": + sdk_deploy = d.getVar("SDK_DEPLOY") + tarball_name = "%s.sh" % d.getVar("TEST_EXPORT_SDK_NAME") + tarball_path = os.path.join(sdk_deploy, tarball_name) + export_sdk_dir = os.path.join(d.getVar("TEST_EXPORT_DIR"), + d.getVar("TEST_EXPORT_SDK_DIR")) + bb.utils.mkdirhier(export_sdk_dir) + shutil.copy2(tarball_path, export_sdk_dir) + + # Create tar file for the sdk + create_tarball(d, "testexport_sdk_%s.tar.gz" % d.getVar("SDK_ARCH"), export_sdk_dir) + + bb.plain("Exported tests to: %s" % export_path) + +def create_tarball(d, tar_name, src_dir): + + import tarfile + + tar_path = os.path.join(d.getVar("TEST_EXPORT_DIR"), tar_name) + current_dir = os.getcwd() + src_dir = src_dir.rstrip('/') + dir_name = os.path.dirname(src_dir) + base_name = os.path.basename(src_dir) + + os.chdir(dir_name) + tar = tarfile.open(tar_path, "w:gz") + tar.add(base_name) + tar.close() + os.chdir(current_dir) + +inherit testimage diff --git a/meta/classes/testimage-auto.bbclass b/meta/classes/testimage-auto.bbclass index 860599d2b5..e0a22b773c 100644 --- a/meta/classes/testimage-auto.bbclass +++ b/meta/classes/testimage-auto.bbclass @@ -18,6 +18,6 @@ inherit testimage python do_testimage_auto() { testimage_main(d) } -addtask testimage_auto before do_build after do_rootfs +addtask testimage_auto before do_build after do_image_complete do_testimage_auto[depends] += "${TESTIMAGEDEPENDS}" do_testimage_auto[lockfiles] += "${TESTIMAGELOCK}" diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass index 8fa00f5fbf..fb214604a9 100644 --- a/meta/classes/testimage.bbclass +++ b/meta/classes/testimage.bbclass @@ -8,7 +8,7 @@ # To use it add testimage to global inherit and call your target image with -c testimage # You can try it out like this: # - first build a qemu core-image-sato -# - add INHERIT += "testimage" in local.conf +# - add IMAGE_CLASSES += "testimage" in local.conf # - then bitbake core-image-sato -c testimage. That will run a standard suite of tests. # You can set (or append to) TEST_SUITES in local.conf to select the tests @@ -30,34 +30,63 @@ TEST_LOG_DIR ?= "${WORKDIR}/testimage" TEST_EXPORT_DIR ?= "${TMPDIR}/testimage/${PN}" -TEST_EXPORT_ONLY ?= "0" - -DEFAULT_TEST_SUITES = "ping auto" -DEFAULT_TEST_SUITES_pn-core-image-minimal = "ping" -DEFAULT_TEST_SUITES_pn-core-image-sato = "ping ssh df connman syslog xorg scp vnc date dmesg parselogs \ - ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python smart rpm', '', d)}" -DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "ping ssh df connman syslog xorg scp vnc date perl ldd gcc kernelmodule dmesg python parselogs \ - ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'smart rpm', '', d)}" +TEST_INSTALL_TMP_DIR ?= "${WORKDIR}/testimage/install_tmp" +TEST_NEEDED_PACKAGES_DIR ?= "${WORKDIR}/testimage/packages" +TEST_EXTRACTED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/extracted" +TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged" + +RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf rpm', '', d)}" +SYSTEMDSUITE = "${@bb.utils.filter('DISTRO_FEATURES', 'systemd', d)}" +MINTESTSUITE = "ping" +NETTESTSUITE = "${MINTESTSUITE} ssh df date scp oe_syslog ${SYSTEMDSUITE}" +DEVTESTSUITE = "gcc kernelmodule ldd" + +DEFAULT_TEST_SUITES = "${MINTESTSUITE} auto" +DEFAULT_TEST_SUITES_pn-core-image-minimal = "${MINTESTSUITE}" +DEFAULT_TEST_SUITES_pn-core-image-minimal-dev = "${MINTESTSUITE}" +DEFAULT_TEST_SUITES_pn-core-image-full-cmdline = "${NETTESTSUITE} perl python logrotate" +DEFAULT_TEST_SUITES_pn-core-image-x11 = "${MINTESTSUITE}" +DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${RPMTESTSUITE}" +DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${RPMTESTSUITE} \ + ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python', '', d)}" +DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} buildcpio buildiptables buildgalculator \ + connman ${DEVTESTSUITE} logrotate perl parselogs python ${RPMTESTSUITE} xorg" +DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}" +DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcpio buildiptables buildgalculator \ + connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${RPMTESTSUITE}" DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto" # aarch64 has no graphics -DEFAULT_TEST_SUITES_remove_aarch64 = "xorg vnc" +DEFAULT_TEST_SUITES_remove_aarch64 = "xorg" + +# qemumips is quite slow and has reached the timeout limit several times on the YP build cluster, +# mitigate this by removing build tests for qemumips machines. +MIPSREMOVE ??= "buildcpio buildiptables buildgalculator" +DEFAULT_TEST_SUITES_remove_qemumips = "${MIPSREMOVE}" +DEFAULT_TEST_SUITES_remove_qemumips64 = "${MIPSREMOVE}" TEST_SUITES ?= "${DEFAULT_TEST_SUITES}" TEST_QEMUBOOT_TIMEOUT ?= "1000" TEST_TARGET ?= "qemu" -TEST_TARGET_IP ?= "" -TEST_SERVER_IP ?= "" TESTIMAGEDEPENDS = "" -TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot" +TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot" +TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}" +TESTIMAGEDEPENDS_qemuall += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}" +TESTIMAGEDEPENDS_qemuall += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-native:do_populate_sysroot', '', d)}" +TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf-native:do_populate_sysroot', '', d)}" +TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg-utils-native:do_populate_sysroot', '', d)}" +TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt-native:do_populate_sysroot', '', d)}" +TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-native:do_populate_sysroot', '', d)}" TESTIMAGELOCK = "${TMPDIR}/testimage.lock" TESTIMAGELOCK_qemuall = "" TESTIMAGE_DUMP_DIR ?= "/tmp/oe-saved-tests/" +TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR" + testimage_dump_target () { top -bn1 ps @@ -75,327 +104,270 @@ testimage_dump_target () { testimage_dump_host () { top -bn1 + iostat -x -z -N -d -p ALL 20 2 ps -ef free df memstat dmesg + ip -s link netstat -an } python do_testimage() { + + testimage_sanity(d) + + if (d.getVar('IMAGE_PKGTYPE') == 'rpm' + and 'dnf' in d.getVar('TEST_SUITES')): + create_rpm_index(d) + testimage_main(d) } + addtask testimage do_testimage[nostamp] = "1" do_testimage[depends] += "${TESTIMAGEDEPENDS}" do_testimage[lockfiles] += "${TESTIMAGELOCK}" -python do_testsdk() { - testsdk_main(d) -} -addtask testsdk -do_testsdk[nostamp] = "1" -do_testsdk[depends] += "${TESTIMAGEDEPENDS}" -do_testsdk[lockfiles] += "${TESTIMAGELOCK}" - -# get testcase list from specified file -# if path is a relative path, then relative to build/conf/ -def read_testlist(d, fpath): - if not os.path.isabs(fpath): - builddir = d.getVar("TOPDIR", True) - fpath = os.path.join(builddir, "conf", fpath) - if not os.path.exists(fpath): - bb.fatal("No such manifest file: ", fpath) - tcs = [] - for line in open(fpath).readlines(): - line = line.strip() - if line and not line.startswith("#"): - tcs.append(line) - return " ".join(tcs) - -def get_tests_list(d, type="runtime"): - testsuites = [] - testslist = [] - manifests = d.getVar("TEST_SUITES_MANIFEST", True) - if manifests is not None: - manifests = manifests.split() - for manifest in manifests: - testsuites.extend(read_testlist(d, manifest).split()) - else: - testsuites = d.getVar("TEST_SUITES", True).split() - if type == "sdk": - testsuites = (d.getVar("TEST_SUITES_SDK", True) or "auto").split() - bbpath = d.getVar("BBPATH", True).split(':') - - # This relies on lib/ under each directory in BBPATH being added to sys.path - # (as done by default in base.bbclass) - for testname in testsuites: - if testname != "auto": - if testname.startswith("oeqa."): - testslist.append(testname) - continue - found = False - for p in bbpath: - if os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname + '.py')): - testslist.append("oeqa." + type + "." + testname) - found = True - break - if not found: - bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname) - - if "auto" in testsuites: - def add_auto_list(path): - if not os.path.exists(os.path.join(path, '__init__.py')): - bb.fatal('Tests directory %s exists but is missing __init__.py' % path) - files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) - for f in files: - module = 'oeqa.' + type + '.' + f[:-3] - if module not in testslist: - testslist.append(module) - - for p in bbpath: - testpath = os.path.join(p, 'lib', 'oeqa', type) - bb.debug(2, 'Searching for tests in %s' % testpath) - if os.path.exists(testpath): - add_auto_list(testpath) - - return testslist - - -def exportTests(d,tc): - import json - import shutil - import pkgutil - - exportpath = d.getVar("TEST_EXPORT_DIR", True) - - savedata = {} - savedata["d"] = {} - savedata["target"] = {} - savedata["host_dumper"] = {} - for key in tc.__dict__: - # special cases - if key != "d" and key != "target" and key != "host_dumper": - savedata[key] = getattr(tc, key) - savedata["target"]["ip"] = tc.target.ip or d.getVar("TEST_TARGET_IP", True) - savedata["target"]["server_ip"] = tc.target.server_ip or d.getVar("TEST_SERVER_IP", True) - - keys = [ key for key in d.keys() if not key.startswith("_") and not key.startswith("BB") \ - and not key.startswith("B_pn") and not key.startswith("do_") and not d.getVarFlag(key, "func")] - for key in keys: - try: - savedata["d"][key] = d.getVar(key, True) - except bb.data_smart.ExpansionError: - # we don't care about those anyway - pass - - savedata["host_dumper"]["parent_dir"] = tc.host_dumper.parent_dir - savedata["host_dumper"]["cmds"] = tc.host_dumper.cmds - - with open(os.path.join(exportpath, "testdata.json"), "w") as f: - json.dump(savedata, f, skipkeys=True, indent=4, sort_keys=True) - - # now start copying files - # we'll basically copy everything under meta/lib/oeqa, with these exceptions - # - oeqa/targetcontrol.py - not needed - # - oeqa/selftest - something else - # That means: - # - all tests from oeqa/runtime defined in TEST_SUITES (including from other layers) - # - the contents of oeqa/utils and oeqa/runtime/files - # - oeqa/oetest.py and oeqa/runexport.py (this will get copied to exportpath not exportpath/oeqa) - # - __init__.py files - bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/runtime/files")) - bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/utils")) - # copy test modules, this should cover tests in other layers too - for t in tc.testslist: - mod = pkgutil.get_loader(t) - shutil.copy2(mod.filename, os.path.join(exportpath, "oeqa/runtime")) - # copy __init__.py files - oeqadir = pkgutil.get_loader("oeqa").filename - shutil.copy2(os.path.join(oeqadir, "__init__.py"), os.path.join(exportpath, "oeqa")) - shutil.copy2(os.path.join(oeqadir, "runtime/__init__.py"), os.path.join(exportpath, "oeqa/runtime")) - # copy oeqa/oetest.py and oeqa/runexported.py - shutil.copy2(os.path.join(oeqadir, "oetest.py"), os.path.join(exportpath, "oeqa")) - shutil.copy2(os.path.join(oeqadir, "runexported.py"), exportpath) - # copy oeqa/utils/*.py - for root, dirs, files in os.walk(os.path.join(oeqadir, "utils")): - for f in files: - if f.endswith(".py"): - shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/utils")) - # copy oeqa/runtime/files/* - for root, dirs, files in os.walk(os.path.join(oeqadir, "runtime/files")): - for f in files: - shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files")) - - bb.plain("Exported tests to: %s" % exportpath) - +def testimage_sanity(d): + if (d.getVar('TEST_TARGET') == 'simpleremote' + and (not d.getVar('TEST_TARGET_IP') + or not d.getVar('TEST_SERVER_IP'))): + bb.fatal('When TEST_TARGET is set to "simpleremote" ' + 'TEST_TARGET_IP and TEST_SERVER_IP are needed too.') def testimage_main(d): - import unittest import os - import oeqa.runtime - import time - from oeqa.oetest import loadTests, runTests - from oeqa.targetcontrol import get_target_controller - from oeqa.utils.dump import get_host_dumper - - pn = d.getVar("PN", True) - export = oe.utils.conditional("TEST_EXPORT_ONLY", "1", True, False, d) - bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) - if export: - bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True) - bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True)) - - # tests in TEST_SUITES become required tests - # they won't be skipped even if they aren't suitable for a image (like xorg for minimal) - # testslist is what we'll actually pass to the unittest loader - testslist = get_tests_list(d) - testsrequired = [t for t in d.getVar("TEST_SUITES", True).split() if t != "auto"] + import json + import signal + import logging + + from bb.utils import export_proxies + from oeqa.core.utils.misc import updateTestData + from oeqa.runtime.context import OERuntimeTestContext + from oeqa.runtime.context import OERuntimeTestContextExecutor + from oeqa.core.target.qemu import supported_fstypes + from oeqa.core.utils.test import getSuiteCases + from oeqa.utils import make_logger_bitbake_compatible + + def sigterm_exception(signum, stackframe): + """ + Catch SIGTERM from worker in order to stop qemu. + """ + raise RuntimeError + + logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) + pn = d.getVar("PN") + + bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR")) + + image_name = ("%s/%s" % (d.getVar('DEPLOY_DIR_IMAGE'), + d.getVar('IMAGE_LINK_NAME'))) + + tdname = "%s.testdata.json" % image_name + try: + td = json.load(open(tdname, "r")) + except (FileNotFoundError) as err: + bb.fatal('File %s Not Found. Have you built the image with INHERIT+="testimage" in the conf/local.conf?' % tdname) + + # Some variables need to be updates (mostly paths) with the + # ones of the current environment because some tests require them. + updateTestData(d, td, d.getVar('TESTIMAGE_UPDATE_VARS').split()) + + image_manifest = "%s.manifest" % image_name + image_packages = OERuntimeTestContextExecutor.readPackagesManifest(image_manifest) + + extract_dir = d.getVar("TEST_EXTRACTED_DIR") + + # Get machine + machine = d.getVar("MACHINE") + + # Get rootfs + fstypes = [fs for fs in d.getVar('IMAGE_FSTYPES').split(' ') + if fs in supported_fstypes] + if not fstypes: + bb.fatal('Unsupported image type built. Add a comptible image to ' + 'IMAGE_FSTYPES. Supported types: %s' % + ', '.join(supported_fstypes)) + rootfs = '%s.%s' % (image_name, fstypes[0]) + + # Get tmpdir (not really used, just for compatibility) + tmpdir = d.getVar("TMPDIR") + + # Get deploy_dir_image (not really used, just for compatibility) + dir_image = d.getVar("DEPLOY_DIR_IMAGE") + + # Get bootlog + bootlog = os.path.join(d.getVar("TEST_LOG_DIR"), + 'qemu_boot_log.%s' % d.getVar('DATETIME')) + + # Get display + display = d.getVar("BB_ORIGENV").getVar("DISPLAY") + + # Get kernel + kernel_name = ('%s-%s.bin' % (d.getVar("KERNEL_IMAGETYPE"), machine)) + kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), kernel_name) + + # Get boottime + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")) + + # Get use_kvm + qemu_use_kvm = d.getVar("QEMU_USE_KVM") + if qemu_use_kvm and \ + (qemu_use_kvm == 'True' and 'x86' in machine or \ + d.getVar('MACHINE') in qemu_use_kvm.split()): + kvm = True + else: + kvm = False + + # TODO: We use the current implementatin of qemu runner because of + # time constrains, qemu runner really needs a refactor too. + target_kwargs = { 'machine' : machine, + 'rootfs' : rootfs, + 'tmpdir' : tmpdir, + 'dir_image' : dir_image, + 'display' : display, + 'kernel' : kernel, + 'boottime' : boottime, + 'bootlog' : bootlog, + 'kvm' : kvm, + } + + # TODO: Currently BBPATH is needed for custom loading of targets. + # It would be better to find these modules using instrospection. + target_kwargs['target_modules_path'] = d.getVar('BBPATH') + + # runtime use network for download projects for build + export_proxies(d) # we need the host dumper in test context - host_dumper = get_host_dumper(d) + host_dumper = OERuntimeTestContextExecutor.getHostDumper( + d.getVar("testimage_dump_host"), + d.getVar("TESTIMAGE_DUMP_DIR")) # the robot dance - target = get_target_controller(d) - - class TestContext(object): - def __init__(self): - self.d = d - self.testslist = testslist - self.testsrequired = testsrequired - self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files") - self.target = target - self.host_dumper = host_dumper - self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() - self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() - manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + ".manifest") - try: - with open(manifest) as f: - self.pkgmanifest = f.read() - except IOError as e: - bb.fatal("No package manifest file found. Did you build the image?\n%s" % e) + target = OERuntimeTestContextExecutor.getTarget( + d.getVar("TEST_TARGET"), None, d.getVar("TEST_TARGET_IP"), + d.getVar("TEST_SERVER_IP"), **target_kwargs) # test context - tc = TestContext() + tc = OERuntimeTestContext(td, logger, target, host_dumper, + image_packages, extract_dir) - # this is a dummy load of tests - # we are doing that to find compile errors in the tests themselves - # before booting the image - try: - loadTests(tc) - except Exception as e: - import traceback - bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) + # Load tests before starting the target + test_paths = get_runtime_paths(d) + test_modules = d.getVar('TEST_SUITES') + tc.loadTests(test_paths, modules=test_modules) + + if not getSuiteCases(tc.suites): + bb.fatal('Empty test suite, please verify TEST_SUITES variable') + + package_extraction(d, tc.suites) - target.deploy() + bootparams = None + if d.getVar('VIRTUAL-RUNTIME_init_manager', '') == 'systemd': + # Add systemd.log_level=debug to enable systemd debug logging + bootparams = 'systemd.log_target=console' - target.start() + results = None + orig_sigterm_handler = signal.signal(signal.SIGTERM, sigterm_exception) try: - if export: - exportTests(d,tc) + # We need to check if runqemu ends unexpectedly + # or if the worker send us a SIGTERM + tc.target.start(extra_bootparams=bootparams) + results = tc.runTests() + except (RuntimeError, BlockingIOError) as err: + if isinstance(err, RuntimeError): + bb.error('testimage received SIGTERM, shutting down...') else: - starttime = time.time() - result = runTests(tc) - stoptime = time.time() - if result.wasSuccessful(): - bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) - msg = "%s - OK - All required tests passed" % pn - skipped = len(result.skipped) - if skipped: - msg += " (skipped=%d)" % skipped - bb.plain(msg) - else: - raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn ) + bb.error('runqemu failed, shutting down...') + if results: + results.stop() + results = None finally: - target.stop() - -testimage_main[vardepsexclude] =+ "BB_ORIGENV" - - -def testsdk_main(d): - import unittest - import os - import glob - import oeqa.runtime - import oeqa.sdk - import time + signal.signal(signal.SIGTERM, orig_sigterm_handler) + tc.target.stop() + + # Show results (if we have them) + if not results: + bb.fatal('%s - FAILED - tests were interrupted during execution' % pn) + tc.logSummary(results, pn) + tc.logDetails() + if not results.wasSuccessful(): + bb.fatal('%s - FAILED - check the task log and the ssh log' % pn) + +def get_runtime_paths(d): + """ + Returns a list of paths where runtime test must reside. + + Runtime tests are expected in <LAYER_DIR>/lib/oeqa/runtime/cases/ + """ + paths = [] + + for layer in d.getVar('BBLAYERS').split(): + path = os.path.join(layer, 'lib/oeqa/runtime/cases') + if os.path.isdir(path): + paths.append(path) + return paths + +def create_index(arg): import subprocess - from oeqa.oetest import loadTests, runTests - - pn = d.getVar("PN", True) - bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) - - # tests in TEST_SUITES become required tests - # they won't be skipped even if they aren't suitable. - # testslist is what we'll actually pass to the unittest loader - testslist = get_tests_list(d, "sdk") - testsrequired = [t for t in (d.getVar("TEST_SUITES_SDK", True) or "auto").split() if t != "auto"] - - tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh") - if not os.path.exists(tcname): - bb.fatal("The toolchain is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' .") - - class TestContext(object): - def __init__(self): - self.d = d - self.testslist = testslist - self.testsrequired = testsrequired - self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files") - self.sdktestdir = sdktestdir - self.sdkenv = sdkenv - self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() - self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() - manifest = d.getVar("SDK_TARGET_MANIFEST", True) - try: - with open(manifest) as f: - self.pkgmanifest = f.read() - except IOError as e: - bb.fatal("No package manifest file found. Did you build the sdk image?\n%s" % e) - hostmanifest = d.getVar("SDK_HOST_MANIFEST", True) - try: - with open(hostmanifest) as f: - self.hostpkgmanifest = f.read() - except IOError as e: - bb.fatal("No host package manifest file found. Did you build the sdk image?\n%s" % e) - - sdktestdir = d.expand("${WORKDIR}/testimage-sdk/") - bb.utils.remove(sdktestdir, True) - bb.utils.mkdirhier(sdktestdir) - subprocess.call("cd %s; %s <<EOF\n./tc\nY\nEOF" % (sdktestdir, tcname), shell=True) + index_cmd = arg try: - targets = glob.glob(d.expand(sdktestdir + "/tc/environment-setup-*")) - bb.warn(str(targets)) - for sdkenv in targets: - bb.plain("Testing %s" % sdkenv) - # test context - tc = TestContext() - - # this is a dummy load of tests - # we are doing that to find compile errors in the tests themselves - # before booting the image - try: - loadTests(tc, "sdk") - except Exception as e: - import traceback - bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) - - - starttime = time.time() - result = runTests(tc, "sdk") - stoptime = time.time() - if result.wasSuccessful(): - bb.plain("%s SDK(%s):%s - Ran %d test%s in %.3fs" % (pn, os.path.basename(tcname), os.path.basename(sdkenv),result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) - msg = "%s - OK - All required tests passed" % pn - skipped = len(result.skipped) - if skipped: - msg += " (skipped=%d)" % skipped - bb.plain(msg) - else: - raise bb.build.FuncFailed("%s - FAILED - check the task log and the commands log" % pn ) - finally: - bb.utils.remove(sdktestdir, True) - -testsdk_main[vardepsexclude] =+ "BB_ORIGENV" - + bb.note("Executing '%s' ..." % index_cmd) + result = subprocess.check_output(index_cmd, + stderr=subprocess.STDOUT, + shell=True) + result = result.decode('utf-8') + except subprocess.CalledProcessError as e: + return("Index creation command '%s' failed with return code " + '%d:\n%s' % (e.cmd, e.returncode, e.output.decode("utf-8"))) + if result: + bb.note(result) + return None + +def create_rpm_index(d): + # Index RPMs + rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo_c") + index_cmds = [] + archs = (d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or '').replace('-', '_') + + for arch in archs.split(): + rpm_dir = os.path.join(d.getVar('DEPLOY_DIR_RPM'), arch) + idx_path = os.path.join(d.getVar('WORKDIR'), 'oe-testimage-repo', arch) + + if not os.path.isdir(rpm_dir): + continue + + lockfilename = os.path.join(d.getVar('DEPLOY_DIR_RPM'), 'rpm.lock') + lf = bb.utils.lockfile(lockfilename, False) + oe.path.copyhardlinktree(rpm_dir, idx_path) + # Full indexes overload a 256MB image so reduce the number of rpms + # in the feed. Filter to r* since we use the run-postinst packages and + # this leaves some allarch and machine arch packages too. + bb.utils.remove(idx_path + "*/[a-qs-z]*.rpm") + bb.utils.unlockfile(lf) + cmd = '%s --update -q %s' % (rpm_createrepo, idx_path) + + # Create repodata + result = create_index(cmd) + if result: + bb.fatal('%s' % ('\n'.join(result))) + +def package_extraction(d, test_suites): + from oeqa.utils.package_manager import find_packages_to_extract + from oeqa.utils.package_manager import extract_packages + + bb.utils.remove(d.getVar("TEST_NEEDED_PACKAGES_DIR"), recurse=True) + packages = find_packages_to_extract(test_suites) + if packages: + bb.utils.mkdirhier(d.getVar("TEST_INSTALL_TMP_DIR")) + bb.utils.mkdirhier(d.getVar("TEST_PACKAGED_DIR")) + bb.utils.mkdirhier(d.getVar("TEST_EXTRACTED_DIR")) + extract_packages(d, packages) + +testimage_main[vardepsexclude] += "BB_ORIGENV DATETIME" + +inherit testsdk diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass new file mode 100644 index 0000000000..6a201aa41b --- /dev/null +++ b/meta/classes/testsdk.bbclass @@ -0,0 +1,192 @@ +# Copyright (C) 2013 - 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# testsdk.bbclass enables testing for SDK and Extensible SDK +# +# To run SDK tests, run the commands: +# $ bitbake <image-name> -c populate_sdk +# $ bitbake <image-name> -c testsdk +# +# To run eSDK tests, run the commands: +# $ bitbake <image-name> -c populate_sdk_ext +# $ bitbake <image-name> -c testsdkext +# +# where "<image-name>" is an image like core-image-sato. + +def testsdk_main(d): + import os + import subprocess + import json + import logging + + from bb.utils import export_proxies + from oeqa.core.runner import OEStreamLogger + from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor + from oeqa.utils import make_logger_bitbake_compatible + + pn = d.getVar("PN") + logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) + + # sdk use network for download projects for build + export_proxies(d) + + tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh") + if not os.path.exists(tcname): + bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname) + + tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json") + test_data = json.load(open(tdname, "r")) + + target_pkg_manifest = OESDKTestContextExecutor._load_manifest( + d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest")) + host_pkg_manifest = OESDKTestContextExecutor._load_manifest( + d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest")) + + sdk_dir = d.expand("${WORKDIR}/testimage-sdk/") + bb.utils.remove(sdk_dir, True) + bb.utils.mkdirhier(sdk_dir) + try: + subprocess.check_output("cd %s; %s <<EOF\n./\nY\nEOF" % (sdk_dir, tcname), shell=True) + except subprocess.CalledProcessError as e: + bb.fatal("Couldn't install the SDK:\n%s" % e.output.decode("utf-8")) + + fail = False + sdk_envs = OESDKTestContextExecutor._get_sdk_environs(sdk_dir) + for s in sdk_envs: + sdk_env = sdk_envs[s] + bb.plain("SDK testing environment: %s" % s) + tc = OESDKTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir, + sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest, + host_pkg_manifest=host_pkg_manifest) + + try: + tc.loadTests(OESDKTestContextExecutor.default_cases) + except Exception as e: + import traceback + bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) + + result = tc.runTests() + + component = "%s %s" % (pn, OESDKTestContextExecutor.name) + context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) + + tc.logSummary(result, component, context_msg) + tc.logDetails() + + if not result.wasSuccessful(): + fail = True + + if fail: + bb.fatal("%s - FAILED - check the task log and the commands log" % pn) + +testsdk_main[vardepsexclude] =+ "BB_ORIGENV" + +python do_testsdk() { + testsdk_main(d) +} +addtask testsdk +do_testsdk[nostamp] = "1" + +def testsdkext_main(d): + import os + import json + import subprocess + import logging + + from bb.utils import export_proxies + from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak + from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor + + pn = d.getVar("PN") + logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) + + # extensible sdk use network + export_proxies(d) + + subprocesstweak.errors_have_output() + + # extensible sdk can be contaminated if native programs are + # in PATH, i.e. use perl-native instead of eSDK one. + paths_to_avoid = [d.getVar('STAGING_DIR'), + d.getVar('BASE_WORKDIR')] + os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid) + + tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh") + if not os.path.exists(tcname): + bb.fatal("The toolchain ext %s is not built. Build it before running the" \ + " tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname) + + tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json") + test_data = json.load(open(tdname, "r")) + + target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest( + d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest")) + host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest( + d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest")) + + sdk_dir = d.expand("${WORKDIR}/testsdkext/") + bb.utils.remove(sdk_dir, True) + bb.utils.mkdirhier(sdk_dir) + try: + subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True) + except subprocess.CalledProcessError as e: + msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8") + logfn = os.path.join(sdk_dir, 'preparing_build_system.log') + if os.path.exists(logfn): + msg += '\n\nContents of preparing_build_system.log:\n' + with open(logfn, 'r') as f: + for line in f: + msg += line + bb.fatal(msg) + + fail = False + sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir) + for s in sdk_envs: + bb.plain("Extensible SDK testing environment: %s" % s) + + sdk_env = sdk_envs[s] + + # Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache + # and we don't spend hours downloading kernels for the kernel module test + # Abuse auto.conf since local.conf would be overwritten by the SDK + with open(os.path.join(sdk_dir, 'conf', 'auto.conf'), 'a+') as f: + f.write('SSTATE_MIRRORS += " \\n file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR')) + f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR')) + f.write('INHERIT += "own-mirrors"') + + # We need to do this in case we have a minimal SDK + subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % sdk_env, cwd=sdk_dir, shell=True) + + tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir, + sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest, + host_pkg_manifest=host_pkg_manifest) + + try: + tc.loadTests(OESDKExtTestContextExecutor.default_cases) + except Exception as e: + import traceback + bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) + + result = tc.runTests() + + component = "%s %s" % (pn, OESDKExtTestContextExecutor.name) + context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) + + tc.logSummary(result, component, context_msg) + tc.logDetails() + + if not result.wasSuccessful(): + fail = True + + if fail: + bb.fatal("%s - FAILED - check the task log and the commands log" % pn) + +testsdkext_main[vardepsexclude] =+ "BB_ORIGENV" + +python do_testsdkext() { + testsdkext_main(d) +} +addtask testsdkext +do_testsdkext[nostamp] = "1" + diff --git a/meta/classes/texinfo.bbclass b/meta/classes/texinfo.bbclass index 92efbccddf..6b0def0eac 100644 --- a/meta/classes/texinfo.bbclass +++ b/meta/classes/texinfo.bbclass @@ -1,10 +1,10 @@ # This class is inherited by recipes whose upstream packages invoke the # texinfo utilities at build-time. Native and cross recipes are made to use the -# dummy scripts provided by texinfo-dummy-native, for improved performance. -# Target architecture recipes use the genuine Texinfo utilities. By default, +# dummy scripts provided by texinfo-dummy-native, for improved performance. +# Target architecture recipes use the genuine Texinfo utilities. By default, # they use the Texinfo utilities on the host system. If you want to use the -# Texinfo recipe shipped with yoco, you can remove texinfo-native from -# ASSUME_PROVIDED and makeinfo from SANITY_REQUIRED_UTILITIES. +# Texinfo recipe, you can remove texinfo-native from ASSUME_PROVIDED and +# makeinfo from SANITY_REQUIRED_UTILITIES. TEXDEP = "texinfo-native" TEXDEP_class-native = "texinfo-dummy-native" @@ -13,3 +13,6 @@ DEPENDS_append = " ${TEXDEP}" PATH_prepend_class-native = "${STAGING_BINDIR_NATIVE}/texinfo-dummy-native:" PATH_prepend_class-cross = "${STAGING_BINDIR_NATIVE}/texinfo-dummy-native:" +# libtool-cross doesn't inherit cross +TEXDEP_pn-libtool-cross = "texinfo-dummy-native" +PATH_prepend_pn-libtool-cross = "${STAGING_BINDIR_NATIVE}/texinfo-dummy-native:" diff --git a/meta/classes/tinderclient.bbclass b/meta/classes/tinderclient.bbclass index 6984efd1be..00f453cec1 100644 --- a/meta/classes/tinderclient.bbclass +++ b/meta/classes/tinderclient.bbclass @@ -10,10 +10,10 @@ def tinder_http_post(server, selector, content_type, body): h.endheaders() h.send(body) errcode, errmsg, headers = h.getreply() - #print errcode, errmsg, headers + #print(errcode, errmsg, headers) return (errcode,errmsg, headers, h.file) except: - print "Error sending the report!" + print("Error sending the report!") # try again pass @@ -55,22 +55,22 @@ def tinder_format_http_post(d,status,log): # the variables we will need to send on this form post variables = { - "tree" : d.getVar('TINDER_TREE', True), - "machine_name" : d.getVar('TINDER_MACHINE', True), + "tree" : d.getVar('TINDER_TREE'), + "machine_name" : d.getVar('TINDER_MACHINE'), "os" : os.uname()[0], "os_version" : os.uname()[2], "compiler" : "gcc", - "clobber" : d.getVar('TINDER_CLOBBER', True) or "0", - "srcdate" : d.getVar('SRCDATE', True), - "PN" : d.getVar('PN', True), - "PV" : d.getVar('PV', True), - "PR" : d.getVar('PR', True), - "FILE" : d.getVar('FILE', True) or "N/A", - "TARGETARCH" : d.getVar('TARGET_ARCH', True), - "TARGETFPU" : d.getVar('TARGET_FPU', True) or "Unknown", - "TARGETOS" : d.getVar('TARGET_OS', True) or "Unknown", - "MACHINE" : d.getVar('MACHINE', True) or "Unknown", - "DISTRO" : d.getVar('DISTRO', True) or "Unknown", + "clobber" : d.getVar('TINDER_CLOBBER') or "0", + "srcdate" : d.getVar('SRCDATE'), + "PN" : d.getVar('PN'), + "PV" : d.getVar('PV'), + "PR" : d.getVar('PR'), + "FILE" : d.getVar('FILE') or "N/A", + "TARGETARCH" : d.getVar('TARGET_ARCH'), + "TARGETFPU" : d.getVar('TARGET_FPU') or "Unknown", + "TARGETOS" : d.getVar('TARGET_OS') or "Unknown", + "MACHINE" : d.getVar('MACHINE') or "Unknown", + "DISTRO" : d.getVar('DISTRO') or "Unknown", "zecke-rocks" : "sure", } @@ -82,7 +82,7 @@ def tinder_format_http_post(d,status,log): # we only need on build_status.pl but sending it # always does not hurt try: - f = file(d.getVar('TMPDIR',True)+'/tinder-machine.id', 'r') + f = open(d.getVar('TMPDIR')+'/tinder-machine.id', 'r') id = f.read() variables['machine_id'] = id except: @@ -106,16 +106,16 @@ def tinder_build_start(d): # get the body and type content_type, body = tinder_format_http_post(d,None,None) - server = d.getVar('TINDER_HOST', True ) - url = d.getVar('TINDER_URL', True ) + server = d.getVar('TINDER_HOST') + url = d.getVar('TINDER_URL') selector = url + "/xml/build_start.pl" - #print "selector %s and url %s" % (selector, url) + #print("selector %s and url %s" % (selector, url)) # now post it errcode, errmsg, headers, h_file = tinder_http_post(server,selector,content_type, body) - #print errcode, errmsg, headers + #print(errcode, errmsg, headers) report = h_file.read() # now let us find the machine id that was assigned to us @@ -127,7 +127,7 @@ def tinder_build_start(d): # now we will need to save the machine number # we will override any previous numbers - f = file(d.getVar('TMPDIR', True)+"/tinder-machine.id", 'w') + f = open(d.getVar('TMPDIR')+"/tinder-machine.id", 'w') f.write(report) @@ -137,18 +137,18 @@ def tinder_send_http(d, status, _log): """ # get the body and type - server = d.getVar('TINDER_HOST', True) - url = d.getVar('TINDER_URL', True) + server = d.getVar('TINDER_HOST') + url = d.getVar('TINDER_URL') selector = url + "/xml/build_status.pl" - # now post it - in chunks of 10.000 charachters + # now post it - in chunks of 10.000 characters new_log = _log while len(new_log) > 0: content_type, body = tinder_format_http_post(d,status,new_log[0:18000]) errcode, errmsg, headers, h_file = tinder_http_post(server,selector,content_type, body) - #print errcode, errmsg, headers - #print h.file.read() + #print(errcode, errmsg, headers) + #print(h.file.read()) new_log = new_log[18000:] @@ -163,16 +163,16 @@ def tinder_print_info(d): time = tinder_time_string() ops = os.uname()[0] version = os.uname()[2] - url = d.getVar( 'TINDER_URL' , True ) - tree = d.getVar( 'TINDER_TREE', True ) - branch = d.getVar( 'TINDER_BRANCH', True ) - srcdate = d.getVar( 'SRCDATE', True ) - machine = d.getVar( 'MACHINE', True ) - distro = d.getVar( 'DISTRO', True ) - bbfiles = d.getVar( 'BBFILES', True ) - tarch = d.getVar( 'TARGET_ARCH', True ) - fpu = d.getVar( 'TARGET_FPU', True ) - oerev = d.getVar( 'OE_REVISION', True ) or "unknown" + url = d.getVar('TINDER_URL') + tree = d.getVar('TINDER_TREE') + branch = d.getVar('TINDER_BRANCH') + srcdate = d.getVar('SRCDATE') + machine = d.getVar('MACHINE') + distro = d.getVar('DISTRO') + bbfiles = d.getVar('BBFILES') + tarch = d.getVar('TARGET_ARCH') + fpu = d.getVar('TARGET_FPU') + oerev = d.getVar('OE_REVISION') or "unknown" # there is a bug with tipple quoted strings # i will work around but will fix the original @@ -278,7 +278,7 @@ def tinder_do_tinder_report(event): try: # truncate the tinder log file - f = file(event.data.getVar('TINDER_LOG', True), 'w') + f = open(event.data.getVar('TINDER_LOG'), 'w') f.write("") f.close() except: @@ -287,7 +287,7 @@ def tinder_do_tinder_report(event): try: # write a status to the file. This is needed for the -k option # of BitBake - g = file(event.data.getVar('TMPDIR', True)+"/tinder-status", 'w') + g = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w') g.write("") g.close() except IOError: @@ -296,10 +296,10 @@ def tinder_do_tinder_report(event): # Append the Task-Log (compile,configure...) to the log file # we will send to the server if name == "TaskSucceeded" or name == "TaskFailed": - log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T', True), event.task)) + log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T'), event.task)) if len(log_file) != 0: - to_file = event.data.getVar('TINDER_LOG', True) + to_file = event.data.getVar('TINDER_LOG') log += "".join(open(log_file[0], 'r').readlines()) # set the right 'HEADER'/Summary for the TinderBox @@ -310,23 +310,23 @@ def tinder_do_tinder_report(event): elif name == "TaskFailed": log += "<--- TINDERBOX Task %s failed (FAILURE)\n" % event.task elif name == "PkgStarted": - log += "---> TINDERBOX Package %s started\n" % event.data.getVar('PF', True) + log += "---> TINDERBOX Package %s started\n" % event.data.getVar('PF') elif name == "PkgSucceeded": - log += "<--- TINDERBOX Package %s done (SUCCESS)\n" % event.data.getVar('PF', True) + log += "<--- TINDERBOX Package %s done (SUCCESS)\n" % event.data.getVar('PF') elif name == "PkgFailed": - if not event.data.getVar('TINDER_AUTOBUILD', True) == "0": + if not event.data.getVar('TINDER_AUTOBUILD') == "0": build.exec_task('do_clean', event.data) - log += "<--- TINDERBOX Package %s failed (FAILURE)\n" % event.data.getVar('PF', True) + log += "<--- TINDERBOX Package %s failed (FAILURE)\n" % event.data.getVar('PF') status = 200 # remember the failure for the -k case - h = file(event.data.getVar('TMPDIR', True)+"/tinder-status", 'w') + h = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w') h.write("200") elif name == "BuildCompleted": log += "Build Completed\n" status = 100 # Check if we have a old status... try: - h = file(event.data.getVar('TMPDIR',True)+'/tinder-status', 'r') + h = open(event.data.getVar('TMPDIR')+'/tinder-status', 'r') status = int(h.read()) except: pass @@ -342,7 +342,7 @@ def tinder_do_tinder_report(event): log += "Error:Was Runtime: %d\n" % event.isRuntime() status = 200 # remember the failure for the -k case - h = file(event.data.getVar('TMPDIR', True)+"/tinder-status", 'w') + h = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w') h.write("200") # now post the log @@ -360,7 +360,7 @@ python tinderclient_eventhandler() { if e.data is None or bb.event.getName(e) == "MsgNote": return - do_tinder_report = e.data.getVar('TINDER_REPORT', True) + do_tinder_report = e.data.getVar('TINDER_REPORT') if do_tinder_report and do_tinder_report == "1": tinder_do_tinder_report(e) diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass index d63cff532e..296e4764f0 100644 --- a/meta/classes/toaster.bbclass +++ b/meta/classes/toaster.bbclass @@ -33,6 +33,7 @@ python toaster_layerinfo_dumpdata() { def _get_git_branch(layer_path): branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] + branch = branch.decode('utf-8') branch = branch.replace('refs/heads/', '').rstrip() return branch @@ -79,7 +80,7 @@ python toaster_layerinfo_dumpdata() { return layer_info - bblayers = e.data.getVar("BBLAYERS", True) + bblayers = e.data.getVar("BBLAYERS") llayerinfo = {} @@ -112,67 +113,41 @@ def _toaster_load_pkgdatafile(dirpath, filepath): pass # ignore lines without valid key: value pairs return pkgdata - python toaster_package_dumpdata() { """ - Dumps the data created by emit_pkgdata + Dumps the data about the packages created by a recipe """ - # replicate variables from the package.bbclass - - packages = d.getVar('PACKAGES', True) - pkgdest = d.getVar('PKGDEST', True) - pkgdatadir = d.getVar('PKGDESTWORK', True) - - # scan and send data for each package + # No need to try and dumpdata if the recipe isn't generating packages + if not d.getVar('PACKAGES'): + return + pkgdatadir = d.getVar('PKGDESTWORK') lpkgdata = {} - for pkg in packages.split(): - - lpkgdata = _toaster_load_pkgdatafile(pkgdatadir + "/runtime/", pkg) - - # Fire an event containing the pkg data - bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) + datadir = os.path.join(pkgdatadir, 'runtime') + + # scan and send data for each generated package + for datafile in os.listdir(datadir): + if not datafile.endswith('.packaged'): + lpkgdata = _toaster_load_pkgdatafile(datadir, datafile) + # Fire an event containing the pkg data + bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) } # 2. Dump output image files information -python toaster_image_dumpdata() { +python toaster_artifact_dumpdata() { """ - Image filename for output images is not standardized. - image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually - have hardcoded ways to create image file names in them. - So we look for files starting with the set name. + Dump data about SDK variables """ - deploy_dir_image = d.getVar('DEPLOY_DIR_IMAGE', True); - image_name = d.getVar('IMAGE_NAME', True); - - image_info_data = {} - artifact_info_data = {} + event_data = { + "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME") + } - # collect all artifacts - for dirpath, dirnames, filenames in os.walk(deploy_dir_image): - for fn in filenames: - try: - if fn.startswith(image_name): - image_output = os.path.join(dirpath, fn) - image_info_data[image_output] = os.stat(image_output).st_size - else: - import stat - artifact_path = os.path.join(dirpath, fn) - filestat = os.stat(artifact_path) - if not os.path.islink(artifact_path): - artifact_info_data[artifact_path] = filestat.st_size - except OSError as e: - bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) - - bb.event.fire(bb.event.MetadataEvent("ImageFileSize",image_info_data), d) - bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize",artifact_info_data), d) + bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d) } - - # collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data python toaster_collect_task_stats() { @@ -182,29 +157,37 @@ python toaster_collect_task_stats() { import bb.utils import os - if not e.data.getVar('BUILDSTATS_BASE', True): + if not e.data.getVar('BUILDSTATS_BASE'): return # if we don't have buildstats, we cannot collect stats + toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE'), "toasterstatlist") + + def stat_to_float(value): + return float(value.strip('% \n\r')) + def _append_read_list(v): lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True) - with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout: - bn = get_bn(e) - bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) - taskdir = os.path.join(bsdir, e.data.expand("${PF}")) + with open(toaster_statlist_file, "a") as fout: + taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}") fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}"))) bb.utils.unlockfile(lock) def _read_stats(filename): - cpu_usage = 0 - disk_io = 0 - startio = '0' - endio = '0' - started = '0' - ended = '0' - pn = '' + # seconds + cpu_time_user = 0 + cpu_time_system = 0 + + # bytes + disk_io_read = 0 + disk_io_write = 0 + + started = 0 + ended = 0 + taskname = '' + statinfo = {} with open(filename, 'r') as task_bs: @@ -212,43 +195,49 @@ python toaster_collect_task_stats() { k,v = line.strip().split(": ", 1) statinfo[k] = v - if "CPU usage" in statinfo: - cpu_usage = str(statinfo["CPU usage"]).strip('% \n\r') - - if "EndTimeIO" in statinfo: - endio = str(statinfo["EndTimeIO"]).strip('% \n\r') - - if "StartTimeIO" in statinfo: - startio = str(statinfo["StartTimeIO"]).strip('% \n\r') - if "Started" in statinfo: - started = str(statinfo["Started"]).strip('% \n\r') + started = stat_to_float(statinfo["Started"]) if "Ended" in statinfo: - ended = str(statinfo["Ended"]).strip('% \n\r') + ended = stat_to_float(statinfo["Ended"]) - disk_io = int(endio) - int(startio) + if "Child rusage ru_utime" in statinfo: + cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"]) - elapsed_time = float(ended) - float(started) + if "Child rusage ru_stime" in statinfo: + cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"]) - cpu_usage = float(cpu_usage) + if "IO write_bytes" in statinfo: + write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r')) + disk_io_write = disk_io_write + write_bytes - return {'cpu_usage': cpu_usage, 'disk_io': disk_io, 'elapsed_time': elapsed_time} + if "IO read_bytes" in statinfo: + read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r')) + disk_io_read = disk_io_read + read_bytes + return { + 'stat_file': filename, + 'cpu_time_user': cpu_time_user, + 'cpu_time_system': cpu_time_system, + 'disk_io_read': disk_io_read, + 'disk_io_write': disk_io_write, + 'started': started, + 'ended': ended + } if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): _append_read_list(e) pass - - if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")): + if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file): events = [] - with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin: + with open(toaster_statlist_file, "r") as fin: for line in fin: (taskfile, taskname, filename, recipename) = line.strip().split("::") - events.append((taskfile, taskname, _read_stats(filename), recipename)) + stats = _read_stats(filename) + events.append((taskfile, taskname, stats, recipename)) bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data) - os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")) + os.unlink(toaster_statlist_file) } # dump relevant build history data as an event when the build is completed @@ -257,7 +246,7 @@ python toaster_buildhistory_dump() { import re BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory") BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR) - pkgdata_dir = e.data.getVar("PKGDATA_DIR", True) + pkgdata_dir = e.data.getVar("PKGDATA_DIR") # scan the build targets for this build @@ -265,6 +254,7 @@ python toaster_buildhistory_dump() { allpkgs = {} files = {} for target in e._pkgs: + target = target.split(':')[0] # strip ':<task>' suffix from the target installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target)) if os.path.exists(installed_img_path): images[target] = {} @@ -275,8 +265,7 @@ python toaster_buildhistory_dump() { with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin: for line in fin: line = line.rstrip(";") - psize, px = line.split("\t") - punit, pname = px.split(" ") + psize, punit, pname = line.split() # this size is "installed-size" as it measures how much space it takes on disk images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} @@ -298,15 +287,22 @@ python toaster_buildhistory_dump() { images[target][dependsname] = {'size': 0, 'depends' : []} images[target][pname]['depends'].append((dependsname, deptype)) - with open("%s/files-in-image.txt" % installed_img_path, "r") as fin: - for line in fin: - lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] - if lc[0].startswith("l"): - files[target]['syms'].append(lc) - elif lc[0].startswith("d"): - files[target]['dirs'].append(lc) - else: - files[target]['files'].append(lc) + # files-in-image.txt is only generated if an image file is created, + # so the file entries ('syms', 'dirs', 'files') for a target will be + # empty for rootfs builds and other "image" tasks which don't + # produce image files + # (e.g. "bitbake core-image-minimal -c populate_sdk") + files_in_image_path = "%s/files-in-image.txt" % installed_img_path + if os.path.exists(files_in_image_path): + with open(files_in_image_path, "r") as fin: + for line in fin: + lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] + if lc[0].startswith("l"): + files[target]['syms'].append(lc) + elif lc[0].startswith("d"): + files[target]['dirs'].append(lc) + else: + files[target]['files'].append(lc) for pname in images[target]: if not pname in allpkgs: @@ -327,15 +323,28 @@ python toaster_buildhistory_dump() { } -# dump information related to license manifest path - -python toaster_licensemanifest_dump() { - deploy_dir = d.getVar('DEPLOY_DIR', True); - image_name = d.getVar('IMAGE_NAME', True); - - data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name } - - bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d) +# get list of artifacts from sstate manifest +python toaster_artifacts() { + if e.taskname in ["do_deploy", "do_image_complete", "do_populate_sdk", "do_populate_sdk_ext"]: + d2 = d.createCopy() + d2.setVar('FILE', e.taskfile) + # Use 'stamp-extra-info' if present, else use workaround + # to determine 'SSTATE_MANMACH' + extrainf = d2.getVarFlag(e.taskname, 'stamp-extra-info') + if extrainf: + d2.setVar('SSTATE_MANMACH', extrainf) + else: + if "do_populate_sdk" == e.taskname: + d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}${SDKMACHINE}")) + else: + d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}")) + manifest = oe.sstatesig.sstate_get_manifest_filename(e.taskname[3:], d2)[0] + + if os.access(manifest, os.R_OK): + with open(manifest) as fmanifest: + artifacts = [fname.strip() for fname in fmanifest] + data = {"task": e.taskid, "artifacts": artifacts} + bb.event.fire(bb.event.MetadataEvent("TaskArtifacts", data), d2) } # set event handlers @@ -347,9 +356,19 @@ toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSu addhandler toaster_buildhistory_dump toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" + +addhandler toaster_artifacts +toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted" + +do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata " +do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata " + do_package[postfuncs] += "toaster_package_dumpdata " do_package[vardepsexclude] += "toaster_package_dumpdata " -do_rootfs[postfuncs] += "toaster_image_dumpdata " -do_rootfs[postfuncs] += "toaster_licensemanifest_dump " -do_rootfs[vardepsexclude] += "toaster_image_dumpdata toaster_licensemanifest_dump" +#do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata " +#do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata " + +#do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata " +#do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata " + diff --git a/meta/classes/toolchain-scripts-base.bbclass b/meta/classes/toolchain-scripts-base.bbclass new file mode 100644 index 0000000000..2489b9dbeb --- /dev/null +++ b/meta/classes/toolchain-scripts-base.bbclass @@ -0,0 +1,11 @@ +#This function create a version information file +toolchain_create_sdk_version () { + local versionfile=$1 + rm -f $versionfile + touch $versionfile + echo 'Distro: ${DISTRO}' >> $versionfile + echo 'Distro Version: ${DISTRO_VERSION}' >> $versionfile + echo 'Metadata Revision: ${METADATA_REVISION}' >> $versionfile + echo 'Timestamp: ${DATETIME}' >> $versionfile +} +toolchain_create_sdk_version[vardepsexclude] = "DATETIME" diff --git a/meta/classes/toolchain-scripts.bbclass b/meta/classes/toolchain-scripts.bbclass index d0b2b9148c..260ece9676 100644 --- a/meta/classes/toolchain-scripts.bbclass +++ b/meta/classes/toolchain-scripts.bbclass @@ -1,4 +1,4 @@ -inherit siteinfo kernel-arch +inherit toolchain-scripts-base siteinfo kernel-arch # We want to be able to change the value of MULTIMACH_TARGET_SYS, because it # doesn't always match our expectations... but we default to the stock value @@ -6,9 +6,16 @@ REAL_MULTIMACH_TARGET_SYS ?= "${MULTIMACH_TARGET_SYS}" TARGET_CC_ARCH_append_libc-uclibc = " -muclibc" TARGET_CC_ARCH_append_libc-musl = " -mmusl" +# default debug prefix map isn't valid in the SDK +DEBUG_PREFIX_MAP = "" + # This function creates an environment-setup-script for use in a deployable SDK toolchain_create_sdk_env_script () { - # Create environment setup script + # Create environment setup script. Remember that $SDKTARGETSYSROOT should + # only be expanded on the target at runtime. + base_sbindir=${10:-${base_sbindir_nativesdk}} + base_bindir=${9:-${base_bindir_nativesdk}} + sbindir=${8:-${sbindir_nativesdk}} sdkpathnative=${7:-${SDKPATHNATIVE}} prefix=${6:-${prefix_nativesdk}} bindir=${5:-${bindir_nativesdk}} @@ -23,15 +30,14 @@ toolchain_create_sdk_env_script () { for i in ${CANADIANEXTRAOS}; do EXTRAPATH="$EXTRAPATH:$sdkpathnative$bindir/${TARGET_ARCH}${TARGET_VENDOR}-$i" done - echo "export PATH=$sdkpathnative$bindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':$PATH' >> $script - echo "export CCACHE_PATH=$sdkpathnative$bindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':$CCACHE_PATH' >> $script + echo "export PATH=$sdkpathnative$bindir:$sdkpathnative$sbindir:$sdkpathnative$base_bindir:$sdkpathnative$base_sbindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':$PATH' >> $script echo 'export PKG_CONFIG_SYSROOT_DIR=$SDKTARGETSYSROOT' >> $script - echo 'export PKG_CONFIG_PATH=$SDKTARGETSYSROOT'"$libdir"'/pkgconfig' >> $script + echo 'export PKG_CONFIG_PATH=$SDKTARGETSYSROOT'"$libdir"'/pkgconfig:$SDKTARGETSYSROOT'"$prefix"'/share/pkgconfig' >> $script echo 'export CONFIG_SITE=${SDKPATH}/site-config-'"${multimach_target_sys}" >> $script echo "export OECORE_NATIVE_SYSROOT=\"$sdkpathnative\"" >> $script echo 'export OECORE_TARGET_SYSROOT="$SDKTARGETSYSROOT"' >> $script echo "export OECORE_ACLOCAL_OPTS=\"-I $sdkpathnative/usr/share/aclocal\"" >> $script - echo "export PYTHONHOME=$sdkpathnative$prefix" >> $script + echo 'unset command_not_found_handle' >> $script toolchain_shared_env_script } @@ -43,7 +49,6 @@ toolchain_create_tree_env_script () { rm -f $script touch $script echo 'export PATH=${STAGING_DIR_NATIVE}/usr/bin:${PATH}' >> $script - echo 'export CCACHE_PATH=${STAGING_DIR_NATIVE}/usr/bin:${CCACHE_PATH}' >> $script echo 'export PKG_CONFIG_SYSROOT_DIR=${PKG_CONFIG_SYSROOT_DIR}' >> $script echo 'export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}' >> $script echo 'export CONFIG_SITE="${@siteinfo_get_files(d)}"' >> $script @@ -86,21 +91,22 @@ toolchain_shared_env_script () { # Append environment subscripts if [ -d "\$OECORE_TARGET_SYSROOT/environment-setup.d" ]; then for envfile in \$OECORE_TARGET_SYSROOT/environment-setup.d/*.sh; do - source \$envfile + . \$envfile done fi if [ -d "\$OECORE_NATIVE_SYSROOT/environment-setup.d" ]; then for envfile in \$OECORE_NATIVE_SYSROOT/environment-setup.d/*.sh; do - source \$envfile + . \$envfile done fi EOF } #we get the cached site config in the runtime -TOOLCHAIN_CONFIGSITE_NOCACHE = "${@siteinfo_get_files(d, True)}" +TOOLCHAIN_CONFIGSITE_NOCACHE = "${@siteinfo_get_files(d)}" TOOLCHAIN_CONFIGSITE_SYSROOTCACHE = "${STAGING_DIR}/${MLPREFIX}${MACHINE}/${target_datadir}/${TARGET_SYS}_config_site.d" TOOLCHAIN_NEED_CONFIGSITE_CACHE ??= "virtual/${MLPREFIX}libc ncurses" +DEPENDS += "${TOOLCHAIN_NEED_CONFIGSITE_CACHE}" #This function create a site config file toolchain_create_sdk_siteconfig () { @@ -129,24 +135,12 @@ toolchain_create_sdk_siteconfig () { # The immediate expansion above can result in unwanted path dependencies here toolchain_create_sdk_siteconfig[vardepsexclude] = "TOOLCHAIN_CONFIGSITE_SYSROOTCACHE" -#This function create a version information file -toolchain_create_sdk_version () { - local versionfile=$1 - rm -f $versionfile - touch $versionfile - echo 'Distro: ${DISTRO}' >> $versionfile - echo 'Distro Version: ${DISTRO_VERSION}' >> $versionfile - echo 'Metadata Revision: ${METADATA_REVISION}' >> $versionfile - echo 'Timestamp: ${DATETIME}' >> $versionfile -} -toolchain_create_sdk_version[vardepsexclude] = "DATETIME" - python __anonymous () { import oe.classextend deps = "" - for dep in (d.getVar('TOOLCHAIN_NEED_CONFIGSITE_CACHE', True) or "").split(): + for dep in (d.getVar('TOOLCHAIN_NEED_CONFIGSITE_CACHE') or "").split(): deps += " %s:do_populate_sysroot" % dep - for variant in (d.getVar('MULTILIB_VARIANTS', True) or "").split(): + for variant in (d.getVar('MULTILIB_VARIANTS') or "").split(): clsextend = oe.classextend.ClassExtender(variant, d) newdep = clsextend.extend_name(dep) deps += " %s:do_populate_sysroot" % newdep diff --git a/meta/classes/uboot-config.bbclass b/meta/classes/uboot-config.bbclass index cb061af348..10013b7d49 100644 --- a/meta/classes/uboot-config.bbclass +++ b/meta/classes/uboot-config.bbclass @@ -3,7 +3,7 @@ # The format to specify it, in the machine, is: # # UBOOT_CONFIG ??= <default> -# UBOOT_CONFIG[foo] = "config,images" +# UBOOT_CONFIG[foo] = "config,images,binary" # # or # @@ -11,38 +11,51 @@ # # Copyright 2013, 2014 (C) O.S. Systems Software LTDA. +UBOOT_BINARY ?= "u-boot.${UBOOT_SUFFIX}" + python () { - ubootmachine = d.getVar("UBOOT_MACHINE", True) + ubootmachine = d.getVar("UBOOT_MACHINE") ubootconfigflags = d.getVarFlags('UBOOT_CONFIG') + ubootbinary = d.getVar('UBOOT_BINARY') + ubootbinaries = d.getVar('UBOOT_BINARIES') # The "doc" varflag is special, we don't want to see it here ubootconfigflags.pop('doc', None) if not ubootmachine and not ubootconfigflags: - PN = d.getVar("PN", True) - FILE = os.path.basename(d.getVar("FILE", True)) + PN = d.getVar("PN") + FILE = os.path.basename(d.getVar("FILE")) bb.debug(1, "To build %s, see %s for instructions on \ setting up your machine config" % (PN, FILE)) - raise bb.parse.SkipPackage("Either UBOOT_MACHINE or UBOOT_CONFIG must be set in the %s machine configuration." % d.getVar("MACHINE", True)) + raise bb.parse.SkipPackage("Either UBOOT_MACHINE or UBOOT_CONFIG must be set in the %s machine configuration." % d.getVar("MACHINE")) if ubootmachine and ubootconfigflags: raise bb.parse.SkipPackage("You cannot use UBOOT_MACHINE and UBOOT_CONFIG at the same time.") + if ubootconfigflags and ubootbinaries: + raise bb.parse.SkipPackage("You cannot use UBOOT_BINARIES as it is internal to uboot_config.bbclass.") + if not ubootconfigflags: return - ubootconfig = (d.getVar('UBOOT_CONFIG', True) or "").split() + ubootconfig = (d.getVar('UBOOT_CONFIG') or "").split() if len(ubootconfig) > 0: for config in ubootconfig: for f, v in ubootconfigflags.items(): if config == f: items = v.split(',') - if items[0] and len(items) > 2: - raise bb.parse.SkipPackage('Only config,images can be specified!') + if items[0] and len(items) > 3: + raise bb.parse.SkipPackage('Only config,images,binary can be specified!') d.appendVar('UBOOT_MACHINE', ' ' + items[0]) # IMAGE_FSTYPES appending if len(items) > 1 and items[1]: bb.debug(1, "Appending '%s' to IMAGE_FSTYPES." % items[1]) d.appendVar('IMAGE_FSTYPES', ' ' + items[1]) + if len(items) > 2 and items[2]: + bb.debug(1, "Appending '%s' to UBOOT_BINARIES." % items[2]) + d.appendVar('UBOOT_BINARIES', ' ' + items[2]) + else: + bb.debug(1, "Appending '%s' to UBOOT_BINARIES." % ubootbinary) + d.appendVar('UBOOT_BINARIES', ' ' + ubootbinary) break elif len(ubootconfig) == 0: raise bb.parse.SkipPackage('You must set a default in UBOOT_CONFIG.') diff --git a/meta/classes/uboot-extlinux-config.bbclass b/meta/classes/uboot-extlinux-config.bbclass new file mode 100644 index 0000000000..8447a047ee --- /dev/null +++ b/meta/classes/uboot-extlinux-config.bbclass @@ -0,0 +1,152 @@ +# uboot-extlinux-config.bbclass +# +# This class allow the extlinux.conf generation for U-Boot use. The +# U-Boot support for it is given to allow the Generic Distribution +# Configuration specification use by OpenEmbedded-based products. +# +# External variables: +# +# UBOOT_EXTLINUX_CONSOLE - Set to "console=ttyX" to change kernel boot +# default console. +# UBOOT_EXTLINUX_LABELS - A list of targets for the automatic config. +# UBOOT_EXTLINUX_KERNEL_ARGS - Add additional kernel arguments. +# UBOOT_EXTLINUX_KERNEL_IMAGE - Kernel image name. +# UBOOT_EXTLINUX_FDTDIR - Device tree directory. +# UBOOT_EXTLINUX_FDT - Device tree file. +# UBOOT_EXTLINUX_INITRD - Indicates a list of filesystem images to +# concatenate and use as an initrd (optional). +# UBOOT_EXTLINUX_MENU_DESCRIPTION - Name to use as description. +# UBOOT_EXTLINUX_ROOT - Root kernel cmdline. +# UBOOT_EXTLINUX_TIMEOUT - Timeout before DEFAULT selection is made. +# Measured in 1/10 of a second. +# UBOOT_EXTLINUX_DEFAULT_LABEL - Target to be selected by default after +# the timeout period +# +# If there's only one label system will boot automatically and menu won't be +# created. If you want to use more than one labels, e.g linux and alternate, +# use overrides to set menu description, console and others variables. +# +# Ex: +# +# UBOOT_EXTLINUX_LABELS ??= "default fallback" +# +# UBOOT_EXTLINUX_DEFAULT_LABEL ??= "Linux Default" +# UBOOT_EXTLINUX_TIMEOUT ??= "30" +# +# UBOOT_EXTLINUX_KERNEL_IMAGE_default ??= "../zImage" +# UBOOT_EXTLINUX_MENU_DESCRIPTION_default ??= "Linux Default" +# +# UBOOT_EXTLINUX_KERNEL_IMAGE_fallback ??= "../zImage-fallback" +# UBOOT_EXTLINUX_MENU_DESCRIPTION_fallback ??= "Linux Fallback" +# +# Results: +# +# menu title Select the boot mode +# TIMEOUT 30 +# DEFAULT Linux Default +# LABEL Linux Default +# KERNEL ../zImage +# FDTDIR ../ +# APPEND root=/dev/mmcblk2p2 rootwait rw console=${console} +# LABEL Linux Fallback +# KERNEL ../zImage-fallback +# FDTDIR ../ +# APPEND root=/dev/mmcblk2p2 rootwait rw console=${console} +# +# Copyright (C) 2016, O.S. Systems Software LTDA. All Rights Reserved +# Released under the MIT license (see packages/COPYING) +# +# The kernel has an internal default console, which you can override with +# a console=...some_tty... +UBOOT_EXTLINUX_CONSOLE ??= "console=${console}" +UBOOT_EXTLINUX_LABELS ??= "linux" +UBOOT_EXTLINUX_FDT ??= "" +UBOOT_EXTLINUX_FDTDIR ??= "../" +UBOOT_EXTLINUX_KERNEL_IMAGE ??= "../${KERNEL_IMAGETYPE}" +UBOOT_EXTLINUX_KERNEL_ARGS ??= "rootwait rw" +UBOOT_EXTLINUX_MENU_DESCRIPTION_linux ??= "${DISTRO_NAME}" + +UBOOT_EXTLINUX_CONFIG = "${B}/extlinux.conf" + +python create_extlinux_config() { + if d.getVar("UBOOT_EXTLINUX") != "1": + return + + if not d.getVar('WORKDIR'): + bb.error("WORKDIR not defined, unable to package") + + labels = d.getVar('UBOOT_EXTLINUX_LABELS') + if not labels: + bb.fatal("UBOOT_EXTLINUX_LABELS not defined, nothing to do") + + if not labels.strip(): + bb.fatal("No labels, nothing to do") + + cfile = d.getVar('UBOOT_EXTLINUX_CONFIG') + if not cfile: + bb.fatal('Unable to read UBOOT_EXTLINUX_CONFIG') + + localdata = bb.data.createCopy(d) + + try: + with open(cfile, 'w') as cfgfile: + cfgfile.write('# Generic Distro Configuration file generated by OpenEmbedded\n') + + if len(labels.split()) > 1: + cfgfile.write('menu title Select the boot mode\n') + + timeout = localdata.getVar('UBOOT_EXTLINUX_TIMEOUT') + if timeout: + cfgfile.write('TIMEOUT %s\n' % (timeout)) + + if len(labels.split()) > 1: + default = localdata.getVar('UBOOT_EXTLINUX_DEFAULT_LABEL') + if default: + cfgfile.write('DEFAULT %s\n' % (default)) + + for label in labels.split(): + + overrides = localdata.getVar('OVERRIDES') + if not overrides: + bb.fatal('OVERRIDES not defined') + + localdata.setVar('OVERRIDES', label + ':' + overrides) + + extlinux_console = localdata.getVar('UBOOT_EXTLINUX_CONSOLE') + + menu_description = localdata.getVar('UBOOT_EXTLINUX_MENU_DESCRIPTION') + if not menu_description: + menu_description = label + + root = localdata.getVar('UBOOT_EXTLINUX_ROOT') + if not root: + bb.fatal('UBOOT_EXTLINUX_ROOT not defined') + + kernel_image = localdata.getVar('UBOOT_EXTLINUX_KERNEL_IMAGE') + fdtdir = localdata.getVar('UBOOT_EXTLINUX_FDTDIR') + + fdt = localdata.getVar('UBOOT_EXTLINUX_FDT') + + if fdt: + cfgfile.write('LABEL %s\n\tKERNEL %s\n\tFDT %s\n' % + (menu_description, kernel_image, fdt)) + elif fdtdir: + cfgfile.write('LABEL %s\n\tKERNEL %s\n\tFDTDIR %s\n' % + (menu_description, kernel_image, fdtdir)) + else: + cfgfile.write('LABEL %s\n\tKERNEL %s\n' % (menu_description, kernel_image)) + + kernel_args = localdata.getVar('UBOOT_EXTLINUX_KERNEL_ARGS') + + initrd = localdata.getVar('UBOOT_EXTLINUX_INITRD') + if initrd: + cfgfile.write('\tINITRD %s\n'% initrd) + + kernel_args = root + " " + kernel_args + cfgfile.write('\tAPPEND %s %s\n' % (kernel_args, extlinux_console)) + + except OSError: + bb.fatal('Unable to open %s' % (cfile)) +} + +do_install[prefuncs] += "create_extlinux_config" diff --git a/meta/classes/uboot-sign.bbclass b/meta/classes/uboot-sign.bbclass new file mode 100644 index 0000000000..8ee904e7df --- /dev/null +++ b/meta/classes/uboot-sign.bbclass @@ -0,0 +1,95 @@ +# This file is part of U-Boot verified boot support and is intended to be +# inherited from u-boot recipe and from kernel-fitimage.bbclass. +# +# The signature procedure requires the user to generate an RSA key and +# certificate in a directory and to define the following variable: +# +# UBOOT_SIGN_KEYDIR = "/keys/directory" +# UBOOT_SIGN_KEYNAME = "dev" # keys name in keydir (eg. "dev.crt", "dev.key") +# UBOOT_MKIMAGE_DTCOPTS = "-I dts -O dtb -p 2000" +# UBOOT_SIGN_ENABLE = "1" +# +# As verified boot depends on fitImage generation, following is also required: +# +# KERNEL_CLASSES ?= " kernel-fitimage " +# KERNEL_IMAGETYPE ?= "fitImage" +# +# The signature support is limited to the use of CONFIG_OF_SEPARATE in U-Boot. +# +# The tasks sequence is set as below, using DEPLOY_IMAGE_DIR as common place to +# treat the device tree blob: +# +# u-boot:do_deploy_dtb +# u-boot:do_deploy +# virtual/kernel:do_assemble_fitimage +# u-boot:do_concat_dtb +# u-boot:do_install +# +# For more details on signature process, please refer to U-Boot documentation. + +# Signature activation. +UBOOT_SIGN_ENABLE ?= "0" + +# Default value for deployment filenames. +UBOOT_DTB_IMAGE ?= "u-boot-${MACHINE}-${PV}-${PR}.dtb" +UBOOT_DTB_BINARY ?= "u-boot.dtb" +UBOOT_DTB_SYMLINK ?= "u-boot-${MACHINE}.dtb" +UBOOT_NODTB_IMAGE ?= "u-boot-nodtb-${MACHINE}-${PV}-${PR}.${UBOOT_SUFFIX}" +UBOOT_NODTB_BINARY ?= "u-boot-nodtb.${UBOOT_SUFFIX}" +UBOOT_NODTB_SYMLINK ?= "u-boot-nodtb-${MACHINE}.${UBOOT_SUFFIX}" + +# +# Following is relevant only for u-boot recipes: +# + +do_deploy_dtb () { + mkdir -p ${DEPLOYDIR} + cd ${DEPLOYDIR} + + if [ -f ${B}/${UBOOT_DTB_BINARY} ]; then + install ${B}/${UBOOT_DTB_BINARY} ${DEPLOYDIR}/${UBOOT_DTB_IMAGE} + rm -f ${UBOOT_DTB_BINARY} ${UBOOT_DTB_SYMLINK} + ln -sf ${UBOOT_DTB_IMAGE} ${UBOOT_DTB_SYMLINK} + ln -sf ${UBOOT_DTB_IMAGE} ${UBOOT_DTB_BINARY} + fi + if [ -f ${B}/${UBOOT_NODTB_BINARY} ]; then + install ${B}/${UBOOT_NODTB_BINARY} ${DEPLOYDIR}/${UBOOT_NODTB_IMAGE} + rm -f ${UBOOT_NODTB_BINARY} ${UBOOT_NODTB_SYMLINK} + ln -sf ${UBOOT_NODTB_IMAGE} ${UBOOT_NODTB_SYMLINK} + ln -sf ${UBOOT_NODTB_IMAGE} ${UBOOT_NODTB_BINARY} + fi +} + +do_concat_dtb () { + # Concatenate U-Boot w/o DTB & DTB with public key + # (cf. kernel-fitimage.bbclass for more details) + if [ "x${UBOOT_SIGN_ENABLE}" = "x1" ]; then + if [ "x${UBOOT_SUFFIX}" = "ximg" -o "x${UBOOT_SUFFIX}" = "xrom" ] && \ + [ -e "${DEPLOYDIR}/${UBOOT_DTB_IMAGE}" ]; then + cd ${B} + oe_runmake EXT_DTB=${DEPLOYDIR}/${UBOOT_DTB_IMAGE} + install ${B}/${UBOOT_BINARY} ${DEPLOYDIR}/${UBOOT_IMAGE} + install ${B}/${UBOOT_BINARY} ${DEPLOY_DIR_IMAGE}/${UBOOT_IMAGE} + elif [ -e "${DEPLOYDIR}/${UBOOT_NODTB_IMAGE}" -a -e "${DEPLOYDIR}/${UBOOT_DTB_IMAGE}" ]; then + cd ${DEPLOYDIR} + cat ${UBOOT_NODTB_IMAGE} ${UBOOT_DTB_IMAGE} | tee ${B}/${UBOOT_BINARY} > ${UBOOT_IMAGE} + else + bbwarn "Failure while adding public key to u-boot binary. Verified boot won't be available." + fi + fi +} + +python () { + uboot_pn = d.getVar('PREFERRED_PROVIDER_u-boot') or 'u-boot' + if d.getVar('UBOOT_SIGN_ENABLE') == '1' and d.getVar('PN') == uboot_pn: + kernel_pn = d.getVar('PREFERRED_PROVIDER_virtual/kernel') + + # u-boot.dtb and u-boot-nodtb.bin are deployed _before_ do_deploy + # Thus, do_deploy_setscene will also populate them in DEPLOY_IMAGE_DIR + bb.build.addtask('do_deploy_dtb', 'do_deploy', 'do_compile', d) + + # do_concat_dtb is scheduled _before_ do_install as it overwrite the + # u-boot.bin in both DEPLOYDIR and DEPLOY_IMAGE_DIR. + bb.build.addtask('do_concat_dtb', 'do_install', None, d) + d.appendVarFlag('do_concat_dtb', 'depends', ' %s:do_assemble_fitimage' % kernel_pn) +} diff --git a/meta/classes/uninative.bbclass b/meta/classes/uninative.bbclass index 51391dbc4a..8f3448336f 100644 --- a/meta/classes/uninative.bbclass +++ b/meta/classes/uninative.bbclass @@ -1,20 +1,107 @@ -NATIVELSBSTRING = "universal" +UNINATIVE_LOADER ?= "${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux/lib/${@bb.utils.contains('BUILD_ARCH', 'x86_64', 'ld-linux-x86-64.so.2', 'ld-linux.so.2', d)}" +UNINATIVE_STAGING_DIR ?= "${STAGING_DIR}" -UNINATIVE_LOADER = "${STAGING_DIR_NATIVE}/lib/ld-linux-x86-64.so.2" +UNINATIVE_URL ?= "unset" +UNINATIVE_TARBALL ?= "${BUILD_ARCH}-nativesdk-libc.tar.bz2" +# Example checksums +#UNINATIVE_CHECKSUM[i586] = "dead" +#UNINATIVE_CHECKSUM[x86_64] = "dead" +UNINATIVE_DLDIR ?= "${DL_DIR}/uninative/" -addhandler uninative_eventhandler -uninative_eventhandler[eventmask] = "bb.event.BuildStarted" +addhandler uninative_event_fetchloader +uninative_event_fetchloader[eventmask] = "bb.event.BuildStarted" -python uninative_eventhandler() { - loader = e.data.getVar("UNINATIVE_LOADER", True) - if not os.path.exists(loader): - import subprocess - cmd = e.data.expand("mkdir -p ${STAGING_DIR}; cd ${STAGING_DIR}; tar -xjf ${COREBASE}/${BUILD_ARCH}-nativesdk-libc.tar.bz2; ${STAGING_DIR}/relocate_sdk.py ${STAGING_DIR_NATIVE} ${UNINATIVE_LOADER} ${UNINATIVE_LOADER} ${STAGING_BINDIR_NATIVE}/patchelf-uninative") - #bb.warn("nativesdk lib extraction: " + cmd) - subprocess.check_call(cmd, shell=True) +addhandler uninative_event_enable +uninative_event_enable[eventmask] = "bb.event.ConfigParsed" + +python uninative_event_fetchloader() { + """ + This event fires on the parent and will try to fetch the tarball if the + loader isn't already present. + """ + + chksum = d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + if not chksum: + bb.fatal("Uninative selected but not configured correctly, please set UNINATIVE_CHECKSUM[%s]" % d.getVar("BUILD_ARCH")) + + loader = d.getVar("UNINATIVE_LOADER") + loaderchksum = loader + ".chksum" + if os.path.exists(loader) and os.path.exists(loaderchksum): + with open(loaderchksum, "r") as f: + readchksum = f.read().strip() + if readchksum == chksum: + return + + import subprocess + try: + # Save and restore cwd as Fetch.download() does a chdir() + olddir = os.getcwd() + + tarball = d.getVar("UNINATIVE_TARBALL") + tarballdir = os.path.join(d.getVar("UNINATIVE_DLDIR"), chksum) + tarballpath = os.path.join(tarballdir, tarball) + + if not os.path.exists(tarballpath): + bb.utils.mkdirhier(tarballdir) + if d.getVar("UNINATIVE_URL") == "unset": + bb.fatal("Uninative selected but not configured, please set UNINATIVE_URL") + + localdata = bb.data.createCopy(d) + localdata.setVar('FILESPATH', "") + localdata.setVar('DL_DIR', tarballdir) + + srcuri = d.expand("${UNINATIVE_URL}${UNINATIVE_TARBALL};sha256sum=%s" % chksum) + bb.note("Fetching uninative binary shim from %s" % srcuri) + + fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) + fetcher.download() + localpath = fetcher.localpath(srcuri) + if localpath != tarballpath and os.path.exists(localpath) and not os.path.exists(tarballpath): + os.symlink(localpath, tarballpath) + + cmd = d.expand("\ +mkdir -p ${UNINATIVE_STAGING_DIR}-uninative; \ +cd ${UNINATIVE_STAGING_DIR}-uninative; \ +tar -xjf ${UNINATIVE_DLDIR}/%s/${UNINATIVE_TARBALL}; \ +${UNINATIVE_STAGING_DIR}-uninative/relocate_sdk.py \ + ${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux \ + ${UNINATIVE_LOADER} \ + ${UNINATIVE_LOADER} \ + ${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux/${bindir_native}/patchelf-uninative \ + ${UNINATIVE_STAGING_DIR}-uninative/${BUILD_ARCH}-linux${base_libdir_native}/libc*.so" % chksum) + subprocess.check_output(cmd, shell=True) + + with open(loaderchksum, "w") as f: + f.write(chksum) + + enable_uninative(d) + + except bb.fetch2.BBFetchException as exc: + bb.warn("Disabling uninative as unable to fetch uninative tarball: %s" % str(exc)) + bb.warn("To build your own uninative loader, please bitbake uninative-tarball and set UNINATIVE_TARBALL appropriately.") + except subprocess.CalledProcessError as exc: + bb.warn("Disabling uninative as unable to install uninative tarball: %s" % str(exc)) + bb.warn("To build your own uninative loader, please bitbake uninative-tarball and set UNINATIVE_TARBALL appropriately.") + finally: + os.chdir(olddir) } -SSTATEPOSTUNPACKFUNCS_append = " uninative_changeinterp" +python uninative_event_enable() { + """ + This event handler is called in the workers and is responsible for setting + up uninative if a loader is found. + """ + enable_uninative(d) +} + +def enable_uninative(d): + loader = d.getVar("UNINATIVE_LOADER") + if os.path.exists(loader): + bb.debug(2, "Enabling uninative") + d.setVar("NATIVELSBSTRING", "universal%s" % oe.utils.host_gcc_version(d)) + d.appendVar("SSTATEPOSTUNPACKFUNCS", " uninative_changeinterp") + d.appendVarFlag("SSTATEPOSTUNPACKFUNCS", "vardepvalueexclude", "| uninative_changeinterp") + d.prependVar("PATH", "${STAGING_DIR}-uninative/${BUILD_ARCH}-linux${bindir_native}:") python uninative_changeinterp () { import subprocess @@ -24,9 +111,11 @@ python uninative_changeinterp () { if not (bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d)): return - sstateinst = d.getVar('SSTATE_INSTDIR', True) + sstateinst = d.getVar('SSTATE_INSTDIR') for walkroot, dirs, files in os.walk(sstateinst): for file in files: + if file.endswith(".so") or ".so." in file: + continue f = os.path.join(walkroot, file) if os.path.islink(f): continue @@ -36,9 +125,10 @@ python uninative_changeinterp () { elf = oe.qa.ELFFile(f) try: elf.open() - except: + except oe.qa.NotELFFileError: + continue + if not elf.isDynamic(): continue - #bb.warn("patchelf-uninative --set-interpreter %s %s" % (d.getVar("UNINATIVE_LOADER", True), f)) - subprocess.call("patchelf-uninative --set-interpreter %s %s" % (d.getVar("UNINATIVE_LOADER", True), f), shell=True) + subprocess.check_output(("patchelf-uninative", "--set-interpreter", d.getVar("UNINATIVE_LOADER"), f), stderr=subprocess.STDOUT) } diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass index a3c1657fa9..4bba76c3ba 100644 --- a/meta/classes/update-alternatives.bbclass +++ b/meta/classes/update-alternatives.bbclass @@ -61,30 +61,32 @@ ALTERNATIVE_PRIORITY = "10" # We need special processing for vardeps because it can not work on -# modified flag values. So we agregate the flags into a new variable +# modified flag values. So we aggregate the flags into a new variable # and include that vairable in the set. UPDALTVARS = "ALTERNATIVE ALTERNATIVE_LINK_NAME ALTERNATIVE_TARGET ALTERNATIVE_PRIORITY" +PACKAGE_WRITE_DEPS += "virtual/update-alternatives-native" + def gen_updatealternativesvardeps(d): - pkgs = (d.getVar("PACKAGES", True) or "").split() - vars = (d.getVar("UPDALTVARS", True) or "").split() + pkgs = (d.getVar("PACKAGES") or "").split() + vars = (d.getVar("UPDALTVARS") or "").split() # First compute them for non_pkg versions for v in vars: - for flag in (d.getVarFlags(v) or {}): + for flag in sorted((d.getVarFlags(v) or {}).keys()): if flag == "doc" or flag == "vardeps" or flag == "vardepsexp": continue d.appendVar('%s_VARDEPS' % (v), ' %s:%s' % (flag, d.getVarFlag(v, flag, False))) for p in pkgs: for v in vars: - for flag in (d.getVarFlags("%s_%s" % (v,p)) or {}): + for flag in sorted((d.getVarFlags("%s_%s" % (v,p)) or {}).keys()): if flag == "doc" or flag == "vardeps" or flag == "vardepsexp": continue d.appendVar('%s_VARDEPS_%s' % (v,p), ' %s:%s' % (flag, d.getVarFlag('%s_%s' % (v,p), flag, False))) def ua_extend_depends(d): - if not 'virtual/update-alternatives' in d.getVar('PROVIDES', True): + if not 'virtual/update-alternatives' in d.getVar('PROVIDES'): d.appendVar('DEPENDS', ' virtual/${MLPREFIX}update-alternatives') python __anonymous() { @@ -94,6 +96,10 @@ python __anonymous() { bb.data.inherits_class('cross-canadian', d): return + # Disable when targeting mingw32 (no target support) + if d.getVar("TARGET_OS") == "mingw32": + return + # compute special vardeps gen_updatealternativesvardeps(d) @@ -103,8 +109,8 @@ python __anonymous() { def gen_updatealternativesvars(d): ret = [] - pkgs = (d.getVar("PACKAGES", True) or "").split() - vars = (d.getVar("UPDALTVARS", True) or "").split() + pkgs = (d.getVar("PACKAGES") or "").split() + vars = (d.getVar("UPDALTVARS") or "").split() for v in vars: ret.append(v + "_VARDEPS") @@ -123,23 +129,23 @@ populate_packages[vardeps] += "${UPDALTVARS} ${@gen_updatealternativesvars(d)}" # place. python perform_packagecopy_append () { # Check for deprecated usage... - pn = d.getVar('BPN', True) - if d.getVar('ALTERNATIVE_LINKS', True) != None: + pn = d.getVar('BPN') + if d.getVar('ALTERNATIVE_LINKS') != None: bb.fatal('%s: Use of ALTERNATIVE_LINKS/ALTERNATIVE_PATH/ALTERNATIVE_NAME is no longer supported, please convert to the updated syntax, see update-alternatives.bbclass for more info.' % pn) # Do actual update alternatives processing - pkgdest = d.getVar('PKGD', True) - for pkg in (d.getVar('PACKAGES', True) or "").split(): + pkgdest = d.getVar('PKGD') + for pkg in (d.getVar('PACKAGES') or "").split(): # If the src == dest, we know we need to rename the dest by appending ${BPN} link_rename = {} - for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg, True) or "").split(): - alt_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', alt_name, True) + for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg) or "").split(): + alt_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', alt_name) if not alt_link: - alt_link = "%s/%s" % (d.getVar('bindir', True), alt_name) + alt_link = "%s/%s" % (d.getVar('bindir'), alt_name) d.setVarFlag('ALTERNATIVE_LINK_NAME', alt_name, alt_link) - alt_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, alt_name, True) or d.getVarFlag('ALTERNATIVE_TARGET', alt_name, True) - alt_target = alt_target or d.getVar('ALTERNATIVE_TARGET_%s' % pkg, True) or d.getVar('ALTERNATIVE_TARGET', True) or alt_link + alt_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, alt_name) or d.getVarFlag('ALTERNATIVE_TARGET', alt_name) + alt_target = alt_target or d.getVar('ALTERNATIVE_TARGET_%s' % pkg) or d.getVar('ALTERNATIVE_TARGET') or alt_link # Sometimes alt_target is specified as relative to the link name. alt_target = os.path.join(os.path.dirname(alt_link), alt_target) @@ -189,23 +195,23 @@ python perform_packagecopy_append () { PACKAGESPLITFUNCS_prepend = "populate_packages_updatealternatives " python populate_packages_updatealternatives () { - pn = d.getVar('BPN', True) + pn = d.getVar('BPN') # Do actual update alternatives processing - pkgdest = d.getVar('PKGD', True) - for pkg in (d.getVar('PACKAGES', True) or "").split(): + pkgdest = d.getVar('PKGD') + for pkg in (d.getVar('PACKAGES') or "").split(): # Create post install/removal scripts - alt_setup_links = "" - alt_remove_links = "" - for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg, True) or "").split(): - alt_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', alt_name, True) - alt_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, alt_name, True) or d.getVarFlag('ALTERNATIVE_TARGET', alt_name, True) - alt_target = alt_target or d.getVar('ALTERNATIVE_TARGET_%s' % pkg, True) or d.getVar('ALTERNATIVE_TARGET', True) or alt_link + alt_setup_links = "# Begin section update-alternatives\n" + alt_remove_links = "# Begin section update-alternatives\n" + for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg) or "").split(): + alt_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', alt_name) + alt_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, alt_name) or d.getVarFlag('ALTERNATIVE_TARGET', alt_name) + alt_target = alt_target or d.getVar('ALTERNATIVE_TARGET_%s' % pkg) or d.getVar('ALTERNATIVE_TARGET') or alt_link # Sometimes alt_target is specified as relative to the link name. alt_target = os.path.join(os.path.dirname(alt_link), alt_target) - alt_priority = d.getVarFlag('ALTERNATIVE_PRIORITY_%s' % pkg, alt_name, True) or d.getVarFlag('ALTERNATIVE_PRIORITY', alt_name, True) - alt_priority = alt_priority or d.getVar('ALTERNATIVE_PRIORITY_%s' % pkg, True) or d.getVar('ALTERNATIVE_PRIORITY', True) + alt_priority = d.getVarFlag('ALTERNATIVE_PRIORITY_%s' % pkg, alt_name) or d.getVarFlag('ALTERNATIVE_PRIORITY', alt_name) + alt_priority = alt_priority or d.getVar('ALTERNATIVE_PRIORITY_%s' % pkg) or d.getVar('ALTERNATIVE_PRIORITY') # This shouldn't trigger, as it should have been resolved earlier! if alt_link == alt_target: @@ -219,40 +225,55 @@ python populate_packages_updatealternatives () { # Default to generate shell script.. eventually we may want to change this... alt_target = os.path.normpath(alt_target) - alt_setup_links += '\tupdate-alternatives --install %s %s %s %s\n' % (alt_link, alt_name, alt_target, alt_priority) - alt_remove_links += '\tupdate-alternatives --remove %s %s\n' % (alt_name, alt_target) + alt_setup_links += 'update-alternatives --install %s %s %s %s\n' % (alt_link, alt_name, alt_target, alt_priority) + alt_remove_links += 'update-alternatives --remove %s %s\n' % (alt_name, alt_target) + + alt_setup_links += "# End section update-alternatives\n" + alt_remove_links += "# End section update-alternatives\n" - if alt_setup_links: + if len(alt_setup_links.splitlines()) > 2: # RDEPENDS setup - provider = d.getVar('VIRTUAL-RUNTIME_update-alternatives', True) + provider = d.getVar('VIRTUAL-RUNTIME_update-alternatives') if provider: #bb.note('adding runtime requirement for update-alternatives for %s' % pkg) d.appendVar('RDEPENDS_%s' % pkg, ' ' + d.getVar('MLPREFIX', False) + provider) bb.note('adding update-alternatives calls to postinst/prerm for %s' % pkg) bb.note('%s' % alt_setup_links) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) or '#!/bin/sh\n' - postinst += alt_setup_links + postinst = d.getVar('pkg_postinst_%s' % pkg) or '#!/bin/sh\n' + postinst = postinst.splitlines(True) + try: + index = postinst.index('# Begin section update-rc.d\n') + postinst.insert(index, alt_setup_links) + except ValueError: + postinst.append(alt_setup_links) + postinst = ''.join(postinst) d.setVar('pkg_postinst_%s' % pkg, postinst) bb.note('%s' % alt_remove_links) - prerm = d.getVar('pkg_prerm_%s' % pkg, True) or '#!/bin/sh\n' - prerm += alt_remove_links + prerm = d.getVar('pkg_prerm_%s' % pkg) or '#!/bin/sh\n' + prerm = prerm.splitlines(True) + try: + index = prerm.index('# End section update-rc.d\n') + prerm.insert(index + 1, alt_remove_links) + except ValueError: + prerm.append(alt_remove_links) + prerm = ''.join(prerm) d.setVar('pkg_prerm_%s' % pkg, prerm) } python package_do_filedeps_append () { - pn = d.getVar('BPN', True) - pkgdest = d.getVar('PKGDEST', True) + pn = d.getVar('BPN') + pkgdest = d.getVar('PKGDEST') for pkg in packages.split(): - for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg, True) or "").split(): - alt_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', alt_name, True) - alt_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, alt_name, True) or d.getVarFlag('ALTERNATIVE_TARGET', alt_name, True) - alt_target = alt_target or d.getVar('ALTERNATIVE_TARGET_%s' % pkg, True) or d.getVar('ALTERNATIVE_TARGET', True) or alt_link + for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg) or "").split(): + alt_link = d.getVarFlag('ALTERNATIVE_LINK_NAME', alt_name) + alt_target = d.getVarFlag('ALTERNATIVE_TARGET_%s' % pkg, alt_name) or d.getVarFlag('ALTERNATIVE_TARGET', alt_name) + alt_target = alt_target or d.getVar('ALTERNATIVE_TARGET_%s' % pkg) or d.getVar('ALTERNATIVE_TARGET') or alt_link if alt_link == alt_target: - bb.warn('alt_link == alt_target: %s == %s' % (alt_link, alt_target)) + bb.warn('%s: alt_link == alt_target: %s == %s' % (pn, alt_link, alt_target)) alt_target = '%s.%s' % (alt_target, pn) if not os.path.lexists('%s/%s/%s' % (pkgdest, pkg, alt_target)): @@ -261,7 +282,7 @@ python package_do_filedeps_append () { # Add file provide trans_target = oe.package.file_translate(alt_target) d.appendVar('FILERPROVIDES_%s_%s' % (trans_target, pkg), " " + alt_link) - if not trans_target in (d.getVar('FILERPROVIDESFLIST_%s' % pkg, True) or ""): + if not trans_target in (d.getVar('FILERPROVIDESFLIST_%s' % pkg) or ""): d.appendVar('FILERPROVIDESFLIST_%s' % pkg, " " + trans_target) } diff --git a/meta/classes/update-rc.d.bbclass b/meta/classes/update-rc.d.bbclass index 2a0a74a5f5..9ba3daccaf 100644 --- a/meta/classes/update-rc.d.bbclass +++ b/meta/classes/update-rc.d.bbclass @@ -1,6 +1,7 @@ UPDATERCPN ?= "${PN}" -DEPENDS_append_class-target = " update-rc.d-native update-rc.d initscripts" +DEPENDS_append_class-target = "${@bb.utils.contains('DISTRO_FEATURES', 'sysvinit', ' update-rc.d initscripts', '', d)}" + UPDATERCD = "update-rc.d" UPDATERCD_class-cross = "" UPDATERCD_class-native = "" @@ -10,11 +11,20 @@ INITSCRIPT_PARAMS ?= "defaults" INIT_D_DIR = "${sysconfdir}/init.d" +def use_updatercd(d): + # If the distro supports both sysvinit and systemd, and the current recipe + # supports systemd, only call update-rc.d on rootfs creation or if systemd + # is not running. That's because systemctl enable/disable will already call + # update-rc.d if it detects initscripts. + if bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d) and bb.data.inherits_class('systemd', d): + return '[ -n "$D" -o ! -d /run/systemd/system ]' + return 'true' + updatercd_preinst() { -if [ -z "$D" -a -f "${INIT_D_DIR}/${INITSCRIPT_NAME}" ]; then - ${INIT_D_DIR}/${INITSCRIPT_NAME} stop +if ${@use_updatercd(d)} && [ -z "$D" -a -f "${INIT_D_DIR}/${INITSCRIPT_NAME}" ]; then + ${INIT_D_DIR}/${INITSCRIPT_NAME} stop || : fi -if type update-rc.d >/dev/null 2>/dev/null; then +if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then if [ -n "$D" ]; then OPT="-f -r $D" else @@ -24,8 +34,11 @@ if type update-rc.d >/dev/null 2>/dev/null; then fi } +PACKAGE_WRITE_DEPS += "update-rc.d-native" + updatercd_postinst() { -if type update-rc.d >/dev/null 2>/dev/null; then +# Begin section update-rc.d +if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then if [ -n "$D" ]; then OPT="-r $D" else @@ -33,16 +46,19 @@ if type update-rc.d >/dev/null 2>/dev/null; then fi update-rc.d $OPT ${INITSCRIPT_NAME} ${INITSCRIPT_PARAMS} fi +# End section update-rc.d } updatercd_prerm() { -if [ -z "$D" ]; then - ${INIT_D_DIR}/${INITSCRIPT_NAME} stop +# Begin section update-rc.d +if ${@use_updatercd(d)} && [ -z "$D" -a -x "${INIT_D_DIR}/${INITSCRIPT_NAME}" ]; then + ${INIT_D_DIR}/${INITSCRIPT_NAME} stop || : fi +# End section update-rc.d } updatercd_postrm() { -if type update-rc.d >/dev/null 2>/dev/null; then +if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then if [ -n "$D" ]; then OPT="-f -r $D" else @@ -56,15 +72,15 @@ fi def update_rc_after_parse(d): if d.getVar('INITSCRIPT_PACKAGES', False) == None: if d.getVar('INITSCRIPT_NAME', False) == None: - raise bb.build.FuncFailed("%s inherits update-rc.d but doesn't set INITSCRIPT_NAME" % d.getVar('FILE', False)) + bb.fatal("%s inherits update-rc.d but doesn't set INITSCRIPT_NAME" % d.getVar('FILE', False)) if d.getVar('INITSCRIPT_PARAMS', False) == None: - raise bb.build.FuncFailed("%s inherits update-rc.d but doesn't set INITSCRIPT_PARAMS" % d.getVar('FILE', False)) + bb.fatal("%s inherits update-rc.d but doesn't set INITSCRIPT_PARAMS" % d.getVar('FILE', False)) python __anonymous() { update_rc_after_parse(d) } -PACKAGESPLITFUNCS_prepend = "populate_packages_updatercd " +PACKAGESPLITFUNCS_prepend = "${@bb.utils.contains('DISTRO_FEATURES', 'sysvinit', 'populate_packages_updatercd ', '', d)}" PACKAGESPLITFUNCS_remove_class-nativesdk = "populate_packages_updatercd " populate_packages_updatercd[vardeps] += "updatercd_prerm updatercd_postrm updatercd_preinst updatercd_postinst" @@ -79,53 +95,63 @@ python populate_packages_updatercd () { return statement = "grep -q -w '/etc/init.d/functions' %s" % path if subprocess.call(statement, shell=True) == 0: - mlprefix = d.getVar('MLPREFIX', True) or "" + mlprefix = d.getVar('MLPREFIX') or "" d.appendVar('RDEPENDS_' + pkg, ' %sinitscripts-functions' % (mlprefix)) def update_rcd_package(pkg): bb.debug(1, 'adding update-rc.d calls to preinst/postinst/prerm/postrm for %s' % pkg) localdata = bb.data.createCopy(d) - overrides = localdata.getVar("OVERRIDES", True) + overrides = localdata.getVar("OVERRIDES") localdata.setVar("OVERRIDES", "%s:%s" % (pkg, overrides)) - bb.data.update_data(localdata) update_rcd_auto_depend(pkg) - preinst = d.getVar('pkg_preinst_%s' % pkg, True) + preinst = d.getVar('pkg_preinst_%s' % pkg) if not preinst: preinst = '#!/bin/sh\n' - preinst += localdata.getVar('updatercd_preinst', True) + preinst += localdata.getVar('updatercd_preinst') d.setVar('pkg_preinst_%s' % pkg, preinst) - postinst = d.getVar('pkg_postinst_%s' % pkg, True) + postinst = d.getVar('pkg_postinst_%s' % pkg) if not postinst: postinst = '#!/bin/sh\n' - postinst += localdata.getVar('updatercd_postinst', True) + postinst = postinst.splitlines(True) + try: + index = postinst.index('# End section update-alternatives\n') + postinst.insert(index + 1, localdata.getVar('updatercd_postinst')) + except ValueError: + postinst.append(localdata.getVar('updatercd_postinst')) + postinst = ''.join(postinst) d.setVar('pkg_postinst_%s' % pkg, postinst) - prerm = d.getVar('pkg_prerm_%s' % pkg, True) + prerm = d.getVar('pkg_prerm_%s' % pkg) if not prerm: prerm = '#!/bin/sh\n' - prerm += localdata.getVar('updatercd_prerm', True) + prerm = prerm.splitlines(True) + try: + index = prerm.index('# Begin section update-alternatives\n') + prerm.insert(index, localdata.getVar('updatercd_prerm')) + except ValueError: + prerm.append(localdata.getVar('updatercd_prerm')) + prerm = ''.join(prerm) d.setVar('pkg_prerm_%s' % pkg, prerm) - postrm = d.getVar('pkg_postrm_%s' % pkg, True) + postrm = d.getVar('pkg_postrm_%s' % pkg) if not postrm: postrm = '#!/bin/sh\n' - postrm += localdata.getVar('updatercd_postrm', True) + postrm += localdata.getVar('updatercd_postrm') d.setVar('pkg_postrm_%s' % pkg, postrm) d.appendVar('RRECOMMENDS_' + pkg, " ${MLPREFIX}${UPDATERCD}") # Check that this class isn't being inhibited (generally, by # systemd.bbclass) before doing any work. - if bb.utils.contains('DISTRO_FEATURES', 'sysvinit', True, False, d) or \ - not d.getVar("INHIBIT_UPDATERCD_BBCLASS", True): - pkgs = d.getVar('INITSCRIPT_PACKAGES', True) + if not d.getVar("INHIBIT_UPDATERCD_BBCLASS"): + pkgs = d.getVar('INITSCRIPT_PACKAGES') if pkgs == None: - pkgs = d.getVar('UPDATERCPN', True) - packages = (d.getVar('PACKAGES', True) or "").split() + pkgs = d.getVar('UPDATERCPN') + packages = (d.getVar('PACKAGES') or "").split() if not pkgs in packages and packages != []: pkgs = packages[0] for pkg in pkgs.split(): diff --git a/meta/classes/upstream-version-is-even.bbclass b/meta/classes/upstream-version-is-even.bbclass new file mode 100644 index 0000000000..256c752423 --- /dev/null +++ b/meta/classes/upstream-version-is-even.bbclass @@ -0,0 +1,5 @@ +# This class ensures that the upstream version check only +# accepts even minor versions (i.e. 3.0.x, 3.2.x, 3.4.x, etc.) +# This scheme is used by Gnome and a number of other projects +# to signify stable releases vs development releases. +UPSTREAM_CHECK_REGEX = "[^\d\.](?P<pver>\d+\.(\d*[02468])+(\.\d+)+)\.tar" diff --git a/meta/classes/useradd-staticids.bbclass b/meta/classes/useradd-staticids.bbclass index 421a70a6ab..2d282c0d71 100644 --- a/meta/classes/useradd-staticids.bbclass +++ b/meta/classes/useradd-staticids.bbclass @@ -2,22 +2,63 @@ # we need a function to reformat the params based on a static file def update_useradd_static_config(d): import argparse + import itertools import re + import errno class myArgumentParser( argparse.ArgumentParser ): def _print_message(self, message, file=None): - bb.warn("%s - %s: %s" % (d.getVar('PN', True), pkg, message)) + bb.warn("%s - %s: %s" % (d.getVar('PN'), pkg, message)) # This should never be called... def exit(self, status=0, message=None): - message = message or ("%s - %s: useradd.bbclass: Argument parsing exited" % (d.getVar('PN', True), pkg)) + message = message or ("%s - %s: useradd.bbclass: Argument parsing exited" % (d.getVar('PN'), pkg)) error(message) def error(self, message): - raise bb.build.FuncFailed(message) + bb.fatal(message) + + def list_extend(iterable, length, obj = None): + """Ensure that iterable is the specified length by extending with obj + and return it as a list""" + return list(itertools.islice(itertools.chain(iterable, itertools.repeat(obj)), length)) + + def merge_files(file_list, exp_fields): + """Read each passwd/group file in file_list, split each line and create + a dictionary with the user/group names as keys and the split lines as + values. If the user/group name already exists in the dictionary, then + update any fields in the list with the values from the new list (if they + are set).""" + id_table = dict() + for conf in file_list.split(): + try: + with open(conf, "r") as f: + for line in f: + if line.startswith('#'): + continue + # Make sure there always are at least exp_fields + # elements in the field list. This allows for leaving + # out trailing colons in the files. + fields = list_extend(line.rstrip().split(":"), exp_fields) + if fields[0] not in id_table: + id_table[fields[0]] = fields + else: + id_table[fields[0]] = list(map(lambda x, y: x or y, fields, id_table[fields[0]])) + except IOError as e: + if e.errno == errno.ENOENT: + pass + + return id_table + + def handle_missing_id(id, type, pkg): + # For backwards compatibility we accept "1" in addition to "error" + if d.getVar('USERADD_ERROR_DYNAMIC') == 'error' or d.getVar('USERADD_ERROR_DYNAMIC') == '1': + raise NotImplementedError("%s - %s: %sname %s does not have a static ID defined. Skipping it." % (d.getVar('PN'), pkg, type, id)) + elif d.getVar('USERADD_ERROR_DYNAMIC') == 'warn': + bb.warn("%s - %s: %sname %s does not have a static ID defined." % (d.getVar('PN'), pkg, type, id)) # We parse and rewrite the useradd components - def rewrite_useradd(params): + def rewrite_useradd(params, is_pkg): # The following comes from --help on useradd from shadow parser = myArgumentParser(prog='useradd') parser.add_argument("-b", "--base-dir", metavar="BASE_DIR", help="base directory for the home directory of the new account") @@ -31,25 +72,26 @@ def update_useradd_static_config(d): parser.add_argument("-k", "--skel", metavar="SKEL_DIR", help="use this alternative skeleton directory") parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults") parser.add_argument("-l", "--no-log-init", help="do not add the user to the lastlog and faillog databases", action="store_true") - parser.add_argument("-m", "--create-home", help="create the user's home directory", action="store_true") - parser.add_argument("-M", "--no-create-home", help="do not create the user's home directory", action="store_true") - parser.add_argument("-N", "--no-user-group", help="do not create a group with the same name as the user", action="store_true") + parser.add_argument("-m", "--create-home", help="create the user's home directory", action="store_const", const=True) + parser.add_argument("-M", "--no-create-home", dest="create_home", help="do not create the user's home directory", action="store_const", const=False) + parser.add_argument("-N", "--no-user-group", dest="user_group", help="do not create a group with the same name as the user", action="store_const", const=False) parser.add_argument("-o", "--non-unique", help="allow to create users with duplicate (non-unique UID)", action="store_true") parser.add_argument("-p", "--password", metavar="PASSWORD", help="encrypted password of the new account") + parser.add_argument("-P", "--clear-password", metavar="CLEAR_PASSWORD", help="use this clear password for the new account") parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into") parser.add_argument("-r", "--system", help="create a system account", action="store_true") parser.add_argument("-s", "--shell", metavar="SHELL", help="login shell of the new account") parser.add_argument("-u", "--uid", metavar="UID", help="user ID of the new account") - parser.add_argument("-U", "--user-group", help="create a group with the same name as the user", action="store_true") + parser.add_argument("-U", "--user-group", help="create a group with the same name as the user", action="store_const", const=True) parser.add_argument("LOGIN", help="Login name of the new user") # Return a list of configuration files based on either the default # files/passwd or the contents of USERADD_UID_TABLES - # paths are resulved via BBPATH + # paths are resolved via BBPATH def get_passwd_list(d): str = "" - bbpath = d.getVar('BBPATH', True) - passwd_tables = d.getVar('USERADD_UID_TABLES', True) + bbpath = d.getVar('BBPATH') + passwd_tables = d.getVar('USERADD_UID_TABLES') if not passwd_tables: passwd_tables = 'files/passwd' for conf_file in passwd_tables.split(): @@ -57,19 +99,19 @@ def update_useradd_static_config(d): return str newparams = [] + users = None for param in re.split('''[ \t]*;[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params): param = param.strip() if not param: continue try: - uaargs = parser.parse_args(re.split('''[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', param)) + uaargs = parser.parse_args(re.split('''[ \t]+(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', param)) except: - raise bb.build.FuncFailed("%s: Unable to parse arguments for USERADD_PARAM_%s: '%s'" % (d.getVar('PN', True), pkg, param)) + bb.fatal("%s: Unable to parse arguments for USERADD_PARAM_%s: '%s'" % (d.getVar('PN'), pkg, param)) - # files/passwd or the contents of USERADD_UID_TABLES + # Read all passwd files specified in USERADD_UID_TABLES or files/passwd # Use the standard passwd layout: # username:password:user_id:group_id:comment:home_directory:login_shell - # (we want to process in reverse order, as 'last found' in the list wins) # # If a field is left blank, the original value will be used. The 'username' # field is required. @@ -78,98 +120,99 @@ def update_useradd_static_config(d): # in the useradd command may introduce a security hole. It's assumed that # all new users get the default ('*' which prevents login) until the user is # specifically configured by the system admin. - for conf in get_passwd_list(d).split()[::-1]: - if os.path.exists(conf): - f = open(conf, "r") - for line in f: - if line.startswith('#'): - continue - field = line.rstrip().split(":") - if field[0] == uaargs.LOGIN: - if uaargs.uid and field[2] and (uaargs.uid != field[2]): - bb.warn("%s: Changing username %s's uid from (%s) to (%s), verify configuration files!" % (d.getVar('PN', True), uaargs.LOGIN, uaargs.uid, field[2])) - uaargs.uid = [field[2], uaargs.uid][not field[2]] - - # Determine the possible groupname - # Unless the group name (or gid) is specified, we assume that the LOGIN is the groupname - # - # By default the system has creation of the matching groups enabled - # So if the implicit username-group creation is on, then the implicit groupname (LOGIN) - # is used, and we disable the user_group option. - # - uaargs.groupname = [uaargs.gid, uaargs.LOGIN][not uaargs.gid or uaargs.user_group] - uaargs.groupid = [uaargs.gid, uaargs.groupname][not uaargs.gid] - uaargs.groupid = [field[3], uaargs.groupid][not field[3]] - - if not uaargs.gid or uaargs.gid != uaargs.groupid: - if (uaargs.groupid and uaargs.groupid.isdigit()) and (uaargs.groupname and uaargs.groupname.isdigit()) and (uaargs.groupid != uaargs.groupname): - # We want to add a group, but we don't know it's name... so we can't add the group... - # We have to assume the group has previously been added or we'll fail on the adduser... - # Note: specifying the actual gid is very rare in OE, usually the group name is specified. - bb.warn("%s: Changing gid for login %s from (%s) to (%s), verify configuration files!" % (d.getVar('PN', True), uaargs.LOGIN, uaargs.groupname, uaargs.gid)) - elif (uaargs.groupid and not uaargs.groupid.isdigit()) and uaargs.groupid == uaargs.groupname: - # We don't have a number, so we have to add a name - bb.debug(1, "Adding group %s!" % (uaargs.groupname)) - uaargs.gid = uaargs.groupid - uaargs.user_group = False - groupadd = d.getVar("GROUPADD_PARAM_%s" % pkg, True) - newgroup = "%s %s" % (['', ' --system'][uaargs.system], uaargs.groupname) - if groupadd: - d.setVar("GROUPADD_PARAM_%s" % pkg, "%s ; %s" % (groupadd, newgroup)) - else: - d.setVar("GROUPADD_PARAM_%s" % pkg, newgroup) - elif uaargs.groupname and (uaargs.groupid and uaargs.groupid.isdigit()): - # We have a group name and a group number to assign it to - bb.debug(1, "Adding group %s gid (%s)!" % (uaargs.groupname, uaargs.groupid)) - uaargs.gid = uaargs.groupid - uaargs.user_group = False - groupadd = d.getVar("GROUPADD_PARAM_%s" % pkg, True) - newgroup = "-g %s %s" % (uaargs.gid, uaargs.groupname) - if groupadd: - d.setVar("GROUPADD_PARAM_%s" % pkg, "%s ; %s" % (groupadd, newgroup)) - else: - d.setVar("GROUPADD_PARAM_%s" % pkg, newgroup) - - uaargs.comment = ["'%s'" % field[4], uaargs.comment][not field[4]] - uaargs.home_dir = [field[5], uaargs.home_dir][not field[5]] - uaargs.shell = [field[6], uaargs.shell][not field[6]] - break + if not users: + users = merge_files(get_passwd_list(d), 7) + + if uaargs.LOGIN not in users: + handle_missing_id(uaargs.LOGIN, 'user', pkg) + newparams.append(param) + continue + + field = users[uaargs.LOGIN] + + if uaargs.uid and field[2] and (uaargs.uid != field[2]): + bb.warn("%s: Changing username %s's uid from (%s) to (%s), verify configuration files!" % (d.getVar('PN'), uaargs.LOGIN, uaargs.uid, field[2])) + uaargs.uid = field[2] or uaargs.uid + + # Determine the possible groupname + # Unless the group name (or gid) is specified, we assume that the LOGIN is the groupname + # + # By default the system has creation of the matching groups enabled + # So if the implicit username-group creation is on, then the implicit groupname (LOGIN) + # is used, and we disable the user_group option. + # + user_group = uaargs.user_group is None or uaargs.user_group is True + uaargs.groupname = uaargs.LOGIN if user_group else uaargs.gid + uaargs.groupid = field[3] or uaargs.gid or uaargs.groupname + + if uaargs.groupid and uaargs.gid != uaargs.groupid: + newgroup = None + if not uaargs.groupid.isdigit(): + # We don't have a group number, so we have to add a name + bb.debug(1, "Adding group %s!" % uaargs.groupid) + newgroup = "%s %s" % (' --system' if uaargs.system else '', uaargs.groupid) + elif uaargs.groupname and not uaargs.groupname.isdigit(): + # We have a group name and a group number to assign it to + bb.debug(1, "Adding group %s (gid %s)!" % (uaargs.groupname, uaargs.groupid)) + newgroup = "-g %s %s" % (uaargs.groupid, uaargs.groupname) + else: + # We want to add a group, but we don't know it's name... so we can't add the group... + # We have to assume the group has previously been added or we'll fail on the adduser... + # Note: specifying the actual gid is very rare in OE, usually the group name is specified. + bb.warn("%s: Changing gid for login %s to %s, verify configuration files!" % (d.getVar('PN'), uaargs.LOGIN, uaargs.groupid)) + + uaargs.gid = uaargs.groupid + uaargs.user_group = None + if newgroup and is_pkg: + groupadd = d.getVar("GROUPADD_PARAM_%s" % pkg) + if groupadd: + # Only add the group if not already specified + if not uaargs.groupname in groupadd: + d.setVar("GROUPADD_PARAM_%s" % pkg, "%s; %s" % (groupadd, newgroup)) + else: + d.setVar("GROUPADD_PARAM_%s" % pkg, newgroup) + + uaargs.comment = "'%s'" % field[4] if field[4] else uaargs.comment + uaargs.home_dir = field[5] or uaargs.home_dir + uaargs.shell = field[6] or uaargs.shell # Should be an error if a specific option is set... - if d.getVar('USERADD_ERROR_DYNAMIC', True) == '1' and not ((uaargs.uid and uaargs.uid.isdigit()) and uaargs.gid): - #bb.error("Skipping recipe %s, package %s which adds username %s does not have a static uid defined." % (d.getVar('PN', True), pkg, uaargs.LOGIN)) - raise bb.build.FuncFailed("%s - %s: Username %s does not have a static uid defined." % (d.getVar('PN', True), pkg, uaargs.LOGIN)) + if not uaargs.uid or not uaargs.uid.isdigit() or not uaargs.gid: + handle_missing_id(uaargs.LOGIN, 'user', pkg) # Reconstruct the args... newparam = ['', ' --defaults'][uaargs.defaults] newparam += ['', ' --base-dir %s' % uaargs.base_dir][uaargs.base_dir != None] newparam += ['', ' --comment %s' % uaargs.comment][uaargs.comment != None] newparam += ['', ' --home-dir %s' % uaargs.home_dir][uaargs.home_dir != None] - newparam += ['', ' --expiredata %s' % uaargs.expiredate][uaargs.expiredate != None] + newparam += ['', ' --expiredate %s' % uaargs.expiredate][uaargs.expiredate != None] newparam += ['', ' --inactive %s' % uaargs.inactive][uaargs.inactive != None] newparam += ['', ' --gid %s' % uaargs.gid][uaargs.gid != None] newparam += ['', ' --groups %s' % uaargs.groups][uaargs.groups != None] newparam += ['', ' --skel %s' % uaargs.skel][uaargs.skel != None] newparam += ['', ' --key %s' % uaargs.key][uaargs.key != None] newparam += ['', ' --no-log-init'][uaargs.no_log_init] - newparam += ['', ' --create-home'][uaargs.create_home] - newparam += ['', ' --no-create-home'][uaargs.no_create_home] - newparam += ['', ' --no-user-group'][uaargs.no_user_group] + newparam += ['', ' --create-home'][uaargs.create_home is True] + newparam += ['', ' --no-create-home'][uaargs.create_home is False] + newparam += ['', ' --no-user-group'][uaargs.user_group is False] newparam += ['', ' --non-unique'][uaargs.non_unique] - newparam += ['', ' --password %s' % uaargs.password][uaargs.password != None] + if uaargs.password != None: + newparam += ['', ' --password %s' % uaargs.password][uaargs.password != None] + elif uaargs.clear_password: + newparam += ['', ' --clear-password %s' % uaargs.clear_password][uaargs.clear_password != None] newparam += ['', ' --root %s' % uaargs.root][uaargs.root != None] newparam += ['', ' --system'][uaargs.system] newparam += ['', ' --shell %s' % uaargs.shell][uaargs.shell != None] newparam += ['', ' --uid %s' % uaargs.uid][uaargs.uid != None] - newparam += ['', ' --user-group'][uaargs.user_group] + newparam += ['', ' --user-group'][uaargs.user_group is True] newparam += ' %s' % uaargs.LOGIN newparams.append(newparam) - return " ;".join(newparams).strip() + return ";".join(newparams).strip() # We parse and rewrite the groupadd components - def rewrite_groupadd(params): + def rewrite_groupadd(params, is_pkg): # The following comes from --help on groupadd from shadow parser = myArgumentParser(prog='groupadd') parser.add_argument("-f", "--force", help="exit successfully if the group already exists, and cancel -g if the GID is already used", action="store_true") @@ -177,17 +220,18 @@ def update_useradd_static_config(d): parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults") parser.add_argument("-o", "--non-unique", help="allow to create groups with duplicate (non-unique) GID", action="store_true") parser.add_argument("-p", "--password", metavar="PASSWORD", help="use this encrypted password for the new group") + parser.add_argument("-P", "--clear-password", metavar="CLEAR_PASSWORD", help="use this clear password for the new group") parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into") parser.add_argument("-r", "--system", help="create a system account", action="store_true") parser.add_argument("GROUP", help="Group name of the new group") # Return a list of configuration files based on either the default # files/group or the contents of USERADD_GID_TABLES - # paths are resulved via BBPATH + # paths are resolved via BBPATH def get_group_list(d): str = "" - bbpath = d.getVar('BBPATH', True) - group_tables = d.getVar('USERADD_GID_TABLES', True) + bbpath = d.getVar('BBPATH') + group_tables = d.getVar('USERADD_GID_TABLES') if not group_tables: group_tables = 'files/group' for conf_file in group_tables.split(): @@ -195,17 +239,18 @@ def update_useradd_static_config(d): return str newparams = [] + groups = None for param in re.split('''[ \t]*;[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params): param = param.strip() if not param: continue try: # If we're processing multiple lines, we could have left over values here... - gaargs = parser.parse_args(re.split('''[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', param)) + gaargs = parser.parse_args(re.split('''[ \t]+(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', param)) except: - raise bb.build.FuncFailed("%s: Unable to parse arguments for GROUPADD_PARAM_%s: '%s'" % (d.getVar('PN', True), pkg, param)) + bb.fatal("%s: Unable to parse arguments for GROUPADD_PARAM_%s: '%s'" % (d.getVar('PN'), pkg, param)) - # Need to iterate over layers and open the right file(s) + # Read all group files specified in USERADD_GID_TABLES or files/group # Use the standard group layout: # groupname:password:group_id:group_members # @@ -214,55 +259,97 @@ def update_useradd_static_config(d): # # Note: similar to the passwd file, the 'password' filed is ignored # Note: group_members is ignored, group members must be configured with the GROUPMEMS_PARAM - for conf in get_group_list(d).split()[::-1]: - if os.path.exists(conf): - f = open(conf, "r") - for line in f: - if line.startswith('#'): - continue - field = line.rstrip().split(":") - if field[0] == gaargs.GROUP and field[2]: - if gaargs.gid and (gaargs.gid != field[2]): - bb.warn("%s: Changing groupname %s's gid from (%s) to (%s), verify configuration files!" % (d.getVar('PN', True), gaargs.GROUP, gaargs.gid, field[2])) - gaargs.gid = field[2] - break + if not groups: + groups = merge_files(get_group_list(d), 4) + + if gaargs.GROUP not in groups: + handle_missing_id(gaargs.GROUP, 'group', pkg) + newparams.append(param) + continue + + field = groups[gaargs.GROUP] - if d.getVar('USERADD_ERROR_DYNAMIC', True) == '1' and not (gaargs.gid and gaargs.gid.isdigit()): - #bb.error("Skipping recipe %s, package %s which adds groupname %s does not have a static gid defined." % (d.getVar('PN', True), pkg, gaargs.GROUP)) - raise bb.build.FuncFailed("%s - %s: Groupname %s does not have a static gid defined." % (d.getVar('PN', True), pkg, gaargs.GROUP)) + if field[2]: + if gaargs.gid and (gaargs.gid != field[2]): + bb.warn("%s: Changing groupname %s's gid from (%s) to (%s), verify configuration files!" % (d.getVar('PN'), gaargs.GROUP, gaargs.gid, field[2])) + gaargs.gid = field[2] + + if not gaargs.gid or not gaargs.gid.isdigit(): + handle_missing_id(gaargs.GROUP, 'group', pkg) # Reconstruct the args... newparam = ['', ' --force'][gaargs.force] newparam += ['', ' --gid %s' % gaargs.gid][gaargs.gid != None] newparam += ['', ' --key %s' % gaargs.key][gaargs.key != None] newparam += ['', ' --non-unique'][gaargs.non_unique] - newparam += ['', ' --password %s' % gaargs.password][gaargs.password != None] + if gaargs.password != None: + newparam += ['', ' --password %s' % gaargs.password][gaargs.password != None] + elif gaargs.clear_password: + newparam += ['', ' --clear-password %s' % gaargs.clear_password][gaargs.clear_password != None] newparam += ['', ' --root %s' % gaargs.root][gaargs.root != None] newparam += ['', ' --system'][gaargs.system] newparam += ' %s' % gaargs.GROUP newparams.append(newparam) - return " ;".join(newparams).strip() + return ";".join(newparams).strip() + + # The parsing of the current recipe depends on the content of + # the files listed in USERADD_UID/GID_TABLES. We need to tell bitbake + # about that explicitly to trigger re-parsing and thus re-execution of + # this code when the files change. + bbpath = d.getVar('BBPATH') + for varname, default in (('USERADD_UID_TABLES', 'files/passwd'), + ('USERADD_GID_TABLES', 'files/group')): + tables = d.getVar(varname) + if not tables: + tables = default + for conf_file in tables.split(): + bb.parse.mark_dependency(d, bb.utils.which(bbpath, conf_file)) # Load and process the users and groups, rewriting the adduser/addgroup params - useradd_packages = d.getVar('USERADD_PACKAGES', True) + useradd_packages = d.getVar('USERADD_PACKAGES') or "" for pkg in useradd_packages.split(): # Groupmems doesn't have anything we might want to change, so simply validating # is a bit of a waste -- only process useradd/groupadd - useradd_param = d.getVar('USERADD_PARAM_%s' % pkg, True) + useradd_param = d.getVar('USERADD_PARAM_%s' % pkg) if useradd_param: #bb.warn("Before: 'USERADD_PARAM_%s' - '%s'" % (pkg, useradd_param)) - d.setVar('USERADD_PARAM_%s' % pkg, rewrite_useradd(useradd_param)) - #bb.warn("After: 'USERADD_PARAM_%s' - '%s'" % (pkg, d.getVar('USERADD_PARAM_%s' % pkg, True))) + d.setVar('USERADD_PARAM_%s' % pkg, rewrite_useradd(useradd_param, True)) + #bb.warn("After: 'USERADD_PARAM_%s' - '%s'" % (pkg, d.getVar('USERADD_PARAM_%s' % pkg))) - groupadd_param = d.getVar('GROUPADD_PARAM_%s' % pkg, True) + groupadd_param = d.getVar('GROUPADD_PARAM_%s' % pkg) if groupadd_param: #bb.warn("Before: 'GROUPADD_PARAM_%s' - '%s'" % (pkg, groupadd_param)) - d.setVar('GROUPADD_PARAM_%s' % pkg, rewrite_groupadd(groupadd_param)) - #bb.warn("After: 'GROUPADD_PARAM_%s' - '%s'" % (pkg, d.getVar('GROUPADD_PARAM_%s' % pkg, True))) + d.setVar('GROUPADD_PARAM_%s' % pkg, rewrite_groupadd(groupadd_param, True)) + #bb.warn("After: 'GROUPADD_PARAM_%s' - '%s'" % (pkg, d.getVar('GROUPADD_PARAM_%s' % pkg))) + + # Load and process extra users and groups, rewriting only adduser/addgroup params + pkg = d.getVar('PN') + extrausers = d.getVar('EXTRA_USERS_PARAMS') or "" + + #bb.warn("Before: 'EXTRA_USERS_PARAMS' - '%s'" % (d.getVar('EXTRA_USERS_PARAMS'))) + new_extrausers = [] + for cmd in re.split('''[ \t]*;[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', extrausers): + cmd = cmd.strip() + if not cmd: + continue + + if re.match('''useradd (.*)''', cmd): + useradd_param = re.match('''useradd (.*)''', cmd).group(1) + useradd_param = rewrite_useradd(useradd_param, False) + cmd = 'useradd %s' % useradd_param + elif re.match('''groupadd (.*)''', cmd): + groupadd_param = re.match('''groupadd (.*)''', cmd).group(1) + groupadd_param = rewrite_groupadd(groupadd_param, False) + cmd = 'groupadd %s' % groupadd_param + + new_extrausers.append(cmd) + new_extrausers.append('') + d.setVar('EXTRA_USERS_PARAMS', ';'.join(new_extrausers)) + #bb.warn("After: 'EXTRA_USERS_PARAMS' - '%s'" % (d.getVar('EXTRA_USERS_PARAMS'))) python __anonymous() { @@ -270,7 +357,7 @@ python __anonymous() { and not bb.data.inherits_class('native', d): try: update_useradd_static_config(d) - except bb.build.FuncFailed as f: - bb.debug(1, "Skipping recipe %s: %s" % (d.getVar('PN', True), f)) + except NotImplementedError as f: + bb.debug(1, "Skipping recipe %s: %s" % (d.getVar('PN'), f)) raise bb.parse.SkipPackage(f) } diff --git a/meta/classes/useradd.bbclass b/meta/classes/useradd.bbclass index 4577e56f5a..4373677bd6 100644 --- a/meta/classes/useradd.bbclass +++ b/meta/classes/useradd.bbclass @@ -3,11 +3,8 @@ inherit useradd_base # base-passwd-cross provides the default passwd and group files in the # target sysroot, and shadow -native and -sysroot provide the utilities # and support files needed to add and modify user and group accounts -DEPENDS_append = "${USERADDDEPENDS}" -USERADDDEPENDS = " base-files shadow-native shadow-sysroot shadow" -USERADDDEPENDS_class-cross = "" -USERADDDEPENDS_class-native = "" -USERADDDEPENDS_class-nativesdk = "" +DEPENDS_append_class-target = " base-files shadow-native shadow-sysroot shadow base-passwd" +PACKAGE_WRITE_DEPS += "shadow-native" # This preinstall function can be run in four different contexts: # @@ -50,65 +47,80 @@ fi # Perform group additions first, since user additions may depend # on these groups existing -if test "x$GROUPADD_PARAM" != "x"; then +if test "x`echo $GROUPADD_PARAM | tr -d '[:space:]'`" != "x"; then echo "Running groupadd commands..." # Invoke multiple instances of groupadd for parameter lists # separated by ';' - opts=`echo "$GROUPADD_PARAM" | cut -d ';' -f 1` - remaining=`echo "$GROUPADD_PARAM" | cut -d ';' -f 2-` + opts=`echo "$GROUPADD_PARAM" | cut -d ';' -f 1 | sed -e 's#[ \t]*$##'` + remaining=`echo "$GROUPADD_PARAM" | cut -d ';' -f 2- | sed -e 's#[ \t]*$##'` while test "x$opts" != "x"; do - perform_groupadd "$SYSROOT" "$OPT $opts" 10 + perform_groupadd "$SYSROOT" "$OPT $opts" if test "x$opts" = "x$remaining"; then break fi - opts=`echo "$remaining" | cut -d ';' -f 1` - remaining=`echo "$remaining" | cut -d ';' -f 2-` + opts=`echo "$remaining" | cut -d ';' -f 1 | sed -e 's#[ \t]*$##'` + remaining=`echo "$remaining" | cut -d ';' -f 2- | sed -e 's#[ \t]*$##'` done fi -if test "x$USERADD_PARAM" != "x"; then +if test "x`echo $USERADD_PARAM | tr -d '[:space:]'`" != "x"; then echo "Running useradd commands..." # Invoke multiple instances of useradd for parameter lists # separated by ';' - opts=`echo "$USERADD_PARAM" | cut -d ';' -f 1` - remaining=`echo "$USERADD_PARAM" | cut -d ';' -f 2-` + opts=`echo "$USERADD_PARAM" | cut -d ';' -f 1 | sed -e 's#[ \t]*$##'` + remaining=`echo "$USERADD_PARAM" | cut -d ';' -f 2- | sed -e 's#[ \t]*$##'` while test "x$opts" != "x"; do - perform_useradd "$SYSROOT" "$OPT $opts" 10 + perform_useradd "$SYSROOT" "$OPT $opts" if test "x$opts" = "x$remaining"; then break fi - opts=`echo "$remaining" | cut -d ';' -f 1` - remaining=`echo "$remaining" | cut -d ';' -f 2-` + opts=`echo "$remaining" | cut -d ';' -f 1 | sed -e 's#[ \t]*$##'` + remaining=`echo "$remaining" | cut -d ';' -f 2- | sed -e 's#[ \t]*$##'` done fi -if test "x$GROUPMEMS_PARAM" != "x"; then +if test "x`echo $GROUPMEMS_PARAM | tr -d '[:space:]'`" != "x"; then echo "Running groupmems commands..." # Invoke multiple instances of groupmems for parameter lists # separated by ';' - opts=`echo "$GROUPMEMS_PARAM" | cut -d ';' -f 1` - remaining=`echo "$GROUPMEMS_PARAM" | cut -d ';' -f 2-` + opts=`echo "$GROUPMEMS_PARAM" | cut -d ';' -f 1 | sed -e 's#[ \t]*$##'` + remaining=`echo "$GROUPMEMS_PARAM" | cut -d ';' -f 2- | sed -e 's#[ \t]*$##'` while test "x$opts" != "x"; do - perform_groupmems "$SYSROOT" "$OPT $opts" 10 + perform_groupmems "$SYSROOT" "$OPT $opts" if test "x$opts" = "x$remaining"; then break fi - opts=`echo "$remaining" | cut -d ';' -f 1` - remaining=`echo "$remaining" | cut -d ';' -f 2-` + opts=`echo "$remaining" | cut -d ';' -f 1 | sed -e 's#[ \t]*$##'` + remaining=`echo "$remaining" | cut -d ';' -f 2- | sed -e 's#[ \t]*$##'` done fi } useradd_sysroot () { - # Pseudo may (do_install) or may not (do_populate_sysroot_setscene) be running + # Pseudo may (do_prepare_recipe_sysroot) or may not (do_populate_sysroot_setscene) be running # at this point so we're explicit about the environment so pseudo can load if # not already present. - export PSEUDO="${FAKEROOTENV} PSEUDO_LOCALSTATEDIR=${STAGING_DIR_TARGET}${localstatedir}/pseudo ${STAGING_DIR_NATIVE}${bindir}/pseudo" + export PSEUDO="${FAKEROOTENV} PSEUDO_LOCALSTATEDIR=${STAGING_DIR_TARGET}${localstatedir}/pseudo ${PSEUDO_SYSROOT}${bindir_native}/pseudo" # Explicitly set $D since it isn't set to anything - # before do_install + # before do_prepare_recipe_sysroot D=${STAGING_DIR_TARGET} + # base-passwd's postinst may not have run yet in which case we'll get called later, just exit. + # Beware that in some cases we might see the fake pseudo passwd here, in which case we also must + # exit. + if [ ! -f $D${sysconfdir}/passwd ] || + grep -q this-is-the-pseudo-passwd $D${sysconfdir}/passwd; then + exit 0 + fi + + # It is also possible we may be in a recipe which doesn't have useradd dependencies and hence the + # useradd/groupadd tools are unavailable. If there is no dependency, we assume we don't want to + # create users in the sysroot + if ! command -v useradd; then + exit 0 + fi + # Add groups and users defined for all recipe packages GROUPADD_PARAM="${@get_all_cmd_params(d, 'groupadd')}" USERADD_PARAM="${@get_all_cmd_params(d, 'useradd')}" @@ -120,40 +132,53 @@ useradd_sysroot () { useradd_preinst } -useradd_sysroot_sstate () { - if [ "${BB_CURRENTTASK}" = "package_setscene" -o "${BB_CURRENTTASK}" = "populate_sysroot_setscene" ] - then - useradd_sysroot - fi +python useradd_sysroot_sstate () { + task = d.getVar("BB_CURRENTTASK") + if task == "package_setscene": + bb.build.exec_func("useradd_sysroot", d) + elif task == "prepare_recipe_sysroot": + # Used to update this recipe's own sysroot so the user/groups are available to do_install + scriptfile = d.expand("${RECIPE_SYSROOT}${bindir}/postinst-useradd-${PN}") + bb.utils.mkdirhier(os.path.dirname(scriptfile)) + with open(scriptfile, 'w') as script: + script.write("#!/bin/sh\n") + bb.data.emit_func("useradd_sysroot", script, d) + script.write("useradd_sysroot\n") + os.chmod(scriptfile, 0o755) + bb.build.exec_func("useradd_sysroot", d) + elif task == "populate_sysroot": + # Used when installed in dependent task sysroots + scriptfile = d.expand("${SYSROOT_DESTDIR}${bindir}/postinst-useradd-${PN}") + bb.utils.mkdirhier(os.path.dirname(scriptfile)) + with open(scriptfile, 'w') as script: + script.write("#!/bin/sh\n") + bb.data.emit_func("useradd_sysroot", script, d) + script.write("useradd_sysroot\n") + os.chmod(scriptfile, 0o755) } -do_install[prefuncs] += "${SYSROOTFUNC}" -SYSROOTFUNC = "useradd_sysroot" -SYSROOTFUNC_class-cross = "" -SYSROOTFUNC_class-native = "" -SYSROOTFUNC_class-nativesdk = "" -SSTATEPREINSTFUNCS += "${SYSROOTPOSTFUNC}" -SYSROOTPOSTFUNC = "useradd_sysroot_sstate" -SYSROOTPOSTFUNC_class-cross = "" -SYSROOTPOSTFUNC_class-native = "" -SYSROOTPOSTFUNC_class-nativesdk = "" - -USERADDSETSCENEDEPS = "${MLPREFIX}base-passwd:do_populate_sysroot_setscene pseudo-native:do_populate_sysroot_setscene shadow-native:do_populate_sysroot_setscene ${MLPREFIX}shadow-sysroot:do_populate_sysroot_setscene" -USERADDSETSCENEDEPS_class-cross = "" -USERADDSETSCENEDEPS_class-native = "" -USERADDSETSCENEDEPS_class-nativesdk = "" +do_prepare_recipe_sysroot[postfuncs] += "${SYSROOTFUNC}" +SYSROOTFUNC_class-target = "useradd_sysroot_sstate" +SYSROOTFUNC = "" + +SYSROOT_PREPROCESS_FUNCS += "${SYSROOTFUNC}" + +SSTATEPREINSTFUNCS_append_class-target = " useradd_sysroot_sstate" + do_package_setscene[depends] += "${USERADDSETSCENEDEPS}" do_populate_sysroot_setscene[depends] += "${USERADDSETSCENEDEPS}" +USERADDSETSCENEDEPS_class-target = "${MLPREFIX}base-passwd:do_populate_sysroot_setscene pseudo-native:do_populate_sysroot_setscene shadow-native:do_populate_sysroot_setscene ${MLPREFIX}shadow-sysroot:do_populate_sysroot_setscene" +USERADDSETSCENEDEPS = "" # Recipe parse-time sanity checks def update_useradd_after_parse(d): - useradd_packages = d.getVar('USERADD_PACKAGES', True) + useradd_packages = d.getVar('USERADD_PACKAGES') if not useradd_packages: - raise bb.build.FuncFailed("%s inherits useradd but doesn't set USERADD_PACKAGES" % d.getVar('FILE', False)) + bb.fatal("%s inherits useradd but doesn't set USERADD_PACKAGES" % d.getVar('FILE', False)) for pkg in useradd_packages.split(): - if not d.getVar('USERADD_PARAM_%s' % pkg, True) and not d.getVar('GROUPADD_PARAM_%s' % pkg, True) and not d.getVar('GROUPMEMS_PARAM_%s' % pkg, True): + if not d.getVar('USERADD_PARAM_%s' % pkg) and not d.getVar('GROUPADD_PARAM_%s' % pkg) and not d.getVar('GROUPMEMS_PARAM_%s' % pkg): bb.fatal("%s inherits useradd but doesn't set USERADD_PARAM, GROUPADD_PARAM or GROUPMEMS_PARAM for package %s" % (d.getVar('FILE', False), pkg)) python __anonymous() { @@ -170,11 +195,11 @@ def get_all_cmd_params(d, cmd_type): param_type = cmd_type.upper() + "_PARAM_%s" params = [] - useradd_packages = d.getVar('USERADD_PACKAGES', True) or "" + useradd_packages = d.getVar('USERADD_PACKAGES') or "" for pkg in useradd_packages.split(): - param = d.getVar(param_type % pkg, True) + param = d.getVar(param_type % pkg) if param: - params.append(param) + params.append(param.rstrip(" ;")) return "; ".join(params) @@ -188,20 +213,20 @@ fakeroot python populate_packages_prepend () { required to execute on the target. Not doing so may cause useradd preinst to be invoked twice, causing unwanted warnings. """ - preinst = d.getVar('pkg_preinst_%s' % pkg, True) or d.getVar('pkg_preinst', True) + preinst = d.getVar('pkg_preinst_%s' % pkg) or d.getVar('pkg_preinst') if not preinst: preinst = '#!/bin/sh\n' preinst += 'bbnote () {\n\techo "NOTE: $*"\n}\n' preinst += 'bbwarn () {\n\techo "WARNING: $*"\n}\n' preinst += 'bbfatal () {\n\techo "ERROR: $*"\n\texit 1\n}\n' - preinst += 'perform_groupadd () {\n%s}\n' % d.getVar('perform_groupadd', True) - preinst += 'perform_useradd () {\n%s}\n' % d.getVar('perform_useradd', True) - preinst += 'perform_groupmems () {\n%s}\n' % d.getVar('perform_groupmems', True) - preinst += d.getVar('useradd_preinst', True) + preinst += 'perform_groupadd () {\n%s}\n' % d.getVar('perform_groupadd') + preinst += 'perform_useradd () {\n%s}\n' % d.getVar('perform_useradd') + preinst += 'perform_groupmems () {\n%s}\n' % d.getVar('perform_groupmems') + preinst += d.getVar('useradd_preinst') d.setVar('pkg_preinst_%s' % pkg, preinst) # RDEPENDS setup - rdepends = d.getVar("RDEPENDS_%s" % pkg, True) or "" + rdepends = d.getVar("RDEPENDS_%s" % pkg) or "" rdepends += ' ' + d.getVar('MLPREFIX', False) + 'base-passwd' rdepends += ' ' + d.getVar('MLPREFIX', False) + 'shadow' # base-files is where the default /etc/skel is packaged @@ -212,7 +237,7 @@ fakeroot python populate_packages_prepend () { # to packages specified by USERADD_PACKAGES if not bb.data.inherits_class('nativesdk', d) \ and not bb.data.inherits_class('native', d): - useradd_packages = d.getVar('USERADD_PACKAGES', True) or "" + useradd_packages = d.getVar('USERADD_PACKAGES') or "" for pkg in useradd_packages.split(): update_useradd_package(pkg) } diff --git a/meta/classes/useradd_base.bbclass b/meta/classes/useradd_base.bbclass index 802f3a1085..551c82c322 100644 --- a/meta/classes/useradd_base.bbclass +++ b/meta/classes/useradd_base.bbclass @@ -4,7 +4,7 @@ # The following functions basically have similar logic. # *) Perform necessary checks before invoking the actual command -# *) Invoke the actual command, make retries if necessary +# *) Invoke the actual command with flock # *) Error out if an error occurs. # Note that before invoking these functions, make sure the global variable @@ -13,26 +13,15 @@ perform_groupadd () { local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing groupadd with [$opts] and $retries times of retry" + bbnote "${PN}: Performing groupadd with [$opts]" local groupname=`echo "$opts" | awk '{ print $NF }'` local group_exists="`grep "^$groupname:" $rootdir/etc/group || true`" if test "x$group_exists" = "x"; then - local count=0 - while true; do - eval $PSEUDO groupadd $opts || true - group_exists="`grep "^$groupname:" $rootdir/etc/group || true`" - if test "x$group_exists" = "x"; then - bbwarn "${PN}: groupadd command did not succeed. Retrying..." - else - break - fi - count=`expr $count + 1` - if test $count = $retries; then - bbfatal "${PN}: Tried running groupadd command $retries times without success, giving up" - fi - sleep $count - done + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO groupadd \$opts\" || true + group_exists="`grep "^$groupname:" $rootdir/etc/group || true`" + if test "x$group_exists" = "x"; then + bbfatal "${PN}: groupadd command did not succeed." + fi else bbnote "${PN}: group $groupname already exists, not re-creating it" fi @@ -41,26 +30,15 @@ perform_groupadd () { perform_useradd () { local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing useradd with [$opts] and $retries times of retry" + bbnote "${PN}: Performing useradd with [$opts]" local username=`echo "$opts" | awk '{ print $NF }'` local user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" if test "x$user_exists" = "x"; then - local count=0 - while true; do - eval $PSEUDO useradd $opts || true - user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" - if test "x$user_exists" = "x"; then - bbwarn "${PN}: useradd command did not succeed. Retrying..." - else - break - fi - count=`expr $count + 1` - if test $count = $retries; then - bbfatal "${PN}: Tried running useradd command $retries times without success, giving up" - fi - sleep $count - done + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO useradd \$opts\" || true + user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" + if test "x$user_exists" = "x"; then + bbfatal "${PN}: useradd command did not succeed." + fi else bbnote "${PN}: user $username already exists, not re-creating it" fi @@ -69,102 +47,63 @@ perform_useradd () { perform_groupmems () { local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing groupmems with [$opts] and $retries times of retry" + bbnote "${PN}: Performing groupmems with [$opts]" local groupname=`echo "$opts" | awk '{ for (i = 1; i < NF; i++) if ($i == "-g" || $i == "--group") print $(i+1) }'` local username=`echo "$opts" | awk '{ for (i = 1; i < NF; i++) if ($i == "-a" || $i == "--add") print $(i+1) }'` bbnote "${PN}: Running groupmems command with group $groupname and user $username" - # groupmems fails if /etc/gshadow does not exist - local gshadow="" - if [ -f $rootdir${sysconfdir}/gshadow ]; then - gshadow="yes" - else - gshadow="no" - touch $rootdir${sysconfdir}/gshadow - fi local mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*" $rootdir/etc/group || true`" if test "x$mem_exists" = "x"; then - local count=0 - while true; do - eval $PSEUDO groupmems $opts || true - mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*" $rootdir/etc/group || true`" - if test "x$mem_exists" = "x"; then - bbwarn "${PN}: groupmems command did not succeed. Retrying..." - else - break - fi - count=`expr $count + 1` - if test $count = $retries; then - if test "x$gshadow" = "xno"; then - rm -f $rootdir${sysconfdir}/gshadow - rm -f $rootdir${sysconfdir}/gshadow- - fi - bbfatal "${PN}: Tried running groupmems command $retries times without success, giving up" - fi - sleep $count - done + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO groupmems \$opts\" || true + mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*" $rootdir/etc/group || true`" + if test "x$mem_exists" = "x"; then + bbfatal "${PN}: groupmems command did not succeed." + fi else - bbwarn "${PN}: group $groupname already contains $username, not re-adding it" - fi - if test "x$gshadow" = "xno"; then - rm -f $rootdir${sysconfdir}/gshadow - rm -f $rootdir${sysconfdir}/gshadow- + bbnote "${PN}: group $groupname already contains $username, not re-adding it" fi } perform_groupdel () { local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing groupdel with [$opts] and $retries times of retry" + bbnote "${PN}: Performing groupdel with [$opts]" local groupname=`echo "$opts" | awk '{ print $NF }'` local group_exists="`grep "^$groupname:" $rootdir/etc/group || true`" + if test "x$group_exists" != "x"; then - local count=0 - while true; do - eval $PSEUDO groupdel $opts || true + local awk_input='BEGIN {FS=":"}; $1=="'$groupname'" { print $3 }' + local groupid=`echo "$awk_input" | awk -f- $rootdir/etc/group` + local awk_check_users='BEGIN {FS=":"}; $4=="'$groupid'" {print $1}' + local other_users=`echo "$awk_check_users" | awk -f- $rootdir/etc/passwd` + + if test "x$other_users" = "x"; then + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO groupdel \$opts\" || true group_exists="`grep "^$groupname:" $rootdir/etc/group || true`" if test "x$group_exists" != "x"; then - bbwarn "${PN}: groupdel command did not succeed. Retrying..." - else - break + bbfatal "${PN}: groupdel command did not succeed." fi - count=`expr $count + 1` - if test $count = $retries; then - bbfatal "${PN}: Tried running groupdel command $retries times without success, giving up" - fi - sleep $count - done + else + bbnote "${PN}: '$groupname' is primary group for users '$other_users', not removing it" + fi else - bbwarn "${PN}: group $groupname doesn't exist, not removing it" + bbnote "${PN}: group $groupname doesn't exist, not removing it" fi } perform_userdel () { local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing userdel with [$opts] and $retries times of retry" + bbnote "${PN}: Performing userdel with [$opts]" local username=`echo "$opts" | awk '{ print $NF }'` local user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" if test "x$user_exists" != "x"; then - local count=0 - while true; do - eval $PSEUDO userdel $opts || true - user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" - if test "x$user_exists" != "x"; then - bbwarn "${PN}: userdel command did not succeed. Retrying..." - else - break - fi - count=`expr $count + 1` - if test $count = $retries; then - bbfatal "${PN}: Tried running userdel command $retries times without success, giving up" - fi - sleep $count - done + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO userdel \$opts\" || true + user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" + if test "x$user_exists" != "x"; then + bbfatal "${PN}: userdel command did not succeed." + fi else - bbwarn "${PN}: user $username doesn't exist, not removing it" + bbnote "${PN}: user $username doesn't exist, not removing it" fi } @@ -174,25 +113,14 @@ perform_groupmod () { set +e local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing groupmod with [$opts] and $retries times of retry" + bbnote "${PN}: Performing groupmod with [$opts]" local groupname=`echo "$opts" | awk '{ print $NF }'` local group_exists="`grep "^$groupname:" $rootdir/etc/group || true`" if test "x$group_exists" != "x"; then - local count=0 - while true; do - eval $PSEUDO groupmod $opts - if test $? != 0; then - bbwarn "${PN}: groupmod command did not succeed. Retrying..." - else - break - fi - count=`expr $count + 1` - if test $count = $retries; then - bbfatal "${PN}: Tried running groupmod command $retries times without success, giving up" - fi - sleep $count - done + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO groupmod \$opts\" + if test $? != 0; then + bbwarn "${PN}: groupmod command did not succeed." + fi else bbwarn "${PN}: group $groupname doesn't exist, unable to modify it" fi @@ -204,25 +132,14 @@ perform_usermod () { set +e local rootdir="$1" local opts="$2" - local retries="$3" - bbnote "${PN}: Performing usermod with [$opts] and $retries times of retry" + bbnote "${PN}: Performing usermod with [$opts]" local username=`echo "$opts" | awk '{ print $NF }'` local user_exists="`grep "^$username:" $rootdir/etc/passwd || true`" if test "x$user_exists" != "x"; then - local count=0 - while true; do - eval $PSEUDO usermod $opts - if test $? != 0; then - bbwarn "${PN}: usermod command did not succeed. Retrying..." - else - break - fi - count=`expr $count + 1` - if test $count = $retries; then - bbfatal "${PN}: Tried running usermod command $retries times without success, giving up" - fi - sleep $count - done + eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO usermod \$opts\" + if test $? != 0; then + bbfatal "${PN}: usermod command did not succeed." + fi else bbwarn "${PN}: user $username doesn't exist, unable to modify it" fi diff --git a/meta/classes/utility-tasks.bbclass b/meta/classes/utility-tasks.bbclass index e817b899a6..587bfd4ab5 100644 --- a/meta/classes/utility-tasks.bbclass +++ b/meta/classes/utility-tasks.bbclass @@ -28,29 +28,26 @@ python do_clean() { bb.note("Removing " + dir) oe.path.remove(dir) - dir = "%s.*" % bb.data.expand(d.getVar('STAMP', False), d) + dir = "%s.*" % d.getVar('STAMP') bb.note("Removing " + dir) oe.path.remove(dir) - for f in (d.getVar('CLEANFUNCS', True) or '').split(): + for f in (d.getVar('CLEANFUNCS') or '').split(): bb.build.exec_func(f, d) } addtask checkuri do_checkuri[nostamp] = "1" python do_checkuri() { - src_uri = (d.getVar('SRC_URI', True) or "").split() + src_uri = (d.getVar('SRC_URI') or "").split() if len(src_uri) == 0: return - localdata = bb.data.createCopy(d) - bb.data.update_data(localdata) - try: - fetcher = bb.fetch2.Fetch(src_uri, localdata) + fetcher = bb.fetch2.Fetch(src_uri, d) fetcher.checkstatus() - except bb.fetch2.BBFetchException, e: - raise bb.build.FuncFailed(e) + except bb.fetch2.BBFetchException as e: + bb.fatal(str(e)) } addtask checkuriall after do_checkuri diff --git a/meta/classes/utils.bbclass b/meta/classes/utils.bbclass index 81b92cb5eb..96463ab323 100644 --- a/meta/classes/utils.bbclass +++ b/meta/classes/utils.bbclass @@ -24,6 +24,7 @@ def base_version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d): return oe.utils.version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d) def base_contains(variable, checkvalues, truevalue, falsevalue, d): + bb.note('base_contains is deprecated, please use bb.utils.contains instead.') return bb.utils.contains(variable, checkvalues, truevalue, falsevalue, d) def base_both_contain(variable1, variable2, checkvalue, d): @@ -40,9 +41,9 @@ def oe_filter_out(f, str, d): def machine_paths(d): """List any existing machine specific filespath directories""" - machine = d.getVar("MACHINE", True) - filespathpkg = d.getVar("FILESPATHPKG", True).split(":") - for basepath in d.getVar("FILESPATHBASE", True).split(":"): + machine = d.getVar("MACHINE") + filespathpkg = d.getVar("FILESPATHPKG").split(":") + for basepath in d.getVar("FILESPATHBASE").split(":"): for pkgpath in filespathpkg: machinepath = os.path.join(basepath, pkgpath, machine) if os.path.isdir(machinepath): @@ -51,7 +52,7 @@ def machine_paths(d): def is_machine_specific(d): """Determine whether the current recipe is machine specific""" machinepaths = set(machine_paths(d)) - srcuri = d.getVar("SRC_URI", True).split() + srcuri = d.getVar("SRC_URI").split() for url in srcuri: fetcher = bb.fetch2.Fetch([srcuri], d) if url.startswith("file://"): @@ -61,15 +62,18 @@ def is_machine_specific(d): oe_soinstall() { # Purpose: Install shared library file and # create the necessary links - # Example: - # - # oe_ - # - #bbnote installing shared library $1 to $2 - # + # Example: oe_soinstall libfoo.so.1.2.3 ${D}${libdir} libname=`basename $1` + case "$libname" in + *.so) + bbfatal "oe_soinstall: Shared library must haved versioned filename (e.g. libfoo.so.1.2.3)" + ;; + esac install -m 755 $1 $2/$libname sonamelink=`${HOST_PREFIX}readelf -d $1 |grep 'Library soname:' |sed -e 's/.*\[\(.*\)\].*/\1/'` + if [ -z $sonamelink ]; then + bbfatal "oe_soinstall: $libname is missing ELF tag 'SONAME'." + fi solink=`echo $libname | sed -e 's/\.so\..*/.so/'` ln -sf $libname $2/$sonamelink ln -sf $libname $2/$solink @@ -248,7 +252,7 @@ oe_machinstall() { create_cmdline_wrapper () { # Create a wrapper script where commandline options are needed # - # These are useful to work around relocation issues, by passing extra options + # These are useful to work around relocation issues, by passing extra options # to a program # # Usage: create_cmdline_wrapper FILENAME <extra-options> @@ -260,10 +264,17 @@ create_cmdline_wrapper () { mv $cmd $cmd.real cmdname=`basename $cmd` + dirname=`dirname $cmd` + cmdoptions=$@ + if [ "${base_prefix}" != "" ]; then + relpath=`python3 -c "import os; print(os.path.relpath('${D}${base_prefix}', '$dirname'))"` + cmdoptions=`echo $@ | sed -e "s:${base_prefix}:\\$realdir/$relpath:g"` + fi cat <<END >$cmd #!/bin/bash realpath=\`readlink -fn \$0\` -exec -a \`dirname \$realpath\`/$cmdname \`dirname \$realpath\`/$cmdname.real $@ "\$@" +realdir=\`dirname \$realpath\` +exec -a \`dirname \$realpath\`/$cmdname \`dirname \$realpath\`/$cmdname.real $cmdoptions "\$@" END chmod +x $cmd } @@ -283,10 +294,17 @@ create_wrapper () { mv $cmd $cmd.real cmdname=`basename $cmd` + dirname=`dirname $cmd` + exportstring=$@ + if [ "${base_prefix}" != "" ]; then + relpath=`python3 -c "import os; print(os.path.relpath('${D}${base_prefix}', '$dirname'))"` + exportstring=`echo $@ | sed -e "s:${base_prefix}:\\$realdir/$relpath:g"` + fi cat <<END >$cmd #!/bin/bash realpath=\`readlink -fn \$0\` -export $@ +realdir=\`dirname \$realpath\` +export $exportstring exec -a \`dirname \$realpath\`/$cmdname \`dirname \$realpath\`/$cmdname.real "\$@" END chmod +x $cmd @@ -302,8 +320,8 @@ hardlinkdir () { def check_app_exists(app, d): - app = d.expand(app) - path = d.getVar('PATH', d, True) + app = d.expand(app).strip() + path = d.getVar('PATH') return bool(bb.utils.which(path, app)) def explode_deps(s): @@ -311,25 +329,25 @@ def explode_deps(s): def base_set_filespath(path, d): filespath = [] - extrapaths = (d.getVar("FILESEXTRAPATHS", True) or "") + extrapaths = (d.getVar("FILESEXTRAPATHS") or "") # Remove default flag which was used for checking extrapaths = extrapaths.replace("__default:", "") # Don't prepend empty strings to the path list if extrapaths != "": path = extrapaths.split(":") + path # The ":" ensures we have an 'empty' override - overrides = (":" + (d.getVar("FILESOVERRIDES", True) or "")).split(":") + overrides = (":" + (d.getVar("FILESOVERRIDES") or "")).split(":") overrides.reverse() for o in overrides: for p in path: - if p != "": + if p != "": filespath.append(os.path.join(p, o)) return ":".join(filespath) def extend_variants(d, var, extend, delim=':'): """Return a string of all bb class extend variants for the given extend""" variants = [] - whole = d.getVar(var, True) or "" + whole = d.getVar(var) or "" for ext in whole.split(): eext = ext.split(delim) if len(eext) > 1 and eext[0] == extend: @@ -337,7 +355,7 @@ def extend_variants(d, var, extend, delim=':'): return " ".join(variants) def multilib_pkg_extend(d, pkg): - variants = (d.getVar("MULTILIB_VARIANTS", True) or "").split() + variants = (d.getVar("MULTILIB_VARIANTS") or "").split() if not variants: return pkg pkgs = pkg @@ -345,24 +363,27 @@ def multilib_pkg_extend(d, pkg): pkgs = pkgs + " " + v + "-" + pkg return pkgs +def get_multilib_datastore(variant, d): + localdata = bb.data.createCopy(d) + overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant + localdata.setVar("OVERRIDES", overrides) + localdata.setVar("MLPREFIX", variant + "-") + return localdata + def all_multilib_tune_values(d, var, unique = True, need_split = True, delim = ' '): """Return a string of all ${var} in all multilib tune configuration""" values = [] - value = d.getVar(var, True) or "" + value = d.getVar(var) or "" if value != "": if need_split: for item in value.split(delim): values.append(item) else: values.append(value) - variants = d.getVar("MULTILIB_VARIANTS", True) or "" + variants = d.getVar("MULTILIB_VARIANTS") or "" for item in variants.split(): - localdata = bb.data.createCopy(d) - overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + item - localdata.setVar("OVERRIDES", overrides) - localdata.setVar("MLPREFIX", item + "-") - bb.data.update_data(localdata) - value = localdata.getVar(var, True) or "" + localdata = get_multilib_datastore(item, d) + value = localdata.getVar(var) or "" if value != "": if need_split: for item in value.split(delim): @@ -378,3 +399,45 @@ def all_multilib_tune_values(d, var, unique = True, need_split = True, delim = ' else: ret = values return " ".join(ret) + +def all_multilib_tune_list(vars, d): + """ + Return a list of ${VAR} for each variable VAR in vars from each + multilib tune configuration. + Is safe to be called from a multilib recipe/context as it can + figure out the original tune and remove the multilib overrides. + """ + values = {} + for v in vars: + values[v] = [] + + localdata = bb.data.createCopy(d) + overrides = localdata.getVar("OVERRIDES", False).split(":") + newoverrides = [] + for o in overrides: + if not o.startswith("virtclass-multilib-"): + newoverrides.append(o) + localdata.setVar("OVERRIDES", ":".join(newoverrides)) + localdata.setVar("MLPREFIX", "") + origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL") + if origdefault: + localdata.setVar("DEFAULTTUNE", origdefault) + values['ml'] = [''] + for v in vars: + values[v].append(localdata.getVar(v)) + variants = d.getVar("MULTILIB_VARIANTS") or "" + for item in variants.split(): + localdata = get_multilib_datastore(item, d) + values[v].append(localdata.getVar(v)) + values['ml'].append(item) + return values + +# If the user hasn't set up their name/email, set some defaults +check_git_config() { + if ! git config user.email > /dev/null ; then + git config --local user.email "${PATCH_GIT_USER_EMAIL}" + fi + if ! git config user.name > /dev/null ; then + git config --local user.name "${PATCH_GIT_USER_NAME}" + fi +} diff --git a/meta/classes/vala.bbclass b/meta/classes/vala.bbclass index 9ff664ac41..615eb379ad 100644 --- a/meta/classes/vala.bbclass +++ b/meta/classes/vala.bbclass @@ -16,3 +16,9 @@ FILES_${PN}-dev += "\ ${datadir}/vala/vapi/*.deps \ ${datadir}/gir-1.0 \ " + +# Remove vapigen.m4 that is bundled with tarballs +# because it does not yet have our cross-compile fixes +do_configure_prepend() { + rm -f ${S}/m4/vapigen.m4 +} diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass index 3a221e7082..c4698e910a 100644 --- a/meta/classes/waf.bbclass +++ b/meta/classes/waf.bbclass @@ -1,9 +1,36 @@ +# avoids build breaks when using no-static-libs.inc +DISABLE_STATIC = "" + +EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" + +def get_waf_parallel_make(d): + pm = d.getVar('PARALLEL_MAKE') + if pm: + # look for '-j' and throw other options (e.g. '-l') away + # because they might have different meaning in bjam + pm = pm.split() + while pm: + v = None + opt = pm.pop(0) + if opt == '-j': + v = pm.pop(0) + elif opt.startswith('-j'): + v = opt[2:].strip() + else: + v = None + + if v: + v = min(64, int(v)) + return '-j' + str(v) + + return "" + waf_do_configure() { ${S}/waf configure --prefix=${prefix} ${EXTRA_OECONF} } waf_do_compile() { - ${S}/waf build ${PARALLEL_MAKE} + ${S}/waf build ${@get_waf_parallel_make(d)} } waf_do_install() { |