diff options
Diffstat (limited to 'meta/classes/buildhistory.bbclass')
| -rw-r--r-- | meta/classes/buildhistory.bbclass | 162 |
1 files changed, 103 insertions, 59 deletions
diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass index 6995d06950..f543bb73d6 100644 --- a/meta/classes/buildhistory.bbclass +++ b/meta/classes/buildhistory.bbclass @@ -47,6 +47,11 @@ sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" # then the value added to SSTATEPOSTINSTFUNCS: SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" +# Similarly for our function that gets the output signatures +SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs" +sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs" +SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs" + # All items excepts those listed here will be removed from a recipe's # build history directory by buildhistory_emit_pkghistory(). This is # necessary because some of these items (package directories, files that @@ -57,22 +62,25 @@ SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" # class. BUILDHISTORY_PRESERVE = "latest latest_srcrev" +PATCH_GIT_USER_EMAIL ?= "buildhistory@oe" +PATCH_GIT_USER_NAME ?= "OpenEmbedded" + # # Write out metadata about this package for comparison when writing future packages # python buildhistory_emit_pkghistory() { - if not d.getVar('BB_CURRENTTASK', True) in ['packagedata', 'packagedata_setscene']: + if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: return 0 - if not "package" in (d.getVar('BUILDHISTORY_FEATURES', True) or "").split(): + if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): return 0 import re import json import errno - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) - oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') + oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') class RecipeInfo: def __init__(self, name): @@ -83,6 +91,7 @@ python buildhistory_emit_pkghistory() { self.depends = "" self.packages = "" self.srcrev = "" + self.layer = "" class PackageInfo: @@ -179,12 +188,13 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) - pn = d.getVar('PN', True) - pe = d.getVar('PE', True) or "0" - pv = d.getVar('PV', True) - pr = d.getVar('PR', True) + pn = d.getVar('PN') + pe = d.getVar('PE') or "0" + pv = d.getVar('PV') + pr = d.getVar('PR') + layer = bb.utils.get_file_layer(d.getVar('FILE', True), d) - pkgdata_dir = d.getVar('PKGDATA_DIR', True) + pkgdata_dir = d.getVar('PKGDATA_DIR') packages = "" try: with open(os.path.join(pkgdata_dir, pn)) as f: @@ -200,7 +210,7 @@ python buildhistory_emit_pkghistory() { raise packagelist = packages.split() - preserve = d.getVar('BUILDHISTORY_PRESERVE', True).split() + preserve = d.getVar('BUILDHISTORY_PRESERVE').split() if not os.path.exists(pkghistdir): bb.utils.mkdirhier(pkghistdir) else: @@ -220,11 +230,12 @@ python buildhistory_emit_pkghistory() { rcpinfo.pe = pe rcpinfo.pv = pv rcpinfo.pr = pr - rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS', True) or "")) + rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or "")) rcpinfo.packages = packages + rcpinfo.layer = layer write_recipehistory(rcpinfo, d) - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') for pkg in packagelist: pkgdata = {} with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: @@ -286,11 +297,34 @@ python buildhistory_emit_pkghistory() { bb.build.exec_func("buildhistory_list_pkg_files", d) } +python buildhistory_emit_outputsigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output') + bb.utils.mkdirhier(taskoutdir) + currenttask = d.getVar('BB_CURRENTTASK') + pn = d.getVar('PN') + taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask)) + + cwd = os.getcwd() + filesigs = {} + for root, _, files in os.walk(cwd): + for fname in files: + if fname == 'fixmepath': + continue + fullpath = os.path.join(root, fname) + filesigs[os.path.relpath(fullpath, cwd)] = bb.utils.sha256_file(fullpath) + with open(taskfile, 'w') as f: + for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]): + f.write('%s %s\n' % (fpath, fsig)) +} + def write_recipehistory(rcpinfo, d): bb.debug(2, "Writing recipe history") - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') infofile = os.path.join(pkghistdir, "latest") with open(infofile, "w") as f: @@ -300,12 +334,13 @@ def write_recipehistory(rcpinfo, d): f.write(u"PR = %s\n" % rcpinfo.pr) f.write(u"DEPENDS = %s\n" % rcpinfo.depends) f.write(u"PACKAGES = %s\n" % rcpinfo.packages) + f.write(u"LAYER = %s\n" % rcpinfo.layer) def write_pkghistory(pkginfo, d): bb.debug(2, "Writing package history for package %s" % pkginfo.name) - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') pkgpath = os.path.join(pkghistdir, pkginfo.name) if not os.path.exists(pkgpath): @@ -318,16 +353,14 @@ def write_pkghistory(pkginfo, d): f.write(u"PV = %s\n" % pkginfo.pv) f.write(u"PR = %s\n" % pkginfo.pr) - pkgvars = {} - pkgvars['PKG'] = pkginfo.pkg if pkginfo.pkg != pkginfo.name else '' - pkgvars['PKGE'] = pkginfo.pkge if pkginfo.pkge != pkginfo.pe else '' - pkgvars['PKGV'] = pkginfo.pkgv if pkginfo.pkgv != pkginfo.pv else '' - pkgvars['PKGR'] = pkginfo.pkgr if pkginfo.pkgr != pkginfo.pr else '' - for pkgvar in pkgvars: - val = pkgvars[pkgvar] - if val: - f.write(u"%s = %s\n" % (pkgvar, val)) - + if pkginfo.pkg != pkginfo.name: + f.write(u"PKG = %s\n" % pkginfo.pkg) + if pkginfo.pkge != pkginfo.pe: + f.write(u"PKGE = %s\n" % pkginfo.pkge) + if pkginfo.pkgv != pkginfo.pv: + f.write(u"PKGV = %s\n" % pkginfo.pkgv) + if pkginfo.pkgr != pkginfo.pr: + f.write(u"PKGR = %s\n" % pkginfo.pkgr) f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides) f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends) f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends) @@ -368,7 +401,7 @@ def buildhistory_list_installed(d, rootfs_type="image"): pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target") for output_type, output_file in process_list: - output_file_full = os.path.join(d.getVar('WORKDIR', True), output_file) + output_file_full = os.path.join(d.getVar('WORKDIR'), output_file) with open(output_file_full, 'w') as output: output.write(format_pkg_list(pkgs, output_type)) @@ -413,7 +446,7 @@ buildhistory_get_installed() { # Produce installed package sizes list oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp - cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB " $1}' | sort -n -r > $1/installed-package-sizes.txt + cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt rm $1/installed-package-sizes.tmp # We're now done with the cache, delete it @@ -549,7 +582,9 @@ END python buildhistory_get_extra_sdkinfo() { import operator import math - if d.getVar('BB_CURRENTTASK', True) == 'populate_sdk_ext': + + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \ + "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): tasksizes = {} filesizes = {} for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')): @@ -561,21 +596,25 @@ python buildhistory_get_extra_sdkinfo() { tasksizes[task] = origtotal + fsize filesizes[fn] = fsize with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f: - filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1), reverse=True) + filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1, 0), reverse=True) for fn, size in filesizes_sorted: f.write('%10d KiB %s\n' % (size, fn)) with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f: - tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1), reverse=True) + tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1, 0), reverse=True) for task, size in tasksizes_sorted: f.write('%10d KiB %s\n' % (size, task)) } # By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of # unneeded packages but before the removal of packaging files -ROOTFS_POSTUNINSTALL_COMMAND += " buildhistory_list_installed_image ;\ - buildhistory_get_image_installed ; " +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;" +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed" -IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; " +IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo" # We want these to be the last run so that we get called after complementary package installation POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;" @@ -589,11 +628,21 @@ POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_insta SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " +python buildhistory_write_sigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + # Create sigs file + if hasattr(bb.parse.siggen, 'dump_siglist'): + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task') + bb.utils.mkdirhier(taskoutdir) + bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt')) +} + def buildhistory_get_build_id(d): - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" localdata = bb.data.createCopy(d) - bb.data.update_data(localdata) statuslines = [] for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): g = globals() @@ -604,12 +653,12 @@ def buildhistory_get_build_id(d): if flines: statuslines.extend(flines) - statusheader = d.getVar('BUILDCFG_HEADER', True) + statusheader = d.getVar('BUILDCFG_HEADER') return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) def buildhistory_get_metadata_revs(d): # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want - layers = (d.getVar("BBLAYERS", True) or "").split() + layers = (d.getVar("BBLAYERS") or "").split() medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \ base_get_metadata_git_branch(i, None).strip(), \ base_get_metadata_git_revision(i, None)) \ @@ -621,7 +670,7 @@ def outputvars(vars, listvars, d): listvars = listvars.split() ret = "" for var in vars: - value = d.getVar(var, True) or "" + value = d.getVar(var) or "" if var in listvars: # Squash out spaces value = oe.utils.squashspaces(value) @@ -629,19 +678,19 @@ def outputvars(vars, listvars, d): return ret.rstrip('\n') def buildhistory_get_imagevars(d): - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(imagevars, listvars, d) def buildhistory_get_sdkvars(d): - if d.getVar('BB_WORKERCONTEXT', True) != '1': + if d.getVar('BB_WORKERCONTEXT') != '1': return "" sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" - if d.getVar('BB_CURRENTTASK', True) == 'populate_sdk_ext': + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext': # Extensible SDK uses some additional variables - sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS" + sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN" listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST" return outputvars(sdkvars, listvars, d) @@ -710,15 +759,9 @@ END git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true git tag -f build-minus-1 > /dev/null 2>&1 || true fi - # If the user hasn't set up their name/email, set some defaults - # just for this repo (otherwise the commit will fail with older - # versions of git) - if ! git config user.email > /dev/null ; then - git config --local user.email "buildhistory@${DISTRO}" - fi - if ! git config user.name > /dev/null ; then - git config --local user.name "buildhistory" - fi + + check_git_config + # Check if there are new/changed files to commit (other than metadata-revs) repostatus=`git status --porcelain | grep -v " metadata-revs$"` HOSTNAME=`hostname 2>/dev/null || echo unknown` @@ -740,16 +783,16 @@ END } python buildhistory_eventhandler() { - if e.data.getVar('BUILDHISTORY_FEATURES', True).strip(): - reset = e.data.getVar("BUILDHISTORY_RESET", True) - olddir = e.data.getVar("BUILDHISTORY_OLD_DIR", True) + if e.data.getVar('BUILDHISTORY_FEATURES').strip(): + reset = e.data.getVar("BUILDHISTORY_RESET") + olddir = e.data.getVar("BUILDHISTORY_OLD_DIR") if isinstance(e, bb.event.BuildStarted): if reset: import shutil # Clean up after potentially interrupted build. if os.path.isdir(olddir): shutil.rmtree(olddir) - rootdir = e.data.getVar("BUILDHISTORY_DIR", True) + rootdir = e.data.getVar("BUILDHISTORY_DIR") entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ] bb.utils.mkdirhier(olddir) for entry in entries: @@ -759,8 +802,9 @@ python buildhistory_eventhandler() { if reset: import shutil shutil.rmtree(olddir) - if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1": + if e.data.getVar("BUILDHISTORY_COMMIT") == "1": bb.note("Writing buildhistory") + bb.build.exec_func("buildhistory_write_sigs", d) localdata = bb.data.createCopy(e.data) localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures)) interrupted = getattr(e, '_interrupted', 0) @@ -779,7 +823,7 @@ def _get_srcrev_values(d): """ scms = [] - fetcher = bb.fetch.Fetch(d.getVar('SRC_URI', True).split(), d) + fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d) urldata = fetcher.ud for u in urldata: if urldata[u].method.supports_srcrev(): @@ -811,7 +855,7 @@ def _get_srcrev_values(d): do_fetch[postfuncs] += "write_srcrev" do_fetch[vardepsexclude] += "write_srcrev" python write_srcrev() { - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') srcrevfile = os.path.join(pkghistdir, 'latest_srcrev') srcrevs, tag_srcrevs = _get_srcrev_values(d) @@ -838,12 +882,12 @@ python write_srcrev() { f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev)) f.write('SRCREV_%s = "%s"\n' % (name, srcrev)) else: - f.write('SRCREV = "%s"\n' % srcrevs.values()) + f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values()))) if len(tag_srcrevs) > 0: for name, srcrev in tag_srcrevs.items(): f.write('# tag_%s = "%s"\n' % (name, srcrev)) if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev: - pkg = d.getVar('PN', True) + pkg = d.getVar('PN') bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev)) else: |