diff options
Diffstat (limited to 'meta/classes/buildhistory.bbclass')
| -rw-r--r-- | meta/classes/buildhistory.bbclass | 793 |
1 files changed, 630 insertions, 163 deletions
diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass index e121088d59..f543bb73d6 100644 --- a/meta/classes/buildhistory.bbclass +++ b/meta/classes/buildhistory.bbclass @@ -3,38 +3,84 @@ # # Based in part on testlab.bbclass and packagehistory.bbclass # -# Copyright (C) 2011 Intel Corporation +# Copyright (C) 2011-2016 Intel Corporation # Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org> # -BUILDHISTORY_FEATURES ?= "image package" -BUILDHISTORY_DIR ?= "${TMPDIR}/buildhistory" +BUILDHISTORY_FEATURES ?= "image package sdk" +BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}" BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" + +# Setting this to non-empty will remove the old content of the buildhistory as part of +# the current bitbake invocation and replace it with information about what was built +# during the build. +# +# This is meant to be used in continuous integration (CI) systems when invoking bitbake +# for full world builds. The effect in that case is that information about packages +# that no longer get build also gets removed from the buildhistory, which is not +# the case otherwise. +# +# The advantage over manually cleaning the buildhistory outside of bitbake is that +# the "version-going-backwards" check still works. When relying on that, be careful +# about failed world builds: they will lead to incomplete information in the +# buildhistory because information about packages that could not be built will +# also get removed. A CI system should handle that by discarding the buildhistory +# of failed builds. +# +# The expected usage is via auto.conf, but passing via the command line also works +# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 +BUILDHISTORY_RESET ?= "" + +BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}" +BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" +BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}" +BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group" +BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf" BUILDHISTORY_COMMIT ?= "0" BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>" BUILDHISTORY_PUSH_REPO ?= "" -# Must inherit package first before changing PACKAGEFUNCS -inherit package -PACKAGEFUNCS += "buildhistory_emit_pkghistory" +SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory" +# We want to avoid influencing the signatures of sstate tasks - first the function itself: +sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" +# then the value added to SSTATEPOSTINSTFUNCS: +SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" + +# Similarly for our function that gets the output signatures +SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs" +sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs" +SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs" + +# All items excepts those listed here will be removed from a recipe's +# build history directory by buildhistory_emit_pkghistory(). This is +# necessary because some of these items (package directories, files that +# we no longer emit) might be obsolete. +# +# When extending build history, derive your class from buildhistory.bbclass +# and extend this list here with the additional files created by the derived +# class. +BUILDHISTORY_PRESERVE = "latest latest_srcrev" -# We don't want to force a rerun of do_package for everything -# if the buildhistory_emit_pkghistory function or any of the -# variables it refers to changes -do_package[vardepsexclude] += "buildhistory_emit_pkghistory" +PATCH_GIT_USER_EMAIL ?= "buildhistory@oe" +PATCH_GIT_USER_NAME ?= "OpenEmbedded" # -# Called during do_package to write out metadata about this package -# for comparision when writing future packages +# Write out metadata about this package for comparison when writing future packages # python buildhistory_emit_pkghistory() { - import re + if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: + return 0 - if not "package" in (d.getVar('BUILDHISTORY_FEATURES', True) or "").split(): + if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): return 0 - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + import re + import json + import errno + + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') + oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') class RecipeInfo: def __init__(self, name): @@ -44,6 +90,9 @@ python buildhistory_emit_pkghistory() { self.pr = "r0" self.depends = "" self.packages = "" + self.srcrev = "" + self.layer = "" + class PackageInfo: def __init__(self, name): @@ -58,8 +107,12 @@ python buildhistory_emit_pkghistory() { self.pkgr = "" self.size = 0 self.depends = "" + self.rprovides = "" self.rdepends = "" self.rrecommends = "" + self.rsuggests = "" + self.rreplaces = "" + self.rconflicts = "" self.files = "" self.filelist = "" # Variables that need to be written to their own separate file @@ -67,19 +120,11 @@ python buildhistory_emit_pkghistory() { # Should check PACKAGES here to see if anything removed - def getpkgvar(pkg, var): - val = bb.data.getVar('%s_%s' % (var, pkg), d, 1) - if val: - return val - val = bb.data.getVar('%s' % (var), d, 1) - - return val - def readPackageInfo(pkg, histfile): pkginfo = PackageInfo(pkg) with open(histfile, "r") as f: for line in f: - lns = line.split('=') + lns = line.split('=', 1) name = lns[0].strip() value = lns[1].strip(" \t\r\n").strip('"') if name == "PE": @@ -96,12 +141,20 @@ python buildhistory_emit_pkghistory() { pkginfo.pkgv = value elif name == "PKGR": pkginfo.pkgr = value + elif name == "RPROVIDES": + pkginfo.rprovides = value elif name == "RDEPENDS": pkginfo.rdepends = value elif name == "RRECOMMENDS": pkginfo.rrecommends = value + elif name == "RSUGGESTS": + pkginfo.rsuggests = value + elif name == "RREPLACES": + pkginfo.rreplaces = value + elif name == "RCONFLICTS": + pkginfo.rconflicts = value elif name == "PKGSIZE": - pkginfo.size = long(value) + pkginfo.size = int(value) elif name == "FILES": pkginfo.files = value elif name == "FILELIST": @@ -119,13 +172,13 @@ python buildhistory_emit_pkghistory() { def getlastpkgversion(pkg): try: - histfile = os.path.join(pkghistdir, pkg, "latest") + histfile = os.path.join(oldpkghistdir, pkg, "latest") return readPackageInfo(pkg, histfile) except EnvironmentError: return None def sortpkglist(string): - pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+ [^ )]+\))?', string, 0) + pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0) pkglist = [p.group(0) for p in pkgiter] pkglist.sort() return ' '.join(pkglist) @@ -135,38 +188,67 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) - pn = d.getVar('PN', True) - pe = d.getVar('PE', True) or "0" - pv = d.getVar('PV', True) - pr = d.getVar('PR', True) - packages = squashspaces(d.getVar('PACKAGES', True)) + pn = d.getVar('PN') + pe = d.getVar('PE') or "0" + pv = d.getVar('PV') + pr = d.getVar('PR') + layer = bb.utils.get_file_layer(d.getVar('FILE', True), d) + + pkgdata_dir = d.getVar('PKGDATA_DIR') + packages = "" + try: + with open(os.path.join(pkgdata_dir, pn)) as f: + for line in f.readlines(): + if line.startswith('PACKAGES: '): + packages = oe.utils.squashspaces(line.split(': ', 1)[1]) + break + except IOError as e: + if e.errno == errno.ENOENT: + # Probably a -cross recipe, just ignore + return 0 + else: + raise packagelist = packages.split() + preserve = d.getVar('BUILDHISTORY_PRESERVE').split() if not os.path.exists(pkghistdir): - os.makedirs(pkghistdir) + bb.utils.mkdirhier(pkghistdir) else: # Remove files for packages that no longer exist for item in os.listdir(pkghistdir): - if item != "latest": + if item not in preserve: if item not in packagelist: - subdir = os.path.join(pkghistdir, item) - for subfile in os.listdir(subdir): - os.unlink(os.path.join(subdir, subfile)) - os.rmdir(subdir) + itempath = os.path.join(pkghistdir, item) + if os.path.isdir(itempath): + for subfile in os.listdir(itempath): + os.unlink(os.path.join(itempath, subfile)) + os.rmdir(itempath) + else: + os.unlink(itempath) rcpinfo = RecipeInfo(pn) rcpinfo.pe = pe rcpinfo.pv = pv rcpinfo.pr = pr - rcpinfo.depends = sortlist(squashspaces(d.getVar('DEPENDS', True) or "")) + rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or "")) rcpinfo.packages = packages + rcpinfo.layer = layer write_recipehistory(rcpinfo, d) - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') for pkg in packagelist: - pkge = getpkgvar(pkg, 'PKGE') or "0" - pkgv = getpkgvar(pkg, 'PKGV') - pkgr = getpkgvar(pkg, 'PKGR') + pkgdata = {} + with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: + for line in f.readlines(): + item = line.rstrip('\n').split(': ', 1) + key = item[0] + if key.endswith('_' + pkg): + key = key[:-len(pkg)-1] + pkgdata[key] = item[1] + + pkge = pkgdata.get('PKGE', '0') + pkgv = pkgdata['PKGV'] + pkgr = pkgdata['PKGR'] # # Find out what the last version was # Make sure the version did not decrease @@ -178,86 +260,119 @@ python buildhistory_emit_pkghistory() { last_pkgr = lastversion.pkgr r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr)) if r < 0: - bb.error("Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)) + msg = "Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr) + package_qa_handle_error("version-going-backwards", msg, d) pkginfo = PackageInfo(pkg) # Apparently the version can be different on a per-package basis (see Python) - pkginfo.pe = getpkgvar(pkg, 'PE') or "0" - pkginfo.pv = getpkgvar(pkg, 'PV') - pkginfo.pr = getpkgvar(pkg, 'PR') - pkginfo.pkg = getpkgvar(pkg, 'PKG') or pkg + pkginfo.pe = pkgdata.get('PE', '0') + pkginfo.pv = pkgdata['PV'] + pkginfo.pr = pkgdata['PR'] + pkginfo.pkg = pkgdata['PKG'] pkginfo.pkge = pkge pkginfo.pkgv = pkgv pkginfo.pkgr = pkgr - pkginfo.rdepends = sortpkglist(squashspaces(getpkgvar(pkg, 'RDEPENDS') or "")) - pkginfo.rrecommends = sortpkglist(squashspaces(getpkgvar(pkg, 'RRECOMMENDS') or "")) - pkginfo.files = squashspaces(getpkgvar(pkg, 'FILES') or "") + pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', ""))) + pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', ""))) + pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', ""))) + pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(pkgdata.get('RSUGGESTS', ""))) + pkginfo.rreplaces = sortpkglist(oe.utils.squashspaces(pkgdata.get('RREPLACES', ""))) + pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(pkgdata.get('RCONFLICTS', ""))) + pkginfo.files = oe.utils.squashspaces(pkgdata.get('FILES', "")) for filevar in pkginfo.filevars: - pkginfo.filevars[filevar] = getpkgvar(pkg, filevar) + pkginfo.filevars[filevar] = pkgdata.get(filevar, "") # Gather information about packaged files - pkgdestpkg = os.path.join(pkgdest, pkg) - filelist = [] - pkginfo.size = 0 - for root, dirs, files in os.walk(pkgdestpkg): - relpth = os.path.relpath(root, pkgdestpkg) - for f in files: - fstat = os.lstat(os.path.join(root, f)) - pkginfo.size += fstat.st_size - filelist.append(os.sep + os.path.join(relpth, f)) + val = pkgdata.get('FILES_INFO', '') + dictval = json.loads(val) + filelist = list(dictval.keys()) filelist.sort() pkginfo.filelist = " ".join(filelist) + pkginfo.size = int(pkgdata['PKGSIZE']) + write_pkghistory(pkginfo, d) + + # Create files-in-<package-name>.txt files containing a list of files of each recipe's package + bb.build.exec_func("buildhistory_list_pkg_files", d) +} + +python buildhistory_emit_outputsigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output') + bb.utils.mkdirhier(taskoutdir) + currenttask = d.getVar('BB_CURRENTTASK') + pn = d.getVar('PN') + taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask)) + + cwd = os.getcwd() + filesigs = {} + for root, _, files in os.walk(cwd): + for fname in files: + if fname == 'fixmepath': + continue + fullpath = os.path.join(root, fname) + filesigs[os.path.relpath(fullpath, cwd)] = bb.utils.sha256_file(fullpath) + with open(taskfile, 'w') as f: + for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]): + f.write('%s %s\n' % (fpath, fsig)) } def write_recipehistory(rcpinfo, d): bb.debug(2, "Writing recipe history") - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') infofile = os.path.join(pkghistdir, "latest") with open(infofile, "w") as f: if rcpinfo.pe != "0": - f.write("PE = %s\n" % rcpinfo.pe) - f.write("PV = %s\n" % rcpinfo.pv) - f.write("PR = %s\n" % rcpinfo.pr) - f.write("DEPENDS = %s\n" % rcpinfo.depends) - f.write("PACKAGES = %s\n" % rcpinfo.packages) + f.write(u"PE = %s\n" % rcpinfo.pe) + f.write(u"PV = %s\n" % rcpinfo.pv) + f.write(u"PR = %s\n" % rcpinfo.pr) + f.write(u"DEPENDS = %s\n" % rcpinfo.depends) + f.write(u"PACKAGES = %s\n" % rcpinfo.packages) + f.write(u"LAYER = %s\n" % rcpinfo.layer) def write_pkghistory(pkginfo, d): bb.debug(2, "Writing package history for package %s" % pkginfo.name) - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') pkgpath = os.path.join(pkghistdir, pkginfo.name) if not os.path.exists(pkgpath): - os.makedirs(pkgpath) + bb.utils.mkdirhier(pkgpath) infofile = os.path.join(pkgpath, "latest") with open(infofile, "w") as f: if pkginfo.pe != "0": - f.write("PE = %s\n" % pkginfo.pe) - f.write("PV = %s\n" % pkginfo.pv) - f.write("PR = %s\n" % pkginfo.pr) - - pkgvars = {} - pkgvars['PKG'] = pkginfo.pkg if pkginfo.pkg != pkginfo.name else '' - pkgvars['PKGE'] = pkginfo.pkge if pkginfo.pkge != pkginfo.pe else '' - pkgvars['PKGV'] = pkginfo.pkgv if pkginfo.pkgv != pkginfo.pv else '' - pkgvars['PKGR'] = pkginfo.pkgr if pkginfo.pkgr != pkginfo.pr else '' - for pkgvar in pkgvars: - val = pkgvars[pkgvar] - if val: - f.write("%s = %s\n" % (pkgvar, val)) - - f.write("RDEPENDS = %s\n" % pkginfo.rdepends) - f.write("RRECOMMENDS = %s\n" % pkginfo.rrecommends) - f.write("PKGSIZE = %d\n" % pkginfo.size) - f.write("FILES = %s\n" % pkginfo.files) - f.write("FILELIST = %s\n" % pkginfo.filelist) + f.write(u"PE = %s\n" % pkginfo.pe) + f.write(u"PV = %s\n" % pkginfo.pv) + f.write(u"PR = %s\n" % pkginfo.pr) + + if pkginfo.pkg != pkginfo.name: + f.write(u"PKG = %s\n" % pkginfo.pkg) + if pkginfo.pkge != pkginfo.pe: + f.write(u"PKGE = %s\n" % pkginfo.pkge) + if pkginfo.pkgv != pkginfo.pv: + f.write(u"PKGV = %s\n" % pkginfo.pkgv) + if pkginfo.pkgr != pkginfo.pr: + f.write(u"PKGR = %s\n" % pkginfo.pkgr) + f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides) + f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends) + f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends) + if pkginfo.rsuggests: + f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests) + if pkginfo.rreplaces: + f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces) + if pkginfo.rconflicts: + f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts) + f.write(u"PKGSIZE = %d\n" % pkginfo.size) + f.write(u"FILES = %s\n" % pkginfo.files) + f.write(u"FILELIST = %s\n" % pkginfo.filelist) for filevar in pkginfo.filevars: filevarpath = os.path.join(pkgpath, "latest.%s" % filevar) @@ -269,68 +384,162 @@ def write_pkghistory(pkginfo, d): if os.path.exists(filevarpath): os.unlink(filevarpath) +# +# rootfs_type can be: image, sdk_target, sdk_host +# +def buildhistory_list_installed(d, rootfs_type="image"): + from oe.rootfs import image_list_installed_packages + from oe.sdk import sdk_list_installed_packages + from oe.utils import format_pkg_list -buildhistory_get_image_installed() { - # Anything requiring the use of the packaging system should be done in here - # in case the packaging files are going to be removed for this image + process_list = [('file', 'bh_installed_pkgs.txt'),\ + ('deps', 'bh_installed_pkgs_deps.txt')] - if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then - return - fi + if rootfs_type == "image": + pkgs = image_list_installed_packages(d) + else: + pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target") - mkdir -p ${BUILDHISTORY_DIR_IMAGE} + for output_type, output_file in process_list: + output_file_full = os.path.join(d.getVar('WORKDIR'), output_file) - # Get list of installed packages - pkgcache="${BUILDHISTORY_DIR_IMAGE}/installed-packages.tmp" - list_installed_packages file | sort > $pkgcache + with open(output_file_full, 'w') as output: + output.write(format_pkg_list(pkgs, output_type)) - cat $pkgcache | awk '{ print $1 }' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt - cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt +python buildhistory_list_installed_image() { + buildhistory_list_installed(d) +} + +python buildhistory_list_installed_sdk_target() { + buildhistory_list_installed(d, "sdk_target") +} + +python buildhistory_list_installed_sdk_host() { + buildhistory_list_installed(d, "sdk_host") +} + +buildhistory_get_installed() { + mkdir -p $1 + + # Get list of installed packages + pkgcache="$1/installed-packages.tmp" + cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt + + cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt + if [ -s $pkgcache ] ; then + cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt + else + printf "" > $1/installed-packages.txt + fi # Produce dependency graph - # First, filter out characters that cause issues for dot - rootfs_list_installed_depends | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' > ${BUILDHISTORY_DIR_IMAGE}/depends.tmp + # First, quote each name to handle characters that cause issues for dot + sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp && \ + rm ${WORKDIR}/bh_installed_pkgs_deps.txt # Change delimiter from pipe to -> and set style for recommend lines - sed -i -e 's:|: -> :' -e 's:\[REC\]:[style=dotted]:' -e 's:$:;:' ${BUILDHISTORY_DIR_IMAGE}/depends.tmp + sed -i -e 's:|: -> :' -e 's:"\[REC\]":[style=dotted]:' -e 's:$:;:' $1/depends.tmp # Add header, sorted and de-duped contents and footer and then delete the temp file - echo -e "digraph depends {\n node [shape=plaintext]" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot - cat ${BUILDHISTORY_DIR_IMAGE}/depends.tmp | sort | uniq >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot - echo "}" >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot - rm ${BUILDHISTORY_DIR_IMAGE}/depends.tmp + printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot + cat $1/depends.tmp | sort | uniq >> $1/depends.dot + echo "}" >> $1/depends.dot + rm $1/depends.tmp # Produce installed package sizes list - echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp - cat $pkgcache | while read pkg pkgfile - do - if [ -f $pkgfile ] ; then - pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'` - echo $pkgsize $pkg >> ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp - fi - done - cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.txt - rm ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp + oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp + cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt + rm $1/installed-package-sizes.tmp # We're now done with the cache, delete it rm $pkgcache - # Produce some cut-down graphs (for readability) - grep -v kernel_image ${BUILDHISTORY_DIR_IMAGE}/depends.dot | grep -v kernel_2 | grep -v kernel_3 > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot - grep -v libc6 ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot | grep -v libgcc > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot - grep -v update_ ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot - grep -v kernel_module ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate-nomodules.dot + if [ "$2" != "sdk" ] ; then + # Produce some cut-down graphs (for readability) + grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot + grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot + grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot + grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot + fi + + # add complementary package information + if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then + cp ${WORKDIR}/complementary_pkgs.txt $1 + fi } -buildhistory_get_imageinfo() { - if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then +buildhistory_get_image_installed() { + # Anything requiring the use of the packaging system should be done in here + # in case the packaging files are going to be removed for this image + + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then return fi - # List the files in the image, but exclude date/time etc. + buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE} +} + +buildhistory_get_sdk_installed() { + # Anything requiring the use of the packaging system should be done in here + # in case the packaging files are going to be removed for this SDK + + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then + return + fi + + buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk +} + +buildhistory_get_sdk_installed_host() { + buildhistory_get_sdk_installed host +} + +buildhistory_get_sdk_installed_target() { + buildhistory_get_sdk_installed target +} + +buildhistory_list_files() { + # List the files in the specified directory, but exclude date/time etc. # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo - ( cd ${IMAGE_ROOTFS} && find . -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' | sort -k5 > ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt ) + if [ "$3" = "fakeroot" ] ; then + ( cd $1 && ${FAKEROOTENV} ${FAKEROOTCMD} find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 ) + else + ( cd $1 && find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 ) + fi +} + +buildhistory_list_pkg_files() { + # Create individual files-in-package for each recipe's package + for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do + pkgname=$(basename $pkgdir) + outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname" + outfile="$outfolder/files-in-package.txt" + # Make sure the output folder exists so we can create the file + if [ ! -d $outfolder ] ; then + bbdebug 2 "Folder $outfolder does not exist, file $outfile not created" + continue + fi + buildhistory_list_files $pkgdir $outfile fakeroot + done +} + +buildhistory_get_imageinfo() { + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then + return + fi + + mkdir -p ${BUILDHISTORY_DIR_IMAGE} + buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt + + # Collect files requested in BUILDHISTORY_IMAGE_FILES + rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files + for f in ${BUILDHISTORY_IMAGE_FILES}; do + if [ -f ${IMAGE_ROOTFS}/$f ] ; then + mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f` + cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f + fi + done # Record some machine-readable meta-information about the image - echo -n > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt + printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END ${@buildhistory_get_imagevars(d)} END @@ -338,52 +547,197 @@ END echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt # Add some configuration information - echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id + echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt + + cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END +${@buildhistory_get_build_id(d)} +END +} + +buildhistory_get_sdkinfo() { + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then + return + fi + + buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt + + # Collect files requested in BUILDHISTORY_SDK_FILES + rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files + for f in ${BUILDHISTORY_SDK_FILES}; do + if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then + mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f` + cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f + fi + done - cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id <<END -${@buildhistory_get_layers(d)} + # Record some machine-readable meta-information about the SDK + printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt + cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END +${@buildhistory_get_sdkvars(d)} END + sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'` + echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt } -# By prepending we get in before the removal of packaging files -ROOTFS_POSTPROCESS_COMMAND =+ "buildhistory_get_image_installed ; " +python buildhistory_get_extra_sdkinfo() { + import operator + import math + + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \ + "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + tasksizes = {} + filesizes = {} + for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')): + for fn in files: + if fn.endswith('.tgz'): + fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024)) + task = fn.rsplit(':', 1)[1].split('_', 1)[1].split('.')[0] + origtotal = tasksizes.get(task, 0) + tasksizes[task] = origtotal + fsize + filesizes[fn] = fsize + with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f: + filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1, 0), reverse=True) + for fn, size in filesizes_sorted: + f.write('%10d KiB %s\n' % (size, fn)) + with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f: + tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1, 0), reverse=True) + for task, size in tasksizes_sorted: + f.write('%10d KiB %s\n' % (size, task)) +} -IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; " +# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of +# unneeded packages but before the removal of packaging files +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;" +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed" + +IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo" + +# We want these to be the last run so that we get called after complementary package installation +POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;" +POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;" +POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;" + +POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;" +POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;" +POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;" + +SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " +SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " + +python buildhistory_write_sigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + # Create sigs file + if hasattr(bb.parse.siggen, 'dump_siglist'): + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task') + bb.utils.mkdirhier(taskoutdir) + bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt')) +} -def buildhistory_get_layers(d): - layertext = "Configured metadata layers:\n%s\n" % '\n'.join(get_layers_branch_rev(d)) - return layertext +def buildhistory_get_build_id(d): + if d.getVar('BB_WORKERCONTEXT') != '1': + return "" + localdata = bb.data.createCopy(d) + statuslines = [] + for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): + g = globals() + if func not in g: + bb.warn("Build configuration function '%s' does not exist" % func) + else: + flines = g[func](localdata) + if flines: + statuslines.extend(flines) + + statusheader = d.getVar('BUILDCFG_HEADER') + return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) def buildhistory_get_metadata_revs(d): # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want - layers = (d.getVar("BBLAYERS", True) or "").split() + layers = (d.getVar("BBLAYERS") or "").split() medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \ base_get_metadata_git_branch(i, None).strip(), \ base_get_metadata_git_revision(i, None)) \ for i in layers] return '\n'.join(medadata_revs) - -def squashspaces(string): - import re - return re.sub("\s+", " ", string).strip() - - -def buildhistory_get_imagevars(d): - imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" - listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS" - - imagevars = imagevars.split() +def outputvars(vars, listvars, d): + vars = vars.split() listvars = listvars.split() ret = "" - for var in imagevars: - value = d.getVar(var, True) or "" + for var in vars: + value = d.getVar(var) or "" if var in listvars: # Squash out spaces - value = squashspaces(value) + value = oe.utils.squashspaces(value) ret += "%s = %s\n" % (var, value) return ret.rstrip('\n') +def buildhistory_get_imagevars(d): + if d.getVar('BB_WORKERCONTEXT') != '1': + return "" + imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" + listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" + return outputvars(imagevars, listvars, d) + +def buildhistory_get_sdkvars(d): + if d.getVar('BB_WORKERCONTEXT') != '1': + return "" + sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext': + # Extensible SDK uses some additional variables + sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN" + listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST" + return outputvars(sdkvars, listvars, d) + + +def buildhistory_get_cmdline(d): + if sys.argv[0].endswith('bin/bitbake'): + bincmd = 'bitbake' + else: + bincmd = sys.argv[0] + return '%s %s' % (bincmd, ' '.join(sys.argv[1:])) + + +buildhistory_single_commit() { + if [ "$3" = "" ] ; then + commitopts="${BUILDHISTORY_DIR}/ --allow-empty" + item="No changes" + else + commitopts="$3 metadata-revs" + item="$3" + fi + if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then + result="succeeded" + else + result="failed" + fi + case ${BUILDHISTORY_BUILD_INTERRUPTED} in + 1) + result="$result (interrupted)" + ;; + 2) + result="$result (force interrupted)" + ;; + esac + commitmsgfile=`mktemp` + cat > $commitmsgfile << END +$item: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2 + +cmd: $1 + +result: $result + +metadata revisions: +END + cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile + git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null + rm $commitmsgfile +} buildhistory_commit() { if [ ! -d ${BUILDHISTORY_DIR} ] ; then @@ -398,32 +752,145 @@ END ( cd ${BUILDHISTORY_DIR}/ # Initialise the repo if necessary - if [ ! -d .git ] ; then + if [ ! -e .git ] ; then git init -q + else + git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true + git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true + git tag -f build-minus-1 > /dev/null 2>&1 || true fi + + check_git_config + # Check if there are new/changed files to commit (other than metadata-revs) repostatus=`git status --porcelain | grep -v " metadata-revs$"` + HOSTNAME=`hostname 2>/dev/null || echo unknown` + CMDLINE="${@buildhistory_get_cmdline(d)}" if [ "$repostatus" != "" ] ; then - git add . - HOSTNAME=`hostname 2>/dev/null || echo unknown` + git add -A . # porcelain output looks like "?? packages/foo/bar" # Ensure we commit metadata-revs with the first commit for entry in `echo "$repostatus" | awk '{print $2}' | awk -F/ '{print $1}' | sort | uniq` ; do - git commit $entry metadata-revs -m "$entry: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $HOSTNAME" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null + buildhistory_single_commit "$CMDLINE" "$HOSTNAME" "$entry" done - if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then - git push -q ${BUILDHISTORY_PUSH_REPO} - fi + git gc --auto --quiet else - git commit ${BUILDHISTORY_DIR}/ --allow-empty -m "No changes: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $HOSTNAME" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null + buildhistory_single_commit "$CMDLINE" "$HOSTNAME" + fi + if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then + git push -q ${BUILDHISTORY_PUSH_REPO} fi) || true } python buildhistory_eventhandler() { - if isinstance(e, bb.event.BuildCompleted): - if e.data.getVar('BUILDHISTORY_FEATURES', True).strip(): - if e.data.getVar |