diff options
Diffstat (limited to 'meta/classes/buildhistory.bbclass')
| -rw-r--r-- | meta/classes/buildhistory.bbclass | 594 |
1 files changed, 431 insertions, 163 deletions
diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass index 080f2c79b3..f543bb73d6 100644 --- a/meta/classes/buildhistory.bbclass +++ b/meta/classes/buildhistory.bbclass @@ -3,7 +3,7 @@ # # Based in part on testlab.bbclass and packagehistory.bbclass # -# Copyright (C) 2013 Intel Corporation +# Copyright (C) 2011-2016 Intel Corporation # Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org> # @@ -11,32 +11,76 @@ BUILDHISTORY_FEATURES ?= "image package sdk" BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}" BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" -BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}/${IMAGE_BASENAME}" + +# Setting this to non-empty will remove the old content of the buildhistory as part of +# the current bitbake invocation and replace it with information about what was built +# during the build. +# +# This is meant to be used in continuous integration (CI) systems when invoking bitbake +# for full world builds. The effect in that case is that information about packages +# that no longer get build also gets removed from the buildhistory, which is not +# the case otherwise. +# +# The advantage over manually cleaning the buildhistory outside of bitbake is that +# the "version-going-backwards" check still works. When relying on that, be careful +# about failed world builds: they will lead to incomplete information in the +# buildhistory because information about packages that could not be built will +# also get removed. A CI system should handle that by discarding the buildhistory +# of failed builds. +# +# The expected usage is via auto.conf, but passing via the command line also works +# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 +BUILDHISTORY_RESET ?= "" + +BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}" +BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" +BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}" BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group" +BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf" BUILDHISTORY_COMMIT ?= "0" BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>" BUILDHISTORY_PUSH_REPO ?= "" -# Must inherit package first before changing PACKAGEFUNCS -inherit package -PACKAGEFUNCS += "buildhistory_emit_pkghistory" +SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory" +# We want to avoid influencing the signatures of sstate tasks - first the function itself: +sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" +# then the value added to SSTATEPOSTINSTFUNCS: +SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" + +# Similarly for our function that gets the output signatures +SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs" +sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs" +SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs" + +# All items excepts those listed here will be removed from a recipe's +# build history directory by buildhistory_emit_pkghistory(). This is +# necessary because some of these items (package directories, files that +# we no longer emit) might be obsolete. +# +# When extending build history, derive your class from buildhistory.bbclass +# and extend this list here with the additional files created by the derived +# class. +BUILDHISTORY_PRESERVE = "latest latest_srcrev" -# We don't want to force a rerun of do_package for everything -# if the buildhistory_emit_pkghistory function or any of the -# variables it refers to changes -do_package[vardepsexclude] += "buildhistory_emit_pkghistory" +PATCH_GIT_USER_EMAIL ?= "buildhistory@oe" +PATCH_GIT_USER_NAME ?= "OpenEmbedded" # -# Called during do_package to write out metadata about this package -# for comparision when writing future packages +# Write out metadata about this package for comparison when writing future packages # python buildhistory_emit_pkghistory() { - import re + if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: + return 0 - if not "package" in (d.getVar('BUILDHISTORY_FEATURES', True) or "").split(): + if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): return 0 - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + import re + import json + import errno + + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') + oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') class RecipeInfo: def __init__(self, name): @@ -46,10 +90,8 @@ python buildhistory_emit_pkghistory() { self.pr = "r0" self.depends = "" self.packages = "" - self.bbfile = "" - self.src_uri = "" self.srcrev = "" - self.srcrev_autorev = "" + self.layer = "" class PackageInfo: @@ -78,19 +120,11 @@ python buildhistory_emit_pkghistory() { # Should check PACKAGES here to see if anything removed - def getpkgvar(pkg, var): - val = bb.data.getVar('%s_%s' % (var, pkg), d, 1) - if val: - return val - val = bb.data.getVar('%s' % (var), d, 1) - - return val - def readPackageInfo(pkg, histfile): pkginfo = PackageInfo(pkg) with open(histfile, "r") as f: for line in f: - lns = line.split('=') + lns = line.split('=', 1) name = lns[0].strip() value = lns[1].strip(" \t\r\n").strip('"') if name == "PE": @@ -120,7 +154,7 @@ python buildhistory_emit_pkghistory() { elif name == "RCONFLICTS": pkginfo.rconflicts = value elif name == "PKGSIZE": - pkginfo.size = long(value) + pkginfo.size = int(value) elif name == "FILES": pkginfo.files = value elif name == "FILELIST": @@ -138,13 +172,13 @@ python buildhistory_emit_pkghistory() { def getlastpkgversion(pkg): try: - histfile = os.path.join(pkghistdir, pkg, "latest") + histfile = os.path.join(oldpkghistdir, pkg, "latest") return readPackageInfo(pkg, histfile) except EnvironmentError: return None def sortpkglist(string): - pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+ [^ )]+\))?', string, 0) + pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0) pkglist = [p.group(0) for p in pkgiter] pkglist.sort() return ' '.join(pkglist) @@ -154,48 +188,67 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) - pn = d.getVar('PN', True) - pe = d.getVar('PE', True) or "0" - pv = d.getVar('PV', True) - pr = d.getVar('PR', True) - - bbfile = d.getVar('BB_FILENAME', True) - src_uri = d.getVar('SRC_URI', True) - srcrev = d.getVar('SRCREV', True) - srcrev_autorev = 'yes' if d.getVar('SRCREV', False) == 'AUTOINC' else 'no' - - packages = squashspaces(d.getVar('PACKAGES', True)) + pn = d.getVar('PN') + pe = d.getVar('PE') or "0" + pv = d.getVar('PV') + pr = d.getVar('PR') + layer = bb.utils.get_file_layer(d.getVar('FILE', True), d) + + pkgdata_dir = d.getVar('PKGDATA_DIR') + packages = "" + try: + with open(os.path.join(pkgdata_dir, pn)) as f: + for line in f.readlines(): + if line.startswith('PACKAGES: '): + packages = oe.utils.squashspaces(line.split(': ', 1)[1]) + break + except IOError as e: + if e.errno == errno.ENOENT: + # Probably a -cross recipe, just ignore + return 0 + else: + raise packagelist = packages.split() + preserve = d.getVar('BUILDHISTORY_PRESERVE').split() if not os.path.exists(pkghistdir): bb.utils.mkdirhier(pkghistdir) else: # Remove files for packages that no longer exist for item in os.listdir(pkghistdir): - if item != "latest" and item != "latest_srcrev": + if item not in preserve: if item not in packagelist: - subdir = os.path.join(pkghistdir, item) - for subfile in os.listdir(subdir): - os.unlink(os.path.join(subdir, subfile)) - os.rmdir(subdir) + itempath = os.path.join(pkghistdir, item) + if os.path.isdir(itempath): + for subfile in os.listdir(itempath): + os.unlink(os.path.join(itempath, subfile)) + os.rmdir(itempath) + else: + os.unlink(itempath) rcpinfo = RecipeInfo(pn) rcpinfo.pe = pe rcpinfo.pv = pv rcpinfo.pr = pr - rcpinfo.depends = sortlist(squashspaces(d.getVar('DEPENDS', True) or "")) - rcpinfo.bbfile = bbfile - rcpinfo.src_uri = src_uri - rcpinfo.srcrev = srcrev - rcpinfo.srcrev_autorev = srcrev_autorev + rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or "")) rcpinfo.packages = packages + rcpinfo.layer = layer write_recipehistory(rcpinfo, d) - pkgdest = d.getVar('PKGDEST', True) + pkgdest = d.getVar('PKGDEST') for pkg in packagelist: - pkge = getpkgvar(pkg, 'PKGE') or "0" - pkgv = getpkgvar(pkg, 'PKGV') - pkgr = getpkgvar(pkg, 'PKGR') + pkgdata = {} + with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: + for line in f.readlines(): + item = line.rstrip('\n').split(': ', 1) + key = item[0] + if key.endswith('_' + pkg): + key = key[:-len(pkg)-1] + pkgdata[key] = item[1] + + pkge = pkgdata.get('PKGE', '0') + pkgv = pkgdata['PKGV'] + pkgr = pkgdata['PKGR'] # # Find out what the last version was # Make sure the version did not decrease @@ -212,58 +265,82 @@ python buildhistory_emit_pkghistory() { pkginfo = PackageInfo(pkg) # Apparently the version can be different on a per-package basis (see Python) - pkginfo.pe = getpkgvar(pkg, 'PE') or "0" - pkginfo.pv = getpkgvar(pkg, 'PV') - pkginfo.pr = getpkgvar(pkg, 'PR') - pkginfo.pkg = getpkgvar(pkg, 'PKG') or pkg + pkginfo.pe = pkgdata.get('PE', '0') + pkginfo.pv = pkgdata['PV'] + pkginfo.pr = pkgdata['PR'] + pkginfo.pkg = pkgdata['PKG'] pkginfo.pkge = pkge pkginfo.pkgv = pkgv pkginfo.pkgr = pkgr - pkginfo.rprovides = sortpkglist(squashspaces(getpkgvar(pkg, 'RPROVIDES') or "")) - pkginfo.rdepends = sortpkglist(squashspaces(getpkgvar(pkg, 'RDEPENDS') or "")) - pkginfo.rrecommends = sortpkglist(squashspaces(getpkgvar(pkg, 'RRECOMMENDS') or "")) - pkginfo.rsuggests = sortpkglist(squashspaces(getpkgvar(pkg, 'RSUGGESTS') or "")) - pkginfo.rreplaces = sortpkglist(squashspaces(getpkgvar(pkg, 'RREPLACES') or "")) - pkginfo.rconflicts = sortpkglist(squashspaces(getpkgvar(pkg, 'RCONFLICTS') or "")) - pkginfo.files = squashspaces(getpkgvar(pkg, 'FILES') or "") + pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', ""))) + pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', ""))) + pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', ""))) + pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(pkgdata.get('RSUGGESTS', ""))) + pkginfo.rreplaces = sortpkglist(oe.utils.squashspaces(pkgdata.get('RREPLACES', ""))) + pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(pkgdata.get('RCONFLICTS', ""))) + pkginfo.files = oe.utils.squashspaces(pkgdata.get('FILES', "")) for filevar in pkginfo.filevars: - pkginfo.filevars[filevar] = getpkgvar(pkg, filevar) + pkginfo.filevars[filevar] = pkgdata.get(filevar, "") # Gather information about packaged files - pkgdestpkg = os.path.join(pkgdest, pkg) - filelist = [] - pkginfo.size = 0 - for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) - fstat = os.lstat(f) - pkginfo.size += fstat.st_size - filelist.append(os.sep + relpth) + val = pkgdata.get('FILES_INFO', '') + dictval = json.loads(val) + filelist = list(dictval.keys()) filelist.sort() pkginfo.filelist = " ".join(filelist) + pkginfo.size = int(pkgdata['PKGSIZE']) + write_pkghistory(pkginfo, d) + + # Create files-in-<package-name>.txt files containing a list of files of each recipe's package + bb.build.exec_func("buildhistory_list_pkg_files", d) +} + +python buildhistory_emit_outputsigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output') + bb.utils.mkdirhier(taskoutdir) + currenttask = d.getVar('BB_CURRENTTASK') + pn = d.getVar('PN') + taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask)) + + cwd = os.getcwd() + filesigs = {} + for root, _, files in os.walk(cwd): + for fname in files: + if fname == 'fixmepath': + continue + fullpath = os.path.join(root, fname) + filesigs[os.path.relpath(fullpath, cwd)] = bb.utils.sha256_file(fullpath) + with open(taskfile, 'w') as f: + for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]): + f.write('%s %s\n' % (fpath, fsig)) } def write_recipehistory(rcpinfo, d): bb.debug(2, "Writing recipe history") - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') infofile = os.path.join(pkghistdir, "latest") with open(infofile, "w") as f: if rcpinfo.pe != "0": - f.write("PE = %s\n" % rcpinfo.pe) - f.write("PV = %s\n" % rcpinfo.pv) - f.write("PR = %s\n" % rcpinfo.pr) - f.write("DEPENDS = %s\n" % rcpinfo.depends) - f.write("PACKAGES = %s\n" % rcpinfo.packages) + f.write(u"PE = %s\n" % rcpinfo.pe) + f.write(u"PV = %s\n" % rcpinfo.pv) + f.write(u"PR = %s\n" % rcpinfo.pr) + f.write(u"DEPENDS = %s\n" % rcpinfo.depends) + f.write(u"PACKAGES = %s\n" % rcpinfo.packages) + f.write(u"LAYER = %s\n" % rcpinfo.layer) def write_pkghistory(pkginfo, d): bb.debug(2, "Writing package history for package %s" % pkginfo.name) - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') pkgpath = os.path.join(pkghistdir, pkginfo.name) if not os.path.exists(pkgpath): @@ -272,32 +349,30 @@ def write_pkghistory(pkginfo, d): infofile = os.path.join(pkgpath, "latest") with open(infofile, "w") as f: if pkginfo.pe != "0": - f.write("PE = %s\n" % pkginfo.pe) - f.write("PV = %s\n" % pkginfo.pv) - f.write("PR = %s\n" % pkginfo.pr) - - pkgvars = {} - pkgvars['PKG'] = pkginfo.pkg if pkginfo.pkg != pkginfo.name else '' - pkgvars['PKGE'] = pkginfo.pkge if pkginfo.pkge != pkginfo.pe else '' - pkgvars['PKGV'] = pkginfo.pkgv if pkginfo.pkgv != pkginfo.pv else '' - pkgvars['PKGR'] = pkginfo.pkgr if pkginfo.pkgr != pkginfo.pr else '' - for pkgvar in pkgvars: - val = pkgvars[pkgvar] - if val: - f.write("%s = %s\n" % (pkgvar, val)) - - f.write("RPROVIDES = %s\n" % pkginfo.rprovides) - f.write("RDEPENDS = %s\n" % pkginfo.rdepends) - f.write("RRECOMMENDS = %s\n" % pkginfo.rrecommends) + f.write(u"PE = %s\n" % pkginfo.pe) + f.write(u"PV = %s\n" % pkginfo.pv) + f.write(u"PR = %s\n" % pkginfo.pr) + + if pkginfo.pkg != pkginfo.name: + f.write(u"PKG = %s\n" % pkginfo.pkg) + if pkginfo.pkge != pkginfo.pe: + f.write(u"PKGE = %s\n" % pkginfo.pkge) + if pkginfo.pkgv != pkginfo.pv: + f.write(u"PKGV = %s\n" % pkginfo.pkgv) + if pkginfo.pkgr != pkginfo.pr: + f.write(u"PKGR = %s\n" % pkginfo.pkgr) + f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides) + f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends) + f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends) if pkginfo.rsuggests: - f.write("RSUGGESTS = %s\n" % pkginfo.rsuggests) + f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests) if pkginfo.rreplaces: - f.write("RREPLACES = %s\n" % pkginfo.rreplaces) + f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces) if pkginfo.rconflicts: - f.write("RCONFLICTS = %s\n" % pkginfo.rconflicts) - f.write("PKGSIZE = %d\n" % pkginfo.size) - f.write("FILES = %s\n" % pkginfo.files) - f.write("FILELIST = %s\n" % pkginfo.filelist) + f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts) + f.write(u"PKGSIZE = %d\n" % pkginfo.size) + f.write(u"FILES = %s\n" % pkginfo.files) + f.write(u"FILELIST = %s\n" % pkginfo.filelist) for filevar in pkginfo.filevars: filevarpath = os.path.join(pkgpath, "latest.%s" % filevar) @@ -309,13 +384,46 @@ def write_pkghistory(pkginfo, d): if os.path.exists(filevarpath): os.unlink(filevarpath) +# +# rootfs_type can be: image, sdk_target, sdk_host +# +def buildhistory_list_installed(d, rootfs_type="image"): + from oe.rootfs import image_list_installed_packages + from oe.sdk import sdk_list_installed_packages + from oe.utils import format_pkg_list + + process_list = [('file', 'bh_installed_pkgs.txt'),\ + ('deps', 'bh_installed_pkgs_deps.txt')] + + if rootfs_type == "image": + pkgs = image_list_installed_packages(d) + else: + pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target") + + for output_type, output_file in process_list: + output_file_full = os.path.join(d.getVar('WORKDIR'), output_file) + + with open(output_file_full, 'w') as output: + output.write(format_pkg_list(pkgs, output_type)) + +python buildhistory_list_installed_image() { + buildhistory_list_installed(d) +} + +python buildhistory_list_installed_sdk_target() { + buildhistory_list_installed(d, "sdk_target") +} + +python buildhistory_list_installed_sdk_host() { + buildhistory_list_installed(d, "sdk_host") +} buildhistory_get_installed() { mkdir -p $1 # Get list of installed packages pkgcache="$1/installed-packages.tmp" - list_installed_packages file | sort > $pkgcache + cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt if [ -s $pkgcache ] ; then @@ -326,7 +434,8 @@ buildhistory_get_installed() { # Produce dependency graph # First, quote each name to handle characters that cause issues for dot - rootfs_list_installed_depends | sed 's:\([^| ]*\):"\1":g' > $1/depends.tmp + sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp && \ + rm ${WORKDIR}/bh_installed_pkgs_deps.txt # Change delimiter from pipe to -> and set style for recommend lines sed -i -e 's:|: -> :' -e 's:"\[REC\]":[style=dotted]:' -e 's:$:;:' $1/depends.tmp # Add header, sorted and de-duped contents and footer and then delete the temp file @@ -336,17 +445,8 @@ buildhistory_get_installed() { rm $1/depends.tmp # Produce installed package sizes list - printf "" > $1/installed-package-sizes.tmp - cat $pkgcache | while read pkg pkgfile pkgarch - do - for vendor in ${TARGET_VENDOR} ${MULTILIB_VENDORS} ; do - size=`oe-pkgdata-util read-value ${TMPDIR}/pkgdata $vendor-${TARGET_OS} "PKGSIZE" ${pkg}_${pkgarch}` - if [ "$size" != "" ] ; then - echo "$size $pkg" >> $1/installed-package-sizes.tmp - fi - done - done - cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt + oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp + cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt rm $1/installed-package-sizes.tmp # We're now done with the cache, delete it @@ -354,7 +454,7 @@ buildhistory_get_installed() { if [ "$2" != "sdk" ] ; then # Produce some cut-down graphs (for readability) - grep -v kernel_image $1/depends.dot | grep -v kernel-2 | grep -v kernel-3 > $1/depends-nokernel.dot + grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot @@ -370,7 +470,7 @@ buildhistory_get_image_installed() { # Anything requiring the use of the packaging system should be done in here # in case the packaging files are going to be removed for this image - if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then return fi @@ -381,25 +481,52 @@ buildhistory_get_sdk_installed() { # Anything requiring the use of the packaging system should be done in here # in case the packaging files are going to be removed for this SDK - if [ "${@base_contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then return fi buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk } +buildhistory_get_sdk_installed_host() { + buildhistory_get_sdk_installed host +} + +buildhistory_get_sdk_installed_target() { + buildhistory_get_sdk_installed target +} + buildhistory_list_files() { # List the files in the specified directory, but exclude date/time etc. # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo - ( cd $1 && find . -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' | sort -k5 | sed 's/ *$//' > $2 ) + if [ "$3" = "fakeroot" ] ; then + ( cd $1 && ${FAKEROOTENV} ${FAKEROOTCMD} find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 ) + else + ( cd $1 && find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 ) + fi } +buildhistory_list_pkg_files() { + # Create individual files-in-package for each recipe's package + for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do + pkgname=$(basename $pkgdir) + outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname" + outfile="$outfolder/files-in-package.txt" + # Make sure the output folder exists so we can create the file + if [ ! -d $outfolder ] ; then + bbdebug 2 "Folder $outfolder does not exist, file $outfile not created" + continue + fi + buildhistory_list_files $pkgdir $outfile fakeroot + done +} buildhistory_get_imageinfo() { - if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then return fi + mkdir -p ${BUILDHISTORY_DIR_IMAGE} buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt # Collect files requested in BUILDHISTORY_IMAGE_FILES @@ -420,20 +547,29 @@ END echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt # Add some configuration information - echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id + echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt - cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id <<END -${@buildhistory_get_layers(d)} + cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END +${@buildhistory_get_build_id(d)} END } buildhistory_get_sdkinfo() { - if [ "${@base_contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then return fi buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt + # Collect files requested in BUILDHISTORY_SDK_FILES + rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files + for f in ${BUILDHISTORY_SDK_FILES}; do + if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then + mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f` + cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f + fi + done + # Record some machine-readable meta-information about the SDK printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END @@ -443,55 +579,119 @@ END echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt } -# By prepending we get in before the removal of packaging files -ROOTFS_POSTPROCESS_COMMAND =+ "buildhistory_get_image_installed ; " +python buildhistory_get_extra_sdkinfo() { + import operator + import math + + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \ + "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + tasksizes = {} + filesizes = {} + for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')): + for fn in files: + if fn.endswith('.tgz'): + fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024)) + task = fn.rsplit(':', 1)[1].split('_', 1)[1].split('.')[0] + origtotal = tasksizes.get(task, 0) + tasksizes[task] = origtotal + fsize + filesizes[fn] = fsize + with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f: + filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1, 0), reverse=True) + for fn, size in filesizes_sorted: + f.write('%10d KiB %s\n' % (size, fn)) + with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f: + tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1, 0), reverse=True) + for task, size in tasksizes_sorted: + f.write('%10d KiB %s\n' % (size, task)) +} + +# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of +# unneeded packages but before the removal of packaging files +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;" +ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;" +ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed" -IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; " +IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;" +IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo" # We want these to be the last run so that we get called after complementary package installation -POPULATE_SDK_POST_TARGET_COMMAND_append = "buildhistory_get_sdk_installed target ; " -POPULATE_SDK_POST_HOST_COMMAND_append = "buildhistory_get_sdk_installed host ; " +POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;" +POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;" +POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;" + +POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;" +POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;" +POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;" + +SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " +SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " + +python buildhistory_write_sigs() { + if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return + + # Create sigs file + if hasattr(bb.parse.siggen, 'dump_siglist'): + taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task') + bb.utils.mkdirhier(taskoutdir) + bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt')) +} -SDK_POSTPROCESS_COMMAND += "buildhistory_get_sdkinfo ; " +def buildhistory_get_build_id(d): + if d.getVar('BB_WORKERCONTEXT') != '1': + return "" + localdata = bb.data.createCopy(d) + statuslines = [] + for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): + g = globals() + if func not in g: + bb.warn("Build configuration function '%s' does not exist" % func) + else: + flines = g[func](localdata) + if flines: + statuslines.extend(flines) -def buildhistory_get_layers(d): - layertext = "Configured metadata layers:\n%s\n" % '\n'.join(get_layers_branch_rev(d)) - return layertext + statusheader = d.getVar('BUILDCFG_HEADER') + return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) def buildhistory_get_metadata_revs(d): # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want - layers = (d.getVar("BBLAYERS", True) or "").split() + layers = (d.getVar("BBLAYERS") or "").split() medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \ base_get_metadata_git_branch(i, None).strip(), \ base_get_metadata_git_revision(i, None)) \ for i in layers] return '\n'.join(medadata_revs) - -def squashspaces(string): - import re - return re.sub("\s+", " ", string).strip() - def outputvars(vars, listvars, d): vars = vars.split() listvars = listvars.split() ret = "" for var in vars: - value = d.getVar(var, True) or "" + value = d.getVar(var) or "" if var in listvars: # Squash out spaces - value = squashspaces(value) + value = oe.utils.squashspaces(value) ret += "%s = %s\n" % (var, value) return ret.rstrip('\n') def buildhistory_get_imagevars(d): - imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" - listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS" + if d.getVar('BB_WORKERCONTEXT') != '1': + return "" + imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" + listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(imagevars, listvars, d) def buildhistory_get_sdkvars(d): - sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS" - listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS" + if d.getVar('BB_WORKERCONTEXT') != '1': + return "" + sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" + if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext': + # Extensible SDK uses some additional variables + sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN" + listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST" return outputvars(sdkvars, listvars, d) @@ -503,6 +703,42 @@ def buildhistory_get_cmdline(d): return '%s %s' % (bincmd, ' '.join(sys.argv[1:])) +buildhistory_single_commit() { + if [ "$3" = "" ] ; then + commitopts="${BUILDHISTORY_DIR}/ --allow-empty" + item="No changes" + else + commitopts="$3 metadata-revs" + item="$3" + fi + if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then + result="succeeded" + else + result="failed" + fi + case ${BUILDHISTORY_BUILD_INTERRUPTED} in + 1) + result="$result (interrupted)" + ;; + 2) + result="$result (force interrupted)" + ;; + esac + commitmsgfile=`mktemp` + cat > $commitmsgfile << END +$item: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2 + +cmd: $1 + +result: $result + +metadata revisions: +END + cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile + git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null + rm $commitmsgfile +} + buildhistory_commit() { if [ ! -d ${BUILDHISTORY_DIR} ] ; then # Code above that creates this dir never executed, so there can't be anything to commit @@ -516,13 +752,16 @@ END ( cd ${BUILDHISTORY_DIR}/ # Initialise the repo if necessary - if [ ! -d .git ] ; then + if [ ! -e .git ] ; then git init -q else git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true git tag -f build-minus-1 > /dev/null 2>&1 || true fi + + check_git_config + # Check if there are new/changed files to commit (other than metadata-revs) repostatus=`git status --porcelain | grep -v " metadata-revs$"` HOSTNAME=`hostname 2>/dev/null || echo unknown` @@ -532,25 +771,49 @@ END # porcelain output looks like "?? packages/foo/bar" # Ensure we commit metadata-revs with the first commit for entry in `echo "$repostatus" | awk '{print $2}' | awk -F/ '{print $1}' | sort | uniq` ; do - git commit $entry metadata-revs -m "$entry: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $HOSTNAME" -m "cmd: $CMDLINE" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null + buildhistory_single_commit "$CMDLINE" "$HOSTNAME" "$entry" done - if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then - git push -q ${BUILDHISTORY_PUSH_REPO} - fi + git gc --auto --quiet else - git commit ${BUILDHISTORY_DIR}/ --allow-empty -m "No changes: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $HOSTNAME" -m "cmd: $CMDLINE" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null + buildhistory_single_commit "$CMDLINE" "$HOSTNAME" + fi + if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then + git push -q ${BUILDHISTORY_PUSH_REPO} fi) || true } python buildhistory_eventhandler() { - if e.data.getVar('BUILDHISTORY_FEATURES', True).strip(): - if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1": - bb.note("Writing buildhistory") - bb.build.exec_func("buildhistory_commit", e.data) + if e.data.getVar('BUILDHISTORY_FEATURES').strip(): + reset = e.data.getVar("BUILDHISTORY_RESET") + olddir = e.data.getVar("BUILDHISTORY_OLD_DIR") + if isinstance(e, bb.event.BuildStarted): + if reset: + import shutil + # Clean up after potentially interrupted build. + if os.path.isdir(olddir): + shutil.rmtree(olddir) + rootdir = e.data.getVar("BUILDHISTORY_DIR") + entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ] + bb.utils.mkdirhier(olddir) + for entry in entries: + os.rename(os.path.join(rootdir, entry), + os.path.join(olddir, entry)) + elif isinstance(e, bb.event.BuildCompleted): + if reset: + import shutil + shutil.rmtree(olddir) + if e.data.getVar("BUILDHISTORY_COMMIT") == "1": + bb.note("Writing buildhistory") + bb.build.exec_func("buildhistory_write_sigs", d) + localdata = bb.data.createCopy(e.data) + localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures)) + interrupted = getattr(e, '_interrupted', 0) + localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted)) + bb.build.exec_func("buildhistory_commit", localdata) } addhandler buildhistory_eventhandler -buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted" +buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted" # FIXME this ought to be moved into the fetcher @@ -560,7 +823,7 @@ def _get_srcrev_values(d): """ scms = [] - fetcher = bb.fetch.Fetch(d.getVar('SRC_URI', True).split(), d) + fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d) urldata = fetcher.ud for u in urldata: if urldata[u].method.supports_srcrev(): @@ -572,7 +835,11 @@ def _get_srcrev_values(d): for scm in scms: ud = urldata[scm] for name in ud.names: - rev = ud.method.sortable_revision(scm, ud, d, name) + try: + rev = ud.method.sortable_revision(ud, d, name) + except TypeError: + # support old bitbake versions + rev = ud.method.sortable_revision(scm, ud, d, name) # Clean this up when we next bump bitbake version if type(rev) != str: autoinc, rev = rev @@ -586,8 +853,9 @@ def _get_srcrev_values(d): return (dict_srcrevs, dict_tag_srcrevs) do_fetch[postfuncs] += "write_srcrev" +do_fetch[vardepsexclude] += "write_srcrev" python write_srcrev() { - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') srcrevfile = os.path.join(pkghistdir, 'latest_srcrev') srcrevs, tag_srcrevs = _get_srcrev_values(d) @@ -614,12 +882,12 @@ python write_srcrev() { f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev)) f.write('SRCREV_%s = "%s"\n' % (name, srcrev)) else: - f.write('SRCREV = "%s"\n' % srcrevs.itervalues().next()) + f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values()))) if len(tag_srcrevs) > 0: for name, srcrev in tag_srcrevs.items(): f.write('# tag_%s = "%s"\n' % (name, srcrev)) if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev: - pkg = d.getVar('PN', True) + pkg = d.getVar('PN') bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev)) else: |