From f73c64e7295be1e1065f898b49f50a02e812161c Mon Sep 17 00:00:00 2001 From: Khem Raj Date: Sun, 23 Aug 2009 19:24:57 -0700 Subject: base.bbclass: Replace os.system with subprocess call. Often gzip is reporting broken pipe errors with do_unpack of tar.gz files. If you use the commands described above to extract a tar.gz file, gzip sometimes emits a Broken pipe error message. This can safely be ignored if tar extracted all files without any other error message. We do not let python install its SIGPIPE handler and use subprocess call to invoke the command. This is based on the following python bug report. http://bugs.python.org/issue1652 Signed-off-by: Khem Raj --- classes/base.bbclass | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'classes') diff --git a/classes/base.bbclass b/classes/base.bbclass index 598a7bb77f..4e9b65c285 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -728,9 +728,14 @@ base_do_buildall() { : } +def subprocess_setup(): + import signal, subprocess + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) def oe_unpack_file(file, data, url = None): - import bb, os + import bb, os, signal, subprocess if not url: url = "file://%s" % file dots = file.split(".") @@ -799,7 +804,7 @@ def oe_unpack_file(file, data, url = None): cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd) bb.note("Unpacking %s to %s/" % (base_path_out(file, data), base_path_out(os.getcwd(), data))) - ret = os.system(cmd) + ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) os.chdir(save_cwd) -- cgit v1.2.3 From 482e70f47fed52f27c83cc41e6bdeeb8ad36c779 Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 03:58:03 -0700 Subject: package_ipk.bbclass: don't choke on empty SRC_URI. Signed-off-by: Chris Larson --- classes/package_ipk.bbclass | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'classes') diff --git a/classes/package_ipk.bbclass b/classes/package_ipk.bbclass index e3a7522619..e5561082fd 100644 --- a/classes/package_ipk.bbclass +++ b/classes/package_ipk.bbclass @@ -266,10 +266,9 @@ python do_package_ipk () { ctrlfile.write("Replaces: %s\n" % ", ".join(rreplaces)) if rconflicts: ctrlfile.write("Conflicts: %s\n" % ", ".join(rconflicts)) - src_uri = bb.data.getVar("SRC_URI", localdata, 1) - if src_uri: - src_uri = re.sub("\s+", " ", src_uri) - ctrlfile.write("Source: %s\n" % " ".join(src_uri.split())) + src_uri = bb.data.getVar("SRC_URI", localdata, 1) or d.getVar("FILE", True) + src_uri = re.sub("\s+", " ", src_uri) + ctrlfile.write("Source: %s\n" % " ".join(src_uri.split())) ctrlfile.close() for script in ["preinst", "postinst", "prerm", "postrm"]: -- cgit v1.2.3 From 4dc61b382a9e13b3271c69ff6fcbf69432273172 Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 03:58:24 -0700 Subject: srctree.bbclass: add initial version. This enables operation inside of an existing source tree for a project, rather than using the fetch/unpack/patch idiom. By default, it expects that you're keeping the recipe(s) inside the aforementioned source tree, but you could override S to point at an external directory and place the recipes in a normal collection/overlay, if you so chose. It also provides some convenience python functions for assembling your do_clean, if you want to leverage things like 'git clean' to simplify the operation. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 141 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 classes/srctree.bbclass (limited to 'classes') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass new file mode 100644 index 0000000000..2a324cf1be --- /dev/null +++ b/classes/srctree.bbclass @@ -0,0 +1,141 @@ +# Copyright (C) 2009 Chris Larson +# Released under the MIT license (see COPYING.MIT for the terms) +# +# srctree.bbclass enables operation inside of an existing source tree for a +# project, rather than using the fetch/unpack/patch idiom. +# +# By default, it expects that you're keeping the recipe(s) inside the +# aforementioned source tree, but you could override S to point at an external +# directory and place the recipes in a normal collection/overlay, if you so +# chose. +# +# It also provides some convenience python functions for assembling your +# do_clean, if you want to leverage things like 'git clean' to simplify the +# operation. + + +S = "${FILE_DIRNAME}" +SRC_URI = "" + +#TYPE = "${@'${PN}'.replace('${BPN}', '').strip()}" +#WORKDIR = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" +#STAMP = "${WORKDIR}/tasks/stamp" +#WORKDIR_LOCAL = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" +#T = "${WORKDIR_LOCAL}/bitbake-tasks" + +# Hack, so things don't explode in builds that don't inherit package +do_package ?= " pass" +do_package[func] = "1" +do_package[python] = "1" + +# Ensure that do_package depends on populate_staging, rather than install +addtask package after do_populate_staging + +# This stuff is needed to facilitate variants (normal, native, cross, sdk) +# that share a ${S}. It's ugly as hell. Only really necessary for recipes +# that can't use a ${B}, and which have variants like native. Not sure what +# the best solution is long term. +# +# We merge configure+compile+install into the populate_staging task, uses a +# lock file. This ensures that none of the tasks which access ${S} can +# interleave amongst the recipes that share that ${S}. + +def variant_hack(d): + from itertools import chain + + # Kill dependencies on the fromtasks + fromtasks = ["do_configure", "do_compile", "do_install"] + for key in d.keys(): + task = d.getVarFlag(key, "task") + if task: + deps = d.getVarFlag(key, "deps") + for task_ in fromtasks: + if task_ in deps: + deps.remove(task_) + # if not key in fromtasks + ["do_populate_staging"]: + # deps.append("do_populate_staging") + d.setVarFlag(key, "deps", deps) + + # Pull the task deps from fromtasks over to the new task, minus deltasks + deltasks = ("do_patch", "do_unpack", "do_fetch") + deps = set(chain(*[(d.getVarFlag(old, "deps")) for old in fromtasks + ["do_populate_staging"]])) + d.setVarFlag("do_populate_staging", "deps", deps.difference(deltasks)) + + # Pull cross recipe task deps over + d.setVarFlag("do_populate_staging", "depends", " ".join((d.getVarFlag(old, "depends") or "" for old in fromtasks))) + +python () { + variant_hack(d) +} + +python do_populate_staging () { + from bb.build import exec_task, exec_func + + exec_task("do_configure", d) + exec_task("do_compile", d) + exec_task("do_install", d) + exec_func("do_stage", d) +} +do_populate_staging[lockfiles] += "${S}/.lock" + +make_do_clean () { + oe_runmake clean +} + +def clean_builddir(d): + from shutil import rmtree + + builddir = d.getVar("B", True) + srcdir = d.getVar("S", True) + if builddir != srcdir: + rmtree(builddir, ignore_errors=True) + +def clean_stamps(d): + from glob import glob + from bb import note + from bb.data import expand + from os import unlink + + note("Removing stamps") + for stamp in glob(expand('${STAMP}.*', d)): + try: + unlink(stamp) + except OSError: + pass + +def clean_workdir(d): + from shutil import rmtree + from bb import note + + workdir = d.getVar("WORKDIR", 1) + note("Removing %s" % workdir) + rmtree(workdir, ignore_errors=True) + +def clean_git(d): + from subprocess import call + + call(["git", "clean", "-d", "-f", "-X"], cwd=d.getVar("S", True)) + +def clean_make(d): + import bb + + bb.note("Running make clean") + try: + bb.build.exec_func("make_do_clean", d) + except bb.build.FuncFailed: + pass + +python do_clean () { + from os.path import exists + from bb.build import FuncFailed + from bb.data import expand + + clean_stamps(d) + clean_workdir(d) + if exists(expand("${S}/.git", d)) and \ + exists(expand("${S}/.gitignore", d)): + clean_git(d) + else: + clean_builddir(d) + clean_make(d) +} -- cgit v1.2.3 From e5b751653cb17d60665a648c8f9aaeda028aa122 Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 03:59:25 -0700 Subject: gitver.bbclass: add initial version. This provides a GITVER variable which is a (fairly) sane version, for use in ${PV}, extracted from the ${S} git checkout, assuming it is one. This is most useful in concert with srctree.bbclass. Signed-off-by: Chris Larson --- classes/gitver.bbclass | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 classes/gitver.bbclass (limited to 'classes') diff --git a/classes/gitver.bbclass b/classes/gitver.bbclass new file mode 100644 index 0000000000..92c053ae24 --- /dev/null +++ b/classes/gitver.bbclass @@ -0,0 +1,54 @@ +# Copyright (C) 2009 Chris Larson +# Released under the MIT license (see COPYING.MIT for the terms) +# +# gitver.bbclass provides a GITVER variable which is a (fairly) sane version, +# for use in ${PV}, extracted from the ${S} git checkout, assuming it is one. +# This is most useful in concert with srctree.bbclass. + + +GITVER = "${@get_git_pv('${S}', d)}" + +def gitver_mark_dependency(d): + from bb.data import expand + from bb.parse import mark_dependency + from os.path import abspath + + fn = abspath(expand("${S}/.git/HEAD", d)) + mark_dependency(d, fn) + +def get_git_pv(path, d, tagadjust=None): + from subprocess import Popen, PIPE + from os.path import join + from bb import error + + env = {"GIT_DIR": join(d.getVar("S", True), ".git")} + + def popen(cmd, **kwargs): + kwargs["stderr"] = PIPE + kwargs["stdout"] = PIPE + kwargs["env"] = env + try: + pipe = Popen(cmd, **kwargs) + except OSError, e: + #error("Execution of %s failed: %s" % (cmd, e)) + return + + (stdout, stderr) = pipe.communicate(None) + if pipe.returncode != 0: + #error("Execution of %s failed: %s" % (cmd, stderr)) + return + return stdout.rstrip() + + gitver_mark_dependency(d) + + ver = popen(["git", "describe", "--tags"], cwd=path) + if not ver: + ver = popen(["git", "rev-parse", "--short", "HEAD"], cwd=path) + if ver: + return "0.0-%s" % ver + else: + return "0.0" + else: + if tagadjust: + ver = tagadjust(ver) + return ver -- cgit v1.2.3 From af56cd467bcc76f5499199fb1f14252abe1e1e2a Mon Sep 17 00:00:00 2001 From: Khem Raj Date: Wed, 26 Aug 2009 00:38:30 -0700 Subject: base.bbclass: Remove redundant import of subprocess and signal. Signed-off-by: Khem Raj --- classes/base.bbclass | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'classes') diff --git a/classes/base.bbclass b/classes/base.bbclass index 4e9b65c285..a350b38850 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -729,13 +729,13 @@ base_do_buildall() { } def subprocess_setup(): - import signal, subprocess + import signal # Python installs a SIGPIPE handler by default. This is usually not what # non-Python subprocesses expect. signal.signal(signal.SIGPIPE, signal.SIG_DFL) def oe_unpack_file(file, data, url = None): - import bb, os, signal, subprocess + import bb, os, subprocess if not url: url = "file://%s" % file dots = file.split(".") -- cgit v1.2.3 From 6a5e3eadec25bfe03fdeff44197ffec45726484e Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 14:02:19 -0700 Subject: srctree.bbclass: Don't automatically use git clean. This should be an opt-in feature. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'classes') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index 2a324cf1be..e2a4303472 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -126,16 +126,8 @@ def clean_make(d): pass python do_clean () { - from os.path import exists - from bb.build import FuncFailed - from bb.data import expand - clean_stamps(d) clean_workdir(d) - if exists(expand("${S}/.git", d)) and \ - exists(expand("${S}/.gitignore", d)): - clean_git(d) - else: - clean_builddir(d) - clean_make(d) + clean_builddir(d) + clean_make(d) } -- cgit v1.2.3 From c45f041e4a65efd4e8e7565d186f87798d740601 Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 14:02:48 -0700 Subject: srctree: completely reimplement the task merging into do_populate_staging. This fixes issues seen with the use of kernel.bbclass, so in existing source tree kernel builds work happily now. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 102 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 25 deletions(-) (limited to 'classes') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index e2a4303472..67cb9ffe8c 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -27,8 +27,6 @@ SRC_URI = "" do_package ?= " pass" do_package[func] = "1" do_package[python] = "1" - -# Ensure that do_package depends on populate_staging, rather than install addtask package after do_populate_staging # This stuff is needed to facilitate variants (normal, native, cross, sdk) @@ -40,40 +38,94 @@ addtask package after do_populate_staging # lock file. This ensures that none of the tasks which access ${S} can # interleave amongst the recipes that share that ${S}. -def variant_hack(d): +def merge_tasks(d): + """ + merge_tasks performs two operations: + 1) removes do_patch and its deps from the build entirely. + 2) merges all of the operations that occur prior to do_populate_staging + into do_populate_staging. + + This is necessary, because of recipe variants (normal, native, cross, + sdk). If a bitbake run happens to want to build more than one of + these variants in a single run, it's possible for them to step on one + another's toes, due to the shared ${S}. Interleaved + configure/compile/install amongst variants will break things badly. + """ from itertools import chain + from bb import note - # Kill dependencies on the fromtasks - fromtasks = ["do_configure", "do_compile", "do_install"] - for key in d.keys(): - task = d.getVarFlag(key, "task") - if task: - deps = d.getVarFlag(key, "deps") - for task_ in fromtasks: - if task_ in deps: - deps.remove(task_) - # if not key in fromtasks + ["do_populate_staging"]: - # deps.append("do_populate_staging") - d.setVarFlag(key, "deps", deps) - - # Pull the task deps from fromtasks over to the new task, minus deltasks - deltasks = ("do_patch", "do_unpack", "do_fetch") - deps = set(chain(*[(d.getVarFlag(old, "deps")) for old in fromtasks + ["do_populate_staging"]])) - d.setVarFlag("do_populate_staging", "deps", deps.difference(deltasks)) + def __gather_taskdeps(task, seen): + for dep in d.getVarFlag(task, "deps"): + if not dep in seen: + __gather_taskdeps(dep, seen) + if not task in seen: + seen.append(task) + + def gather_taskdeps(task): + items = [] + __gather_taskdeps(task, items) + return items + + newtask = "do_populate_staging" + mergedtasks = gather_taskdeps(newtask) + mergedtasks.pop() + deltasks = gather_taskdeps("do_patch") + + for task in (key for key in d.keys() + if d.getVarFlag(key, "task") and + not key in mergedtasks): + deps = d.getVarFlag(task, "deps") + for mergetask in mergedtasks: + if mergetask in (d.getVarFlag(task, "recrdeptask"), + d.getVarFlag(task, "recdeptask"), + d.getVarFlag(task, "deptask")): + continue + + if mergetask in deps: + deps.remove(mergetask) + #note("removing dep on %s from %s" % (mergetask, task)) + + if not mergetask in deltasks and \ + not newtask in deps: + #note("adding dep on %s to %s" % (newtask, task)) + deps.append(newtask) + d.setVarFlag(task, "deps", deps) + + for task in mergedtasks[:-1]: + deps = d.getVarFlag(task, "deps") + for deltask in deltasks: + if deltask in deps: + deps.remove(deltask) + d.setVarFlag(task, "deps", deps) # Pull cross recipe task deps over - d.setVarFlag("do_populate_staging", "depends", " ".join((d.getVarFlag(old, "depends") or "" for old in fromtasks))) + depends = (d.getVarFlag(task, "depends") or "" + for task in mergedtasks[:-1] + if not task in deltasks) + d.setVarFlag("do_populate_staging", "depends", " ".join(depends)) python () { - variant_hack(d) + merge_tasks(d) } +# Manually run do_install & all of its deps, then do_stage python do_populate_staging () { + from os.path import exists from bb.build import exec_task, exec_func + from bb import note + + stamp = d.getVar("STAMP", True) + + def rec_exec_task(task, seen): + for dep in d.getVarFlag(task, "deps"): + if not dep in seen: + rec_exec_task(dep, seen) + seen.add(task) + #if not exists("%s.%s" % (stamp, task)): + note("Executing task %s" % task) + exec_task(task, d) - exec_task("do_configure", d) - exec_task("do_compile", d) - exec_task("do_install", d) + rec_exec_task("do_install", set()) exec_func("do_stage", d) } do_populate_staging[lockfiles] += "${S}/.lock" -- cgit v1.2.3 From 2259a81628ac8bafb19e2b9ba9acca1c0dad341d Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 14:44:04 -0700 Subject: srctree, clean: cleanup & split out do_clean bits into clean.bbclass. Signed-off-by: Chris Larson --- classes/clean.bbclass | 53 ++++++++++++++++++++++++++++++++ classes/srctree.bbclass | 82 ++++++------------------------------------------- 2 files changed, 62 insertions(+), 73 deletions(-) create mode 100644 classes/clean.bbclass (limited to 'classes') diff --git a/classes/clean.bbclass b/classes/clean.bbclass new file mode 100644 index 0000000000..65c1ab5d76 --- /dev/null +++ b/classes/clean.bbclass @@ -0,0 +1,53 @@ +def clean_builddir(d): + from shutil import rmtree + + builddir = d.getVar("B", True) + srcdir = d.getVar("S", True) + if builddir != srcdir: + rmtree(builddir, ignore_errors=True) + +def clean_stamps(d): + from glob import glob + from bb import note + from bb.data import expand + from os import unlink + + note("Removing stamps") + for stamp in glob(expand('${STAMP}.*', d)): + try: + unlink(stamp) + except OSError: + pass + +def clean_workdir(d): + from shutil import rmtree + from bb import note + + workdir = d.getVar("WORKDIR", 1) + note("Removing %s" % workdir) + rmtree(workdir, ignore_errors=True) + +def clean_git(d): + from subprocess import call + + call(["git", "clean", "-d", "-f", "-X"], cwd=d.getVar("S", True)) + +def clean_make(d): + import bb + + bb.note("Running make clean") + try: + bb.build.exec_func("__do_clean_make", d) + except bb.build.FuncFailed: + pass + +__do_clean_make () { + oe_runmake clean +} + +python do_clean () { + clean_stamps(d) + clean_workdir(d) + clean_builddir(d) + clean_make(d) +} diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index 67cb9ffe8c..ce9cb834bb 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -14,29 +14,13 @@ # operation. +# Grab convenience methods & sane default for do_clean +inherit clean + +# Build here S = "${FILE_DIRNAME}" SRC_URI = "" -#TYPE = "${@'${PN}'.replace('${BPN}', '').strip()}" -#WORKDIR = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" -#STAMP = "${WORKDIR}/tasks/stamp" -#WORKDIR_LOCAL = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" -#T = "${WORKDIR_LOCAL}/bitbake-tasks" - -# Hack, so things don't explode in builds that don't inherit package -do_package ?= " pass" -do_package[func] = "1" -do_package[python] = "1" -addtask package after do_populate_staging - -# This stuff is needed to facilitate variants (normal, native, cross, sdk) -# that share a ${S}. It's ugly as hell. Only really necessary for recipes -# that can't use a ${B}, and which have variants like native. Not sure what -# the best solution is long term. -# -# We merge configure+compile+install into the populate_staging task, uses a -# lock file. This ensures that none of the tasks which access ${S} can -# interleave amongst the recipes that share that ${S}. def merge_tasks(d): """ @@ -130,56 +114,8 @@ python do_populate_staging () { } do_populate_staging[lockfiles] += "${S}/.lock" -make_do_clean () { - oe_runmake clean -} - -def clean_builddir(d): - from shutil import rmtree - - builddir = d.getVar("B", True) - srcdir = d.getVar("S", True) - if builddir != srcdir: - rmtree(builddir, ignore_errors=True) - -def clean_stamps(d): - from glob import glob - from bb import note - from bb.data import expand - from os import unlink - - note("Removing stamps") - for stamp in glob(expand('${STAMP}.*', d)): - try: - unlink(stamp) - except OSError: - pass - -def clean_workdir(d): - from shutil import rmtree - from bb import note - - workdir = d.getVar("WORKDIR", 1) - note("Removing %s" % workdir) - rmtree(workdir, ignore_errors=True) - -def clean_git(d): - from subprocess import call - - call(["git", "clean", "-d", "-f", "-X"], cwd=d.getVar("S", True)) - -def clean_make(d): - import bb - - bb.note("Running make clean") - try: - bb.build.exec_func("make_do_clean", d) - except bb.build.FuncFailed: - pass - -python do_clean () { - clean_stamps(d) - clean_workdir(d) - clean_builddir(d) - clean_make(d) -} +# Hack, so things don't explode in builds that don't inherit package +do_package ?= " pass" +do_package[func] = "1" +do_package[python] = "1" +addtask package after do_populate_staging -- cgit v1.2.3 From 4cd5e9101dbefdefa183e4103989819893b2ffea Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 15:31:03 -0700 Subject: base.bbclass: add cleanall task. Can be particularly useful when using multiple srctree recipes. Signed-off-by: Chris Larson --- classes/base.bbclass | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'classes') diff --git a/classes/base.bbclass b/classes/base.bbclass index a350b38850..97ccf5dfc3 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -523,6 +523,12 @@ python base_do_clean() { os.system('rm -f '+ dir) } +python do_cleanall() { + pass +} +do_cleanall[recrdeptask] = "do_clean" +addtask cleanall after do_clean + #Uncomment this for bitbake 1.8.12 #addtask rebuild after do_${BB_DEFAULT_TASK} addtask rebuild -- cgit v1.2.3 From 09b6a74a6f03bb114cbc3620b00ce60cbd17ab0b Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 15:32:30 -0700 Subject: srctree: prefix messages with ${PF} and kill do_package hacks. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'classes') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index ce9cb834bb..dbf8ad2c3f 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -106,16 +106,10 @@ python do_populate_staging () { rec_exec_task(dep, seen) seen.add(task) #if not exists("%s.%s" % (stamp, task)): - note("Executing task %s" % task) + note("%s: executing task %s" % (d.getVar("PF", True), task)) exec_task(task, d) rec_exec_task("do_install", set()) exec_func("do_stage", d) } do_populate_staging[lockfiles] += "${S}/.lock" - -# Hack, so things don't explode in builds that don't inherit package -do_package ?= " pass" -do_package[func] = "1" -do_package[python] = "1" -addtask package after do_populate_staging -- cgit v1.2.3 From 42dc52e38db1fd2c57dadec26ca54a9f4a6db1aa Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Thu, 27 Aug 2009 11:04:07 -0700 Subject: Add task-metadata-track.bbclass, for tracking task changes to metadata. This class uses events to capture the state of the datastore when the task starts, and after it completes. It diffs those captured states, and emits messages showing which variables changed, and what their values were changed to. It provides a mechanism to blacklist variables you expect to change, both globally and on a per-task basis. Signed-off-by: Chris Larson --- classes/task-metadata-track.bbclass | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 classes/task-metadata-track.bbclass (limited to 'classes') diff --git a/classes/task-metadata-track.bbclass b/classes/task-metadata-track.bbclass new file mode 100644 index 0000000000..d3622aef68 --- /dev/null +++ b/classes/task-metadata-track.bbclass @@ -0,0 +1,96 @@ +# Copyright (C) 2009 Chris Larson +# Released under the MIT license (see COPYING.MIT for the terms) +# +# This class uses events to capture the state of the datastore when the task +# starts, and after it completes. It diffs those captured states, and emits +# messages showing which variables changed, and what their values were changed +# to. +# +# It provides a mechanism to blacklist variables you expect to change, both +# globally and on a per-task basis. +# +# Known instances of tasks changing metadata: +# +# PSTAGE_PKGMANAGER changes by calls to pstage_set_pkgmanager in: +# do_clean, do_setscene, do_package_stage +# +# Subpackage metadata, read by the pkgdata functions in base.bbclass, in: +# do_package, do_package_stage, do_package_write_* + + +TASK_METADATA_BLACKLIST = "\ + __RUNQUEUE_DO_NOT_USE_EXTERNALLY \ +" + +#TASK_METADATA_BLACKLIST_do_clean = "\ +# PSTAGE_PKGMANAGER \ +#" + + +def dict_diff(olddict, newdict): + diff = {} + for key in set(olddict).union(set(newdict)): + old = olddict.get(key) + new = newdict.get(key) + if old != new: + diff[key] = (old, new) + + return diff + +def dict_for_data(data): + newdict = {} + for key in data.keys(): + newdict[key] = data.getVar(key, False) + return newdict + +def task_metadata_track_start(task, data): + originaldata = dict_for_data(data) + data.setVar("__originaldata_%s" % task, originaldata) + +def task_metadata_track_stop(task, data): + from bb import note + + pf = data.getVar("PF", True) + def emit(msg): + note("%s: %s" % (pf, msg)) + + originaldata = data.getVar("__originaldata_%s" % task, False) + newdata = dict_for_data(data) + blacklist = data.getVar("TASK_METADATA_BLACKLIST", True).split() + \ + (data.getVar("TASK_METADATA_BLACKLIST_%s" % task, True) or "").split() + + diff = dict_diff(originaldata, newdata) + diff_clean = [key for key in diff \ + if not key in blacklist and \ + not key.startswith("__originaldata_")] + + if diff_clean: + emit("Variables changed by %s:" % task) + for key in diff_clean: + (old, new) = diff[key] + emit(" %s:" % key) + emit(" '%s' -> '%s'" % (old, new)) + +python __task_metadata_track_eh () { + from bb.build import TaskStarted, TaskSucceeded + + if isinstance(e, TaskStarted): + if e.data is None: + from bb import fatal + fatal("e.data is none for %s" % e) + task_metadata_track_start(e.task, e.data) + elif isinstance(e, TaskSucceeded): + task_metadata_track_stop(e.task, e.data) +} +addhandler __task_metadata_track_eh + +addtask py_listtasks +do_py_listtasks[nostamp] = "1" +do_py_listtasks[recrdeptask] = "do_py_listtasks" +python do_py_listtasks() { + import sys + for e in d.keys(): + if d.getVarFlag(e, "task") and \ + d.getVarFlag(e, "python"): + sys.stdout.write("%s\n" % e) +} -- cgit v1.2.3