From 4dc61b382a9e13b3271c69ff6fcbf69432273172 Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 03:58:24 -0700 Subject: srctree.bbclass: add initial version. This enables operation inside of an existing source tree for a project, rather than using the fetch/unpack/patch idiom. By default, it expects that you're keeping the recipe(s) inside the aforementioned source tree, but you could override S to point at an external directory and place the recipes in a normal collection/overlay, if you so chose. It also provides some convenience python functions for assembling your do_clean, if you want to leverage things like 'git clean' to simplify the operation. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 141 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 classes/srctree.bbclass (limited to 'classes/srctree.bbclass') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass new file mode 100644 index 0000000000..2a324cf1be --- /dev/null +++ b/classes/srctree.bbclass @@ -0,0 +1,141 @@ +# Copyright (C) 2009 Chris Larson +# Released under the MIT license (see COPYING.MIT for the terms) +# +# srctree.bbclass enables operation inside of an existing source tree for a +# project, rather than using the fetch/unpack/patch idiom. +# +# By default, it expects that you're keeping the recipe(s) inside the +# aforementioned source tree, but you could override S to point at an external +# directory and place the recipes in a normal collection/overlay, if you so +# chose. +# +# It also provides some convenience python functions for assembling your +# do_clean, if you want to leverage things like 'git clean' to simplify the +# operation. + + +S = "${FILE_DIRNAME}" +SRC_URI = "" + +#TYPE = "${@'${PN}'.replace('${BPN}', '').strip()}" +#WORKDIR = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" +#STAMP = "${WORKDIR}/tasks/stamp" +#WORKDIR_LOCAL = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" +#T = "${WORKDIR_LOCAL}/bitbake-tasks" + +# Hack, so things don't explode in builds that don't inherit package +do_package ?= " pass" +do_package[func] = "1" +do_package[python] = "1" + +# Ensure that do_package depends on populate_staging, rather than install +addtask package after do_populate_staging + +# This stuff is needed to facilitate variants (normal, native, cross, sdk) +# that share a ${S}. It's ugly as hell. Only really necessary for recipes +# that can't use a ${B}, and which have variants like native. Not sure what +# the best solution is long term. +# +# We merge configure+compile+install into the populate_staging task, uses a +# lock file. This ensures that none of the tasks which access ${S} can +# interleave amongst the recipes that share that ${S}. + +def variant_hack(d): + from itertools import chain + + # Kill dependencies on the fromtasks + fromtasks = ["do_configure", "do_compile", "do_install"] + for key in d.keys(): + task = d.getVarFlag(key, "task") + if task: + deps = d.getVarFlag(key, "deps") + for task_ in fromtasks: + if task_ in deps: + deps.remove(task_) + # if not key in fromtasks + ["do_populate_staging"]: + # deps.append("do_populate_staging") + d.setVarFlag(key, "deps", deps) + + # Pull the task deps from fromtasks over to the new task, minus deltasks + deltasks = ("do_patch", "do_unpack", "do_fetch") + deps = set(chain(*[(d.getVarFlag(old, "deps")) for old in fromtasks + ["do_populate_staging"]])) + d.setVarFlag("do_populate_staging", "deps", deps.difference(deltasks)) + + # Pull cross recipe task deps over + d.setVarFlag("do_populate_staging", "depends", " ".join((d.getVarFlag(old, "depends") or "" for old in fromtasks))) + +python () { + variant_hack(d) +} + +python do_populate_staging () { + from bb.build import exec_task, exec_func + + exec_task("do_configure", d) + exec_task("do_compile", d) + exec_task("do_install", d) + exec_func("do_stage", d) +} +do_populate_staging[lockfiles] += "${S}/.lock" + +make_do_clean () { + oe_runmake clean +} + +def clean_builddir(d): + from shutil import rmtree + + builddir = d.getVar("B", True) + srcdir = d.getVar("S", True) + if builddir != srcdir: + rmtree(builddir, ignore_errors=True) + +def clean_stamps(d): + from glob import glob + from bb import note + from bb.data import expand + from os import unlink + + note("Removing stamps") + for stamp in glob(expand('${STAMP}.*', d)): + try: + unlink(stamp) + except OSError: + pass + +def clean_workdir(d): + from shutil import rmtree + from bb import note + + workdir = d.getVar("WORKDIR", 1) + note("Removing %s" % workdir) + rmtree(workdir, ignore_errors=True) + +def clean_git(d): + from subprocess import call + + call(["git", "clean", "-d", "-f", "-X"], cwd=d.getVar("S", True)) + +def clean_make(d): + import bb + + bb.note("Running make clean") + try: + bb.build.exec_func("make_do_clean", d) + except bb.build.FuncFailed: + pass + +python do_clean () { + from os.path import exists + from bb.build import FuncFailed + from bb.data import expand + + clean_stamps(d) + clean_workdir(d) + if exists(expand("${S}/.git", d)) and \ + exists(expand("${S}/.gitignore", d)): + clean_git(d) + else: + clean_builddir(d) + clean_make(d) +} -- cgit v1.2.3 From 6a5e3eadec25bfe03fdeff44197ffec45726484e Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 14:02:19 -0700 Subject: srctree.bbclass: Don't automatically use git clean. This should be an opt-in feature. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'classes/srctree.bbclass') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index 2a324cf1be..e2a4303472 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -126,16 +126,8 @@ def clean_make(d): pass python do_clean () { - from os.path import exists - from bb.build import FuncFailed - from bb.data import expand - clean_stamps(d) clean_workdir(d) - if exists(expand("${S}/.git", d)) and \ - exists(expand("${S}/.gitignore", d)): - clean_git(d) - else: - clean_builddir(d) - clean_make(d) + clean_builddir(d) + clean_make(d) } -- cgit v1.2.3 From c45f041e4a65efd4e8e7565d186f87798d740601 Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 14:02:48 -0700 Subject: srctree: completely reimplement the task merging into do_populate_staging. This fixes issues seen with the use of kernel.bbclass, so in existing source tree kernel builds work happily now. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 102 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 25 deletions(-) (limited to 'classes/srctree.bbclass') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index e2a4303472..67cb9ffe8c 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -27,8 +27,6 @@ SRC_URI = "" do_package ?= " pass" do_package[func] = "1" do_package[python] = "1" - -# Ensure that do_package depends on populate_staging, rather than install addtask package after do_populate_staging # This stuff is needed to facilitate variants (normal, native, cross, sdk) @@ -40,40 +38,94 @@ addtask package after do_populate_staging # lock file. This ensures that none of the tasks which access ${S} can # interleave amongst the recipes that share that ${S}. -def variant_hack(d): +def merge_tasks(d): + """ + merge_tasks performs two operations: + 1) removes do_patch and its deps from the build entirely. + 2) merges all of the operations that occur prior to do_populate_staging + into do_populate_staging. + + This is necessary, because of recipe variants (normal, native, cross, + sdk). If a bitbake run happens to want to build more than one of + these variants in a single run, it's possible for them to step on one + another's toes, due to the shared ${S}. Interleaved + configure/compile/install amongst variants will break things badly. + """ from itertools import chain + from bb import note - # Kill dependencies on the fromtasks - fromtasks = ["do_configure", "do_compile", "do_install"] - for key in d.keys(): - task = d.getVarFlag(key, "task") - if task: - deps = d.getVarFlag(key, "deps") - for task_ in fromtasks: - if task_ in deps: - deps.remove(task_) - # if not key in fromtasks + ["do_populate_staging"]: - # deps.append("do_populate_staging") - d.setVarFlag(key, "deps", deps) - - # Pull the task deps from fromtasks over to the new task, minus deltasks - deltasks = ("do_patch", "do_unpack", "do_fetch") - deps = set(chain(*[(d.getVarFlag(old, "deps")) for old in fromtasks + ["do_populate_staging"]])) - d.setVarFlag("do_populate_staging", "deps", deps.difference(deltasks)) + def __gather_taskdeps(task, seen): + for dep in d.getVarFlag(task, "deps"): + if not dep in seen: + __gather_taskdeps(dep, seen) + if not task in seen: + seen.append(task) + + def gather_taskdeps(task): + items = [] + __gather_taskdeps(task, items) + return items + + newtask = "do_populate_staging" + mergedtasks = gather_taskdeps(newtask) + mergedtasks.pop() + deltasks = gather_taskdeps("do_patch") + + for task in (key for key in d.keys() + if d.getVarFlag(key, "task") and + not key in mergedtasks): + deps = d.getVarFlag(task, "deps") + for mergetask in mergedtasks: + if mergetask in (d.getVarFlag(task, "recrdeptask"), + d.getVarFlag(task, "recdeptask"), + d.getVarFlag(task, "deptask")): + continue + + if mergetask in deps: + deps.remove(mergetask) + #note("removing dep on %s from %s" % (mergetask, task)) + + if not mergetask in deltasks and \ + not newtask in deps: + #note("adding dep on %s to %s" % (newtask, task)) + deps.append(newtask) + d.setVarFlag(task, "deps", deps) + + for task in mergedtasks[:-1]: + deps = d.getVarFlag(task, "deps") + for deltask in deltasks: + if deltask in deps: + deps.remove(deltask) + d.setVarFlag(task, "deps", deps) # Pull cross recipe task deps over - d.setVarFlag("do_populate_staging", "depends", " ".join((d.getVarFlag(old, "depends") or "" for old in fromtasks))) + depends = (d.getVarFlag(task, "depends") or "" + for task in mergedtasks[:-1] + if not task in deltasks) + d.setVarFlag("do_populate_staging", "depends", " ".join(depends)) python () { - variant_hack(d) + merge_tasks(d) } +# Manually run do_install & all of its deps, then do_stage python do_populate_staging () { + from os.path import exists from bb.build import exec_task, exec_func + from bb import note + + stamp = d.getVar("STAMP", True) + + def rec_exec_task(task, seen): + for dep in d.getVarFlag(task, "deps"): + if not dep in seen: + rec_exec_task(dep, seen) + seen.add(task) + #if not exists("%s.%s" % (stamp, task)): + note("Executing task %s" % task) + exec_task(task, d) - exec_task("do_configure", d) - exec_task("do_compile", d) - exec_task("do_install", d) + rec_exec_task("do_install", set()) exec_func("do_stage", d) } do_populate_staging[lockfiles] += "${S}/.lock" -- cgit v1.2.3 From 2259a81628ac8bafb19e2b9ba9acca1c0dad341d Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 14:44:04 -0700 Subject: srctree, clean: cleanup & split out do_clean bits into clean.bbclass. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 82 ++++++------------------------------------------- 1 file changed, 9 insertions(+), 73 deletions(-) (limited to 'classes/srctree.bbclass') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index 67cb9ffe8c..ce9cb834bb 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -14,29 +14,13 @@ # operation. +# Grab convenience methods & sane default for do_clean +inherit clean + +# Build here S = "${FILE_DIRNAME}" SRC_URI = "" -#TYPE = "${@'${PN}'.replace('${BPN}', '').strip()}" -#WORKDIR = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" -#STAMP = "${WORKDIR}/tasks/stamp" -#WORKDIR_LOCAL = "${S}/tmp${TYPE}.${MULTIMACH_HOST_SYS}" -#T = "${WORKDIR_LOCAL}/bitbake-tasks" - -# Hack, so things don't explode in builds that don't inherit package -do_package ?= " pass" -do_package[func] = "1" -do_package[python] = "1" -addtask package after do_populate_staging - -# This stuff is needed to facilitate variants (normal, native, cross, sdk) -# that share a ${S}. It's ugly as hell. Only really necessary for recipes -# that can't use a ${B}, and which have variants like native. Not sure what -# the best solution is long term. -# -# We merge configure+compile+install into the populate_staging task, uses a -# lock file. This ensures that none of the tasks which access ${S} can -# interleave amongst the recipes that share that ${S}. def merge_tasks(d): """ @@ -130,56 +114,8 @@ python do_populate_staging () { } do_populate_staging[lockfiles] += "${S}/.lock" -make_do_clean () { - oe_runmake clean -} - -def clean_builddir(d): - from shutil import rmtree - - builddir = d.getVar("B", True) - srcdir = d.getVar("S", True) - if builddir != srcdir: - rmtree(builddir, ignore_errors=True) - -def clean_stamps(d): - from glob import glob - from bb import note - from bb.data import expand - from os import unlink - - note("Removing stamps") - for stamp in glob(expand('${STAMP}.*', d)): - try: - unlink(stamp) - except OSError: - pass - -def clean_workdir(d): - from shutil import rmtree - from bb import note - - workdir = d.getVar("WORKDIR", 1) - note("Removing %s" % workdir) - rmtree(workdir, ignore_errors=True) - -def clean_git(d): - from subprocess import call - - call(["git", "clean", "-d", "-f", "-X"], cwd=d.getVar("S", True)) - -def clean_make(d): - import bb - - bb.note("Running make clean") - try: - bb.build.exec_func("make_do_clean", d) - except bb.build.FuncFailed: - pass - -python do_clean () { - clean_stamps(d) - clean_workdir(d) - clean_builddir(d) - clean_make(d) -} +# Hack, so things don't explode in builds that don't inherit package +do_package ?= " pass" +do_package[func] = "1" +do_package[python] = "1" +addtask package after do_populate_staging -- cgit v1.2.3 From 09b6a74a6f03bb114cbc3620b00ce60cbd17ab0b Mon Sep 17 00:00:00 2001 From: Chris Larson Date: Wed, 26 Aug 2009 15:32:30 -0700 Subject: srctree: prefix messages with ${PF} and kill do_package hacks. Signed-off-by: Chris Larson --- classes/srctree.bbclass | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'classes/srctree.bbclass') diff --git a/classes/srctree.bbclass b/classes/srctree.bbclass index ce9cb834bb..dbf8ad2c3f 100644 --- a/classes/srctree.bbclass +++ b/classes/srctree.bbclass @@ -106,16 +106,10 @@ python do_populate_staging () { rec_exec_task(dep, seen) seen.add(task) #if not exists("%s.%s" % (stamp, task)): - note("Executing task %s" % task) + note("%s: executing task %s" % (d.getVar("PF", True), task)) exec_task(task, d) rec_exec_task("do_install", set()) exec_func("do_stage", d) } do_populate_staging[lockfiles] += "${S}/.lock" - -# Hack, so things don't explode in builds that don't inherit package -do_package ?= " pass" -do_package[func] = "1" -do_package[python] = "1" -addtask package after do_populate_staging -- cgit v1.2.3