diff options
| -rw-r--r-- | meta/classes/populate_sdk_ext.bbclass | 148 | ||||
| -rw-r--r-- | meta/lib/oe/copy_buildsystem.py | 59 | ||||
| -rw-r--r-- | scripts/lib/devtool/build_image.py | 79 | ||||
| -rw-r--r-- | scripts/lib/devtool/build_sdk.py | 65 |
4 files changed, 264 insertions, 87 deletions
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass index 6079166980..6afc53d284 100644 --- a/meta/classes/populate_sdk_ext.bbclass +++ b/meta/classes/populate_sdk_ext.bbclass @@ -89,7 +89,14 @@ python copy_buildsystem () { # Copy in all metadata layers + bitbake (as repositories) buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True) - layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers') + + # Determine if we're building a derivative extensible SDK (from devtool build-sdk) + derivative = (d.getVar('SDK_DERIVATIVE', True) or '') == '1' + if derivative: + workspace_name = 'orig-workspace' + else: + workspace_name = None + layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) sdkbblayers = [] corebase = os.path.basename(d.getVar('COREBASE', True)) @@ -158,75 +165,81 @@ python copy_buildsystem () { f.write(' "\n') # Create local.conf - local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split() - local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split() - def handle_var(varname, origvalue, op, newlines): - if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist): - newlines.append('# Removed original setting of %s\n' % varname) - return None, op, 0, True - else: - return origvalue, op, 0, True - varlist = ['[^#=+ ]*'] builddir = d.getVar('TOPDIR', True) - with open(builddir + '/conf/local.conf', 'r') as f: - oldlines = f.readlines() - (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) + if derivative: + shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') + else: + local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split() + local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split() + def handle_var(varname, origvalue, op, newlines): + if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist): + newlines.append('# Removed original setting of %s\n' % varname) + return None, op, 0, True + else: + return origvalue, op, 0, True + varlist = ['[^#=+ ]*'] + with open(builddir + '/conf/local.conf', 'r') as f: + oldlines = f.readlines() + (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) - with open(baseoutpath + '/conf/local.conf', 'w') as f: - f.write('# WARNING: this configuration has been automatically generated and in\n') - f.write('# most cases should not be edited. If you need more flexibility than\n') - f.write('# this configuration provides, it is strongly suggested that you set\n') - f.write('# up a proper instance of the full build system and use that instead.\n\n') - for line in newlines: - if line.strip() and not line.startswith('#'): - f.write(line) - # Write a newline just in case there's none at the end of the original - f.write('\n') + with open(baseoutpath + '/conf/local.conf', 'w') as f: + f.write('# WARNING: this configuration has been automatically generated and in\n') + f.write('# most cases should not be edited. If you need more flexibility than\n') + f.write('# this configuration provides, it is strongly suggested that you set\n') + f.write('# up a proper instance of the full build system and use that instead.\n\n') + for line in newlines: + if line.strip() and not line.startswith('#'): + f.write(line) + # Write a newline just in case there's none at the end of the original + f.write('\n') - f.write('INHERIT += "%s"\n\n' % 'uninative') - f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) + f.write('INHERIT += "%s"\n\n' % 'uninative') + f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) - # Some classes are not suitable for SDK, remove them from INHERIT - f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False)) + # Some classes are not suitable for SDK, remove them from INHERIT + f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False)) - # Bypass the default connectivity check if any - f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') + # Bypass the default connectivity check if any + f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') - # Ensure locked sstate cache objects are re-used without error - f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "none"\n\n') + # Ensure locked sstate cache objects are re-used without error + f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "none"\n\n') - # Hide the config information from bitbake output (since it's fixed within the SDK) - f.write('BUILDCFG_HEADER = ""\n') + # Hide the config information from bitbake output (since it's fixed within the SDK) + f.write('BUILDCFG_HEADER = ""\n') - # Allow additional config through sdk-extra.conf - fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) - if fn: - with open(fn, 'r') as xf: - for line in xf: - f.write(line) + # Allow additional config through sdk-extra.conf + fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) + if fn: + with open(fn, 'r') as xf: + for line in xf: + f.write(line) - # If you define a sdk_extraconf() function then it can contain additional config - # (Though this is awkward; sdk-extra.conf should probably be used instead) - extraconf = (d.getVar('sdk_extraconf', True) or '').strip() - if extraconf: - # Strip off any leading / trailing spaces - for line in extraconf.splitlines(): - f.write(line.strip() + '\n') + # If you define a sdk_extraconf() function then it can contain additional config + # (Though this is awkward; sdk-extra.conf should probably be used instead) + extraconf = (d.getVar('sdk_extraconf', True) or '').strip() + if extraconf: + # Strip off any leading / trailing spaces + for line in extraconf.splitlines(): + f.write(line.strip() + '\n') - f.write('require conf/locked-sigs.inc\n') + f.write('require conf/locked-sigs.inc\n') if os.path.exists(builddir + '/conf/auto.conf'): - with open(builddir + '/conf/auto.conf', 'r') as f: - oldlines = f.readlines() - (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) - with open(baseoutpath + '/conf/auto.conf', 'w') as f: - f.write('# WARNING: this configuration has been automatically generated and in\n') - f.write('# most cases should not be edited. If you need more flexibility than\n') - f.write('# this configuration provides, it is strongly suggested that you set\n') - f.write('# up a proper instance of the full build system and use that instead.\n\n') - for line in newlines: - if line.strip() and not line.startswith('#'): - f.write(line) + if derivative: + shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') + else: + with open(builddir + '/conf/auto.conf', 'r') as f: + oldlines = f.readlines() + (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) + with open(baseoutpath + '/conf/auto.conf', 'w') as f: + f.write('# WARNING: this configuration has been automatically generated and in\n') + f.write('# most cases should not be edited. If you need more flexibility than\n') + f.write('# this configuration provides, it is strongly suggested that you set\n') + f.write('# up a proper instance of the full build system and use that instead.\n\n') + for line in newlines: + if line.strip() and not line.startswith('#'): + f.write(line) # Filter the locked signatures file to just the sstate tasks we are interested in excluded_targets = d.getVar('SDK_TARGETS', True) @@ -253,7 +266,24 @@ python copy_buildsystem () { lockedsigs_pruned, lockedsigs_copy) - if d.getVar('SDK_EXT_TYPE', True) != 'minimal': + if d.getVar('SDK_EXT_TYPE', True) == 'minimal': + if derivative: + # Assume the user is not going to set up an additional sstate + # mirror, thus we need to copy the additional artifacts (from + # workspace recipes) into the derivative SDK + lockedsigs_orig = d.getVar('TOPDIR', True) + '/conf/locked-sigs.inc' + if os.path.exists(lockedsigs_orig): + lockedsigs_extra = d.getVar('WORKDIR', True) + '/locked-sigs-extra.inc' + oe.copy_buildsystem.merge_lockedsigs(None, + lockedsigs_orig, + lockedsigs_pruned, + None, + lockedsigs_extra) + oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, + d.getVar('SSTATE_DIR', True), + sstate_out, d, + fixedlsbstring) + else: oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, d.getVar('SSTATE_DIR', True), sstate_out, d, diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py index 5074a43cb9..dd0e6641e3 100644 --- a/meta/lib/oe/copy_buildsystem.py +++ b/meta/lib/oe/copy_buildsystem.py @@ -20,7 +20,7 @@ class BuildSystem(object): self.layerdirs = d.getVar('BBLAYERS', True).split() self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split() - def copy_bitbake_and_layers(self, destdir): + def copy_bitbake_and_layers(self, destdir, workspace_name=None): # Copy in all metadata layers + bitbake (as repositories) layers_copied = [] bb.utils.mkdirhier(destdir) @@ -34,6 +34,14 @@ class BuildSystem(object): if layer_exclude in layers: layers.remove(layer_exclude) + workspace_newname = workspace_name + if workspace_newname: + layernames = [os.path.basename(layer) for layer in layers] + extranum = 0 + while workspace_newname in layernames: + extranum += 1 + workspace_newname = '%s-%d' % (workspace_name, extranum) + corebase_files = self.d.getVar('COREBASE_FILES', True).split() corebase_files = [corebase + '/' +x for x in corebase_files] # Make sure bitbake goes in @@ -42,18 +50,24 @@ class BuildSystem(object): for layer in layers: layerconf = os.path.join(layer, 'conf', 'layer.conf') + layernewname = os.path.basename(layer) + workspace = False if os.path.exists(layerconf): with open(layerconf, 'r') as f: if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): - bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) - continue + if workspace_newname: + layernewname = workspace_newname + workspace = True + else: + bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) + continue # If the layer was already under corebase, leave it there # since layers such as meta have issues when moved. layerdestpath = destdir if corebase == os.path.dirname(layer): layerdestpath += '/' + os.path.basename(corebase) - layerdestpath += '/' + os.path.basename(layer) + layerdestpath += '/' + layernewname layer_relative = os.path.relpath(layerdestpath, destdir) @@ -73,6 +87,38 @@ class BuildSystem(object): else: _smart_copy(layer, layerdestpath) + if workspace: + # Make some adjustments original workspace layer + # Drop sources (recipe tasks will be locked, so we don't need them) + srcdir = os.path.join(layerdestpath, 'sources') + if os.path.isdir(srcdir): + shutil.rmtree(srcdir) + # Drop all bbappends except the one for the image the SDK is being built for + # (because of externalsrc, the workspace bbappends will interfere with the + # locked signatures if present, and we don't need them anyway) + image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE', True)))[0] + '.bbappend' + appenddir = os.path.join(layerdestpath, 'appends') + if os.path.isdir(appenddir): + for fn in os.listdir(appenddir): + if fn == image_bbappend: + continue + else: + os.remove(os.path.join(appenddir, fn)) + # Drop README + readme = os.path.join(layerdestpath, 'README') + if os.path.exists(readme): + os.remove(readme) + # Filter out comments in layer.conf and change layer name + layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') + with open(layerconf, 'r') as f: + origlines = f.readlines() + with open(layerconf, 'w') as f: + for line in origlines: + if line.startswith('#'): + continue + line = line.replace('workspacelayer', workspace_newname) + f.write(line) + return layers_copied def generate_locked_sigs(sigfile, d): @@ -123,7 +169,7 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu if line.endswith('\\\n'): if not line in merged[invalue]: target, task = line.strip().split(':')[:2] - if task in copy_tasks: + if not copy_tasks or task in copy_tasks: tocopy[invalue].append(line) merged[invalue].append(line) else: @@ -150,7 +196,8 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) write_sigs_file(copy_output, tocopy.keys(), tocopy) - write_sigs_file(merged_output, arch_order, merged) + if merged_output: + write_sigs_file(merged_output, arch_order, merged) def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): bb.note('Generating sstate-cache...') diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py index ff764fa833..e51d766474 100644 --- a/scripts/lib/devtool/build_image.py +++ b/scripts/lib/devtool/build_image.py @@ -25,6 +25,9 @@ from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, Devtool logger = logging.getLogger('devtool') +class TargetNotImageError(Exception): + pass + def _get_packages(tinfoil, workspace, config): """Get list of packages from recipes in the workspace.""" result = [] @@ -51,6 +54,24 @@ def build_image(args, config, basepath, workspace): if not image: raise DevtoolError('Unable to determine image to build, please specify one') + try: + if args.add_packages: + add_packages = args.add_packages.split(',') + else: + add_packages = None + result, outputdir = build_image_task(config, basepath, workspace, image, add_packages) + except TargetNotImageError: + if auto_image: + raise DevtoolError('Unable to determine image to build, please specify one') + else: + raise DevtoolError('Specified recipe %s is not an image recipe' % image) + + if result == 0: + logger.info('Successfully built %s. You can find output files in %s' + % (image, outputdir)) + return result + +def build_image_task(config, basepath, workspace, image, add_packages=None, task=None, extra_append=None): appendfile = os.path.join(config.workspace_path, 'appends', '%s.bbappend' % image) @@ -63,46 +84,60 @@ def build_image(args, config, basepath, workspace): rd = parse_recipe(config, tinfoil, image, True) if not rd: # Error already shown - return 1 + return (1, None) if not bb.data.inherits_class('image', rd): - if auto_image: - raise DevtoolError('Unable to determine image to build, please specify one') - else: - raise DevtoolError('Specified recipe %s is not an image recipe' % image) + raise TargetNotImageError() + outputdir = None try: - if workspace or args.add_packages: - if args.add_packages: - packages = args.add_packages.split(',') + if workspace or add_packages: + if add_packages: + packages = add_packages else: packages = _get_packages(tinfoil, workspace, config) - if packages: - with open(appendfile, 'w') as afile: + else: + packages = None + if not task: + if not packages and not add_packages and workspace: + logger.warning('No recipes in workspace, building image %s unmodified', image) + elif not packages: + logger.warning('No packages to add, building image %s unmodified', image) + + if packages or extra_append: + bb.utils.mkdirhier(os.path.dirname(appendfile)) + with open(appendfile, 'w') as afile: + if packages: # include packages from workspace recipes into the image afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages)) - logger.info('Building image %s with the following ' - 'additional packages: %s', image, ' '.join(packages)) - else: - logger.warning('No packages to add, building image %s unmodified', image) + if not task: + logger.info('Building image %s with the following ' + 'additional packages: %s', image, ' '.join(packages)) + if extra_append: + for line in extra_append: + afile.write('%s\n' % line) + + if task in ['populate_sdk', 'populate_sdk_ext']: + outputdir = rd.getVar('SDK_DEPLOY', True) else: - logger.warning('No recipes in workspace, building image %s unmodified', image) - - deploy_dir_image = tinfoil.config_data.getVar('DEPLOY_DIR_IMAGE', True) + outputdir = rd.getVar('DEPLOY_DIR_IMAGE', True) tinfoil.shutdown() - # run bitbake to build image + options = '' + if task: + options += '-c %s' % task + + # run bitbake to build image (or specified task) try: exec_build_env_command(config.init_path, basepath, - 'bitbake %s' % image, watch=True) + 'bitbake %s %s' % (options, image), watch=True) except ExecutionError as err: - return err.exitcode + return (err.exitcode, None) finally: if os.path.isfile(appendfile): os.unlink(appendfile) + return (0, outputdir) - logger.info('Successfully built %s. You can find output files in %s' - % (image, deploy_dir_image)) def register_commands(subparsers, context): """Register devtool subcommands from the build-image plugin""" diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py new file mode 100644 index 0000000000..b89d65b0cb --- /dev/null +++ b/scripts/lib/devtool/build_sdk.py @@ -0,0 +1,65 @@ +# Development tool - build-sdk command plugin +# +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import subprocess +import logging +import glob +import shutil +import errno +import sys +import tempfile +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError +from devtool import build_image + +logger = logging.getLogger('devtool') + + +def build_sdk(args, config, basepath, workspace): + """Entry point for the devtool build-sdk command""" + + sdk_targets = config.get('SDK', 'sdk_targets', '').split() + if sdk_targets: + image = sdk_targets[0] + else: + raise DevtoolError('Unable to determine image to build SDK for') + + extra_append = ['SDK_DERIVATIVE = "1"'] + try: + result, outputdir = build_image.build_image_task(config, + basepath, + workspace, + image, + task='populate_sdk_ext', + extra_append=extra_append) + except build_image.TargetNotImageError: + raise DevtoolError('Unable to determine image to build SDK for') + + if result == 0: + logger.info('Successfully built SDK. You can find output files in %s' + % outputdir) + return result + + +def register_commands(subparsers, context): + """Register devtool subcommands""" + if context.fixed_setup: + parser_build_sdk = subparsers.add_parser('build-sdk', + help='Build a derivative SDK of this one', + description='Builds an extensible SDK based upon this one and the items in your workspace', + group='advanced') + parser_build_sdk.set_defaults(func=build_sdk) |