summaryrefslogtreecommitdiff
path: root/scripts/lib/devtool/sdk.py
diff options
context:
space:
mode:
authorJohn Klug <john.klug@multitech.com>2022-05-25 17:12:18 -0500
committerJohn Klug <john.klug@multitech.com>2022-05-25 17:12:18 -0500
commit7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8 (patch)
tree56e91417e6f937b3956cadcf4973756cebb4a8b0 /scripts/lib/devtool/sdk.py
downloadmlinux-7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8.tar.gz
mlinux-7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8.tar.bz2
mlinux-7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8.zip
mLinux 6
Diffstat (limited to 'scripts/lib/devtool/sdk.py')
-rw-r--r--scripts/lib/devtool/sdk.py366
1 files changed, 366 insertions, 0 deletions
diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
new file mode 100644
index 0000000..46fd12b
--- /dev/null
+++ b/scripts/lib/devtool/sdk.py
@@ -0,0 +1,366 @@
+# Development tool - sdk-update command plugin
+#
+# Copyright (C) 2015-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import subprocess
+import logging
+import glob
+import shutil
+import errno
+import sys
+import tempfile
+import re
+from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def parse_locked_sigs(sigfile_path):
+ """Return <pn:task>:<hash> dictionary"""
+ sig_dict = {}
+ with open(sigfile_path) as f:
+ lines = f.readlines()
+ for line in lines:
+ if ':' in line:
+ taskkey, _, hashval = line.rpartition(':')
+ sig_dict[taskkey.strip()] = hashval.split()[0]
+ return sig_dict
+
+def generate_update_dict(sigfile_new, sigfile_old):
+ """Return a dict containing <pn:task>:<hash> which indicates what need to be updated"""
+ update_dict = {}
+ sigdict_new = parse_locked_sigs(sigfile_new)
+ sigdict_old = parse_locked_sigs(sigfile_old)
+ for k in sigdict_new:
+ if k not in sigdict_old:
+ update_dict[k] = sigdict_new[k]
+ continue
+ if sigdict_new[k] != sigdict_old[k]:
+ update_dict[k] = sigdict_new[k]
+ continue
+ return update_dict
+
+def get_sstate_objects(update_dict, sstate_dir):
+ """Return a list containing sstate objects which are to be installed"""
+ sstate_objects = []
+ for k in update_dict:
+ files = set()
+ hashval = update_dict[k]
+ p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz'
+ files |= set(glob.glob(p))
+ p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz'
+ files |= set(glob.glob(p))
+ files = list(files)
+ if len(files) == 1:
+ sstate_objects.extend(files)
+ elif len(files) > 1:
+ logger.error("More than one matching sstate object found for %s" % hashval)
+
+ return sstate_objects
+
+def mkdir(d):
+ try:
+ os.makedirs(d)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise e
+
+def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
+ """Install sstate objects into destination SDK"""
+ sstate_dir = os.path.join(dest_sdk, 'sstate-cache')
+ if not os.path.exists(sstate_dir):
+ logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk)
+ raise
+ for sb in sstate_objects:
+ dst = sb.replace(src_sdk, dest_sdk)
+ destdir = os.path.dirname(dst)
+ mkdir(destdir)
+ logger.debug("Copying %s to %s" % (sb, dst))
+ shutil.copy(sb, dst)
+
+def check_manifest(fn, basepath):
+ import bb.utils
+ changedfiles = []
+ with open(fn, 'r') as f:
+ for line in f:
+ splitline = line.split()
+ if len(splitline) > 1:
+ chksum = splitline[0]
+ fpath = splitline[1]
+ curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath))
+ if chksum != curr_chksum:
+ logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum))
+ changedfiles.append(fpath)
+ return changedfiles
+
+def sdk_update(args, config, basepath, workspace):
+ # Fetch locked-sigs.inc file from remote/local destination
+ updateserver = args.updateserver
+ if not updateserver:
+ updateserver = config.get('SDK', 'updateserver', '')
+ logger.debug("updateserver: %s" % updateserver)
+
+ # Make sure we are using sdk-update from within SDK
+ logger.debug("basepath = %s" % basepath)
+ old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
+ if not os.path.exists(old_locked_sig_file_path):
+ logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option")
+ return -1
+ else:
+ logger.debug("Found conf/locked-sigs.inc in %s" % basepath)
+
+ if ':' in updateserver:
+ is_remote = True
+ else:
+ is_remote = False
+
+ layers_dir = os.path.join(basepath, 'layers')
+ conf_dir = os.path.join(basepath, 'conf')
+
+ # Grab variable values
+ tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
+ try:
+ stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True)
+ sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True)
+ site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True)
+ finally:
+ tinfoil.shutdown()
+
+ if not is_remote:
+ # devtool sdk-update /local/path/to/latest/sdk
+ new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc')
+ if not os.path.exists(new_locked_sig_file_path):
+ logger.error("%s doesn't exist or is not an extensible SDK" % updateserver)
+ return -1
+ else:
+ logger.debug("Found conf/locked-sigs.inc in %s" % updateserver)
+ update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
+ logger.debug("update_dict = %s" % update_dict)
+ newsdk_path = updateserver
+ sstate_dir = os.path.join(newsdk_path, 'sstate-cache')
+ if not os.path.exists(sstate_dir):
+ logger.error("sstate-cache directory not found under %s" % newsdk_path)
+ return 1
+ sstate_objects = get_sstate_objects(update_dict, sstate_dir)
+ logger.debug("sstate_objects = %s" % sstate_objects)
+ if len(sstate_objects) == 0:
+ logger.info("No need to update.")
+ return 0
+ logger.info("Installing sstate objects into %s", basepath)
+ install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath)
+ logger.info("Updating configuration files")
+ new_conf_dir = os.path.join(updateserver, 'conf')
+ shutil.rmtree(conf_dir)
+ shutil.copytree(new_conf_dir, conf_dir)
+ logger.info("Updating layers")
+ new_layers_dir = os.path.join(updateserver, 'layers')
+ shutil.rmtree(layers_dir)
+ ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True)
+ if ret != 0:
+ logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir))
+ return ret
+ else:
+ # devtool sdk-update http://myhost/sdk
+ tmpsdk_dir = tempfile.mkdtemp()
+ try:
+ os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
+ new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
+ # Fetch manifest from server
+ tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
+ ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
+ changedfiles = check_manifest(tmpmanifest, basepath)
+ if not changedfiles:
+ logger.info("Already up-to-date")
+ return 0
+ # Update metadata
+ logger.debug("Updating metadata via git ...")
+ #Check for the status before doing a fetch and reset
+ if os.path.exists(os.path.join(basepath, 'layers/.git')):
+ out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir)
+ if not out:
+ ret = subprocess.call("git fetch --all; git reset --hard", shell=True, cwd=layers_dir)
+ else:
+ logger.error("Failed to update metadata as there have been changes made to it. Aborting.");
+ logger.error("Changed files:\n%s" % out);
+ return -1
+ else:
+ ret = -1
+ if ret != 0:
+ ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
+ if ret != 0:
+ logger.error("Updating metadata via git failed")
+ return ret
+ logger.debug("Updating conf files ...")
+ for changedfile in changedfiles:
+ ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
+ if ret != 0:
+ logger.error("Updating %s failed" % changedfile)
+ return ret
+
+ # Check if UNINATIVE_CHECKSUM changed
+ uninative = False
+ if 'conf/local.conf' in changedfiles:
+ def read_uninative_checksums(fn):
+ chksumitems = []
+ with open(fn, 'r') as f:
+ for line in f:
+ if line.startswith('UNINATIVE_CHECKSUM'):
+ splitline = re.split(r'[\[\]"\']', line)
+ if len(splitline) > 3:
+ chksumitems.append((splitline[1], splitline[3]))
+ return chksumitems
+
+ oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf'))
+ newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf'))
+ if oldsums != newsums:
+ uninative = True
+ for buildarch, chksum in newsums:
+ uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch)
+ mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file)))
+ ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir)
+
+ # Ok, all is well at this point - move everything over
+ tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
+ if os.path.exists(tmplayers_dir):
+ shutil.rmtree(layers_dir)
+ shutil.move(tmplayers_dir, layers_dir)
+ for changedfile in changedfiles:
+ destfile = os.path.join(basepath, changedfile)
+ os.remove(destfile)
+ shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile)
+ os.remove(os.path.join(conf_dir, 'sdk-conf-manifest'))
+ shutil.move(tmpmanifest, conf_dir)
+ if uninative:
+ shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative'))
+ shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads'))
+
+ if not sstate_mirrors:
+ with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
+ f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
+ f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
+ finally:
+ shutil.rmtree(tmpsdk_dir)
+
+ if not args.skip_prepare:
+ # Find all potentially updateable tasks
+ sdk_update_targets = []
+ tasks = ['do_populate_sysroot', 'do_packagedata']
+ for root, _, files in os.walk(stamps_dir):
+ for fn in files:
+ if not '.sigdata.' in fn:
+ for task in tasks:
+ if '.%s.' % task in fn or '.%s_setscene.' % task in fn:
+ sdk_update_targets.append('%s:%s' % (os.path.basename(root), task))
+ # Run bitbake command for the whole SDK
+ logger.info("Preparing build system... (This may take some time.)")
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
+ output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
+ runlines = []
+ for line in output.splitlines():
+ if 'Running task ' in line:
+ runlines.append(line)
+ if runlines:
+ logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines))
+ return -1
+ except bb.process.ExecutionError as e:
+ logger.error('Preparation failed:\n%s' % e.stdout)
+ return -1
+ return 0
+
+def sdk_install(args, config, basepath, workspace):
+ """Entry point for the devtool sdk-install command"""
+
+ import oe.recipeutils
+ import bb.process
+
+ for recipe in args.recipename:
+ if recipe in workspace:
+ raise DevtoolError('recipe %s is a recipe in your workspace' % recipe)
+
+ tasks = ['do_populate_sysroot', 'do_packagedata']
+ stampprefixes = {}
+ def checkstamp(recipe):
+ stampprefix = stampprefixes[recipe]
+ stamps = glob.glob(stampprefix + '*')
+ for stamp in stamps:
+ if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')):
+ return True
+ else:
+ return False
+
+ install_recipes = []
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ for recipe in args.recipename:
+ rd = parse_recipe(config, tinfoil, recipe, True)
+ if not rd:
+ return 1
+ stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP', True), tasks[0])
+ if checkstamp(recipe):
+ logger.info('%s is already installed' % recipe)
+ else:
+ install_recipes.append(recipe)
+ finally:
+ tinfoil.shutdown()
+
+ if install_recipes:
+ logger.info('Installing %s...' % ', '.join(install_recipes))
+ install_tasks = []
+ for recipe in install_recipes:
+ for task in tasks:
+ if recipe.endswith('-native') and 'package' in task:
+ continue
+ install_tasks.append('%s:%s' % (recipe, task))
+ options = ''
+ if not args.allow_build:
+ options += ' --setscene-only'
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True)
+ except bb.process.ExecutionError as e:
+ raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e)))
+ failed = False
+ for recipe in install_recipes:
+ if checkstamp(recipe):
+ logger.info('Successfully installed %s' % recipe)
+ else:
+ raise DevtoolError('Failed to install %s - unavailable' % recipe)
+ failed = True
+ if failed:
+ return 2
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from the sdk plugin"""
+ if context.fixed_setup:
+ parser_sdk = subparsers.add_parser('sdk-update',
+ help='Update SDK components',
+ description='Updates installed SDK components from a remote server',
+ group='sdk')
+ updateserver = context.config.get('SDK', 'updateserver', '')
+ if updateserver:
+ parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?')
+ else:
+ parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from')
+ parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
+ parser_sdk.set_defaults(func=sdk_update)
+
+ parser_sdk_install = subparsers.add_parser('sdk-install',
+ help='Install additional SDK components',
+ description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)',
+ group='sdk')
+ parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+')
+ parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true')
+ parser_sdk_install.set_defaults(func=sdk_install)