devtool: sdk-update: improve SDK update process robustness

Make the following improvements to the SDK update process:

* Use a manifest file with sha256sums to track files other than sstate
  and metadata that we need to update - e.g. conf files. This allows us
  to handle where files such as auto.conf may or may not be present,
  as well as the configuration changing without affecting task signatures
  - we still want the config files copied in that case rather than it
  saying nothing needs to be done.
* Write the SSTATE_MIRRORS_append to site.conf rather than local.conf
  so that local.conf remains static (since we don't want to trigger an
  update every time). Also, If there is an SSTATE_MIRRORS value already
  set in the configuration we can skip this and assume it contains the
  needed packages.
* Allow the update process to be run in any directory, don't assume
  we're already at the base of the SDK
* Where practical, fetch remote files into a temporary location and
  then move them to the desired location at the end, to avoid a
  failed update leaving the SDK in a broken state.
* Update all installed do_populate_sysroot / do_packagedata tasks
  instead of using the SDK targets. This ensures any item installed
  through dependencies after installation (e.g. when running
  "devtool build") won't go stale.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 84 insertions, 34 deletions

diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index 4fcd36a0df..80ea8711cc 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -75,6 +75,21 @@ def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
         logger.debug("Copying %s to %s" % (sb, dst))
         shutil.copy(sb, dst)
 
+def check_manifest(fn, basepath):
+    import bb.utils
+    changedfiles = []
+    with open(fn, 'r') as f:
+        for line in f:
+            splitline = line.split()
+            if len(splitline) > 1:
+                chksum = splitline[0]
+                fpath = splitline[1]
+                curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath))
+                if chksum != curr_chksum:
+                    logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum))
+                    changedfiles.append(fpath)
+    return changedfiles
+
 def sdk_update(args, config, basepath, workspace):
     # Fetch locked-sigs.inc file from remote/local destination
     updateserver = args.updateserver
@@ -98,6 +113,18 @@ def sdk_update(args, config, basepath, workspace):
     else:
         is_remote = False
 
+    layers_dir = os.path.join(basepath, 'layers')
+    conf_dir = os.path.join(basepath, 'conf')
+
+    # Grab variable values
+    tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
+    try:
+        stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True)
+        sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True)
+        site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True)
+    finally:
+        tinfoil.shutdown()
+
     if not is_remote:
         # devtool sdk-update /local/path/to/latest/sdk
         new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc')
@@ -121,16 +148,14 @@ def sdk_update(args, config, basepath, workspace):
         install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath)
         logger.info("Updating configuration files")
         new_conf_dir = os.path.join(updateserver, 'conf')
-        old_conf_dir = os.path.join(basepath, 'conf')
-        shutil.rmtree(old_conf_dir)
-        shutil.copytree(new_conf_dir, old_conf_dir)
+        shutil.rmtree(conf_dir)
+        shutil.copytree(new_conf_dir, conf_dir)
         logger.info("Updating layers")
         new_layers_dir = os.path.join(updateserver, 'layers')
-        old_layers_dir = os.path.join(basepath, 'layers')
-        shutil.rmtree(old_layers_dir)
-        ret = subprocess.call("cp -a %s %s" % (new_layers_dir, old_layers_dir), shell=True)
+        shutil.rmtree(layers_dir)
+        ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True)
         if ret != 0:
-            logger.error("Copying %s to %s failed" % (new_layers_dir, old_layers_dir))
+            logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir))
             return ret
     else:
         # devtool sdk-update http://myhost/sdk
@@ -138,50 +163,75 @@ def sdk_update(args, config, basepath, workspace):
         try:
             os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
             new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
-            # Fetch locked-sigs.inc from update server
-            ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s" % (updateserver, new_locked_sig_file_path), shell=True)
-            if ret != 0:
-                logger.error("Fetching conf/locked-sigs.inc from %s to %s failed" % (updateserver, new_locked_sig_file_path))
-                return ret
-            else:
-                logger.info("Fetching conf/locked-sigs.inc from %s to %s succeeded" % (updateserver, new_locked_sig_file_path))
-            update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
-            logger.debug("update_dict = %s" % update_dict)
-            if len(update_dict) == 0:
-                logger.info("No need to update.")
+            # Fetch manifest from server
+            tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
+            ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
+            changedfiles = check_manifest(tmpmanifest, basepath)
+            if not changedfiles:
+                logger.info("Already up-to-date")
                 return 0
             # Update metadata
-            logger.debug("Updating meta data via git ...")
+            logger.debug("Updating metadata via git ...")
             # Try using 'git pull', if failed, use 'git clone'
             if os.path.exists(os.path.join(basepath, 'layers/.git')):
-                ret = subprocess.call("cd layers && git pull %s/layers/.git" % updateserver, shell=True)
+                ret = subprocess.call("git pull %s/layers/.git" % updateserver, shell=True, cwd=layers_dir)
             else:
                 ret = -1
             if ret != 0:
-                ret = subprocess.call("rm -rf layers && git clone %s/layers/.git" % updateserver, shell=True)
-            if ret != 0:
-                logger.error("Updating meta data via git failed")
-                return ret
+                ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
+                if ret != 0:
+                    logger.error("Updating metadata via git failed")
+                    return ret
             logger.debug("Updating conf files ...")
-            conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'locked-sigs.inc']
-            for conf in conf_files:
-                ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (updateserver, conf, conf), shell=True)
+            for changedfile in changedfiles:
+                ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
                 if ret != 0:
-                    logger.error("Update %s failed" % conf)
+                    logger.error("Updating %s failed" % changedfile)
                     return ret
-            with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f:
-                f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
+
+            # Ok, all is well at this point - move everything over
+            tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
+            if os.path.exists(tmplayers_dir):
+                shutil.rmtree(layers_dir)
+                shutil.move(tmplayers_dir, layers_dir)
+            for changedfile in changedfiles:
+                destfile = os.path.join(basepath, changedfile)
+                os.remove(destfile)
+                shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile)
+            os.remove(os.path.join(conf_dir, 'sdk-conf-manifest'))
+            shutil.move(tmpmanifest, conf_dir)
+
+            if not sstate_mirrors:
+                with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
+                    f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
+                    f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
         finally:
             shutil.rmtree(tmpsdk_dir)
 
     if not args.skip_prepare:
+        # Find all potentially updateable tasks
+        sdk_update_targets = []
+        tasks = ['do_populate_sysroot', 'do_packagedata']
+        for root, _, files in os.walk(stamps_dir):
+            for fn in files:
+                if not '.sigdata.' in fn:
+                    for task in tasks:
+                        if '.%s.' % task in fn or '.%s_setscene.' % task in fn:
+                            sdk_update_targets.append('%s:%s' % (os.path.basename(root), task))
         # Run bitbake command for the whole SDK
-        sdk_update_targets = config.get('SDK', 'sdk_update_targets', config.get('SDK', 'sdk_targets'))
         logger.info("Preparing build system... (This may take some time.)")
         try:
-            exec_build_env_command(config.init_path, basepath, 'bitbake %s --setscene-only' % sdk_update_targets)
-        except:
-            logger.error('bitbake %s failed' % sdk_update_targets)
+            exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
+            output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
+            runlines = []
+            for line in output.splitlines():
+                if 'Running task ' in line:
+                    runlines.append(line)
+            if runlines:
+                logger.error('Unexecuted tasks found in preparation log:\n  %s' % '\n  '.join(runlines))
+                return -1
+        except bb.process.ExecutionError as e:
+            logger.error('Preparation failed:\n%s' % e.stdout)
             return -1
     return 0