summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/lib/compatlayer/__init__.py7
-rw-r--r--scripts/lib/compatlayer/cases/bsp.py154
-rwxr-xr-xscripts/yocto-compat-layer.py3
3 files changed, 161 insertions, 3 deletions
diff --git a/scripts/lib/compatlayer/__init__.py b/scripts/lib/compatlayer/__init__.py
index 6130b8548c..0d6f4e9c3b 100644
--- a/scripts/lib/compatlayer/__init__.py
+++ b/scripts/lib/compatlayer/__init__.py
@@ -215,7 +215,7 @@ def check_command(error_msg, cmd):
raise RuntimeError(msg)
return output
-def get_signatures(builddir, failsafe=False):
+def get_signatures(builddir, failsafe=False, machine=None):
import re
# some recipes needs to be excluded like meta-world-pkgdata
@@ -226,7 +226,10 @@ def get_signatures(builddir, failsafe=False):
sigs = {}
tune2tasks = {}
- cmd = 'bitbake '
+ cmd = ''
+ if machine:
+ cmd += 'MACHINE=%s ' % machine
+ cmd += 'bitbake '
if failsafe:
cmd += '-k '
cmd += '-S none world'
diff --git a/scripts/lib/compatlayer/cases/bsp.py b/scripts/lib/compatlayer/cases/bsp.py
index 5d9bf93e4a..90256750af 100644
--- a/scripts/lib/compatlayer/cases/bsp.py
+++ b/scripts/lib/compatlayer/cases/bsp.py
@@ -3,7 +3,7 @@
import unittest
-from compatlayer import LayerType
+from compatlayer import LayerType, get_signatures, check_command, get_depgraph
from compatlayer.case import OECompatLayerTestCase
class BSPCompatLayer(OECompatLayerTestCase):
@@ -24,3 +24,155 @@ class BSPCompatLayer(OECompatLayerTestCase):
self.assertEqual(self.td['bbvars']['MACHINE'], machine,
msg="Layer %s modified machine %s -> %s" % \
(self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine))
+
+ def test_machine_signatures(self):
+ '''
+ Selecting a machine may only affect the signature of tasks that are specific
+ to that machine. In other words, when MACHINE=A and MACHINE=B share a recipe
+ foo and the output of foo, then both machine configurations must build foo
+ in exactly the same way. Otherwise it is not possible to use both machines
+ in the same distribution.
+
+ This criteria can only be tested by testing different machines in combination,
+ i.e. one main layer, potentially several additional BSP layers and an explicit
+ choice of machines:
+ yocto-compat-layer --additional-layers .../meta-intel --machines intel-corei7-64 imx6slevk -- .../meta-freescale
+ '''
+
+ if not self.td['machines']:
+ self.skipTest('No machines set with --machines.')
+
+ # Collect signatures for all machines that we are testing
+ # and merge that into a hash:
+ # tune -> task -> signature -> list of machines with that combination
+ #
+ # It is an error if any tune/task pair has more than one signature,
+ # because that implies that the machines that caused those different
+ # signatures do not agree on how to execute the task.
+ tunes = {}
+ # Preserve ordering of machines as chosen by the user.
+ for machine in self.td['machines']:
+ curr_sigs, tune2tasks = get_signatures(self.td['builddir'], failsafe=True, machine=machine)
+ # Invert the tune -> [tasks] mapping.
+ tasks2tune = {}
+ for tune, tasks in tune2tasks.items():
+ for task in tasks:
+ tasks2tune[task] = tune
+ for task, sighash in curr_sigs.items():
+ tunes.setdefault(tasks2tune[task], {}).setdefault(task, {}).setdefault(sighash, []).append(machine)
+
+ msg = []
+ pruned = 0
+ last_line_key = None
+ # do_fetch, do_unpack, ..., do_build
+ taskname_list = []
+ if tunes:
+ # The output below is most useful when we start with tasks that are at
+ # the bottom of the dependency chain, i.e. those that run first. If
+ # those tasks differ, the rest also does.
+ #
+ # To get an ordering of tasks, we do a topological sort of the entire
+ # depgraph for the base configuration, then on-the-fly flatten that list by stripping
+ # out the recipe names and removing duplicates. The base configuration
+ # is not necessarily representative, but should be close enough. Tasks
+ # that were not encountered get a default priority.
+ depgraph = get_depgraph()
+ depends = depgraph['tdepends']
+ WHITE = 1
+ GRAY = 2
+ BLACK = 3
+ color = {}
+ found = set()
+ def visit(task):
+ color[task] = GRAY
+ for dep in depends.get(task, ()):
+ if color.setdefault(dep, WHITE) == WHITE:
+ visit(dep)
+ color[task] = BLACK
+ pn, taskname = task.rsplit('.', 1)
+ if taskname not in found:
+ taskname_list.append(taskname)
+ found.add(taskname)
+ for task in depends.keys():
+ if color.setdefault(task, WHITE) == WHITE:
+ visit(task)
+
+ taskname_order = dict([(task, index) for index, task in enumerate(taskname_list) ])
+ def task_key(task):
+ pn, taskname = task.rsplit(':', 1)
+ return (pn, taskname_order.get(taskname, len(taskname_list)), taskname)
+
+ for tune in sorted(tunes.keys()):
+ tasks = tunes[tune]
+ # As for test_signatures it would be nicer to sort tasks
+ # by dependencies here, but that is harder because we have
+ # to report on tasks from different machines, which might
+ # have different dependencies. We resort to pruning the
+ # output by reporting only one task per recipe if the set
+ # of machines matches.
+ #
+ # "bitbake-diffsigs -t -s" is intelligent enough to print
+ # diffs recursively, so often it does not matter that much
+ # if we don't pick the underlying difference
+ # here. However, sometimes recursion fails
+ # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=6428).
+ #
+ # To mitigate that a bit, we use a hard-coded ordering of
+ # tasks that represents how they normally run and prefer
+ # to print the ones that run first.
+ for task in sorted(tasks.keys(), key=task_key):
+ signatures = tasks[task]
+ # do_build can be ignored: it is know to have
+ # different signatures in some cases, for example in
+ # the allarch ca-certificates due to RDEPENDS=openssl.
+ # That particular dependency is whitelisted via
+ # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up
+ # in the sstate signature hash because filtering it
+ # out would be hard and running do_build multiple
+ # times doesn't really matter.
+ if len(signatures.keys()) > 1 and \
+ not task.endswith(':do_build'):
+ # Error!
+ #
+ # Sort signatures by machines, because the hex values don't mean anything.
+ # => all-arch adwaita-icon-theme:do_build: 1234... (beaglebone, qemux86) != abcdf... (qemux86-64)
+ #
+ # Skip the line if it is covered already by the predecessor (same pn, same sets of machines).
+ pn, taskname = task.rsplit(':', 1)
+ next_line_key = (pn, sorted(signatures.values()))
+ if next_line_key != last_line_key:
+ line = ' %s %s: ' % (tune, task)
+ line += ' != '.join(['%s (%s)' % (signature, ', '.join([m for m in signatures[signature]])) for
+ signature in sorted(signatures.keys(), key=lambda s: signatures[s])])
+ last_line_key = next_line_key
+ msg.append(line)
+ # Randomly pick two mismatched signatures and remember how to invoke
+ # bitbake-diffsigs for them.
+ iterator = iter(signatures.items())
+ a = next(iterator)
+ b = next(iterator)
+ diffsig_machines = '(%s) != (%s)' % (', '.join(a[1]), ', '.join(b[1]))
+ diffsig_params = '-t %s %s -s %s %s' % (pn, taskname, a[0], b[0])
+ else:
+ pruned += 1
+
+ if msg:
+ msg.insert(0, 'The machines have conflicting signatures for some shared tasks:')
+ if pruned > 0:
+ msg.append('')
+ msg.append('%d tasks where not listed because some other task of the recipe already differed.' % pruned)
+ msg.append('It is likely that differences from different recipes also have the same root cause.')
+ msg.append('')
+ # Explain how to investigate...
+ msg.append('To investigate, run bitbake-diffsigs -t recipename taskname -s fromsig tosig.')
+ cmd = 'bitbake-diffsigs %s' % diffsig_params
+ msg.append('Example: %s in the last line' % diffsig_machines)
+ msg.append('Command: %s' % cmd)
+ # ... and actually do it automatically for that example, but without aborting
+ # when that fails.
+ try:
+ output = check_command('Comparing signatures failed.', cmd).decode('utf-8')
+ except RuntimeError as ex:
+ output = str(ex)
+ msg.extend([' ' + line for line in output.splitlines()])
+ self.fail('\n'.join(msg))
diff --git a/scripts/yocto-compat-layer.py b/scripts/yocto-compat-layer.py
index daf3a6a32a..0d5700b538 100755
--- a/scripts/yocto-compat-layer.py
+++ b/scripts/yocto-compat-layer.py
@@ -49,6 +49,8 @@ def main():
help='File to output log (optional)', action='store')
parser.add_argument('--dependency', nargs="+",
help='Layers to process for dependencies', action='store')
+ parser.add_argument('--machines', nargs="+",
+ help='List of MACHINEs to be used during testing', action='store')
parser.add_argument('--additional-layers', nargs="+",
help='List of additional layers to add during testing', action='store')
parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery',
@@ -162,6 +164,7 @@ def main():
logger.info('Getting initial signatures ...')
td['builddir'] = builddir
td['sigs'], td['tunetasks'] = get_signatures(td['builddir'])
+ td['machines'] = args.machines
if not add_layer(bblayersconf, layer, dep_layers, logger):
logger.info('Skipping %s ???.' % layer['name'])