diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/lib/compatlayer/__init__.py | 7 | ||||
| -rw-r--r-- | scripts/lib/compatlayer/cases/bsp.py | 154 | ||||
| -rwxr-xr-x | scripts/yocto-compat-layer.py | 3 |
3 files changed, 161 insertions, 3 deletions
diff --git a/scripts/lib/compatlayer/__init__.py b/scripts/lib/compatlayer/__init__.py index 6130b8548c..0d6f4e9c3b 100644 --- a/scripts/lib/compatlayer/__init__.py +++ b/scripts/lib/compatlayer/__init__.py @@ -215,7 +215,7 @@ def check_command(error_msg, cmd): raise RuntimeError(msg) return output -def get_signatures(builddir, failsafe=False): +def get_signatures(builddir, failsafe=False, machine=None): import re # some recipes needs to be excluded like meta-world-pkgdata @@ -226,7 +226,10 @@ def get_signatures(builddir, failsafe=False): sigs = {} tune2tasks = {} - cmd = 'bitbake ' + cmd = '' + if machine: + cmd += 'MACHINE=%s ' % machine + cmd += 'bitbake ' if failsafe: cmd += '-k ' cmd += '-S none world' diff --git a/scripts/lib/compatlayer/cases/bsp.py b/scripts/lib/compatlayer/cases/bsp.py index 5d9bf93e4a..90256750af 100644 --- a/scripts/lib/compatlayer/cases/bsp.py +++ b/scripts/lib/compatlayer/cases/bsp.py @@ -3,7 +3,7 @@ import unittest -from compatlayer import LayerType +from compatlayer import LayerType, get_signatures, check_command, get_depgraph from compatlayer.case import OECompatLayerTestCase class BSPCompatLayer(OECompatLayerTestCase): @@ -24,3 +24,155 @@ class BSPCompatLayer(OECompatLayerTestCase): self.assertEqual(self.td['bbvars']['MACHINE'], machine, msg="Layer %s modified machine %s -> %s" % \ (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine)) + + def test_machine_signatures(self): + ''' + Selecting a machine may only affect the signature of tasks that are specific + to that machine. In other words, when MACHINE=A and MACHINE=B share a recipe + foo and the output of foo, then both machine configurations must build foo + in exactly the same way. Otherwise it is not possible to use both machines + in the same distribution. + + This criteria can only be tested by testing different machines in combination, + i.e. one main layer, potentially several additional BSP layers and an explicit + choice of machines: + yocto-compat-layer --additional-layers .../meta-intel --machines intel-corei7-64 imx6slevk -- .../meta-freescale + ''' + + if not self.td['machines']: + self.skipTest('No machines set with --machines.') + + # Collect signatures for all machines that we are testing + # and merge that into a hash: + # tune -> task -> signature -> list of machines with that combination + # + # It is an error if any tune/task pair has more than one signature, + # because that implies that the machines that caused those different + # signatures do not agree on how to execute the task. + tunes = {} + # Preserve ordering of machines as chosen by the user. + for machine in self.td['machines']: + curr_sigs, tune2tasks = get_signatures(self.td['builddir'], failsafe=True, machine=machine) + # Invert the tune -> [tasks] mapping. + tasks2tune = {} + for tune, tasks in tune2tasks.items(): + for task in tasks: + tasks2tune[task] = tune + for task, sighash in curr_sigs.items(): + tunes.setdefault(tasks2tune[task], {}).setdefault(task, {}).setdefault(sighash, []).append(machine) + + msg = [] + pruned = 0 + last_line_key = None + # do_fetch, do_unpack, ..., do_build + taskname_list = [] + if tunes: + # The output below is most useful when we start with tasks that are at + # the bottom of the dependency chain, i.e. those that run first. If + # those tasks differ, the rest also does. + # + # To get an ordering of tasks, we do a topological sort of the entire + # depgraph for the base configuration, then on-the-fly flatten that list by stripping + # out the recipe names and removing duplicates. The base configuration + # is not necessarily representative, but should be close enough. Tasks + # that were not encountered get a default priority. + depgraph = get_depgraph() + depends = depgraph['tdepends'] + WHITE = 1 + GRAY = 2 + BLACK = 3 + color = {} + found = set() + def visit(task): + color[task] = GRAY + for dep in depends.get(task, ()): + if color.setdefault(dep, WHITE) == WHITE: + visit(dep) + color[task] = BLACK + pn, taskname = task.rsplit('.', 1) + if taskname not in found: + taskname_list.append(taskname) + found.add(taskname) + for task in depends.keys(): + if color.setdefault(task, WHITE) == WHITE: + visit(task) + + taskname_order = dict([(task, index) for index, task in enumerate(taskname_list) ]) + def task_key(task): + pn, taskname = task.rsplit(':', 1) + return (pn, taskname_order.get(taskname, len(taskname_list)), taskname) + + for tune in sorted(tunes.keys()): + tasks = tunes[tune] + # As for test_signatures it would be nicer to sort tasks + # by dependencies here, but that is harder because we have + # to report on tasks from different machines, which might + # have different dependencies. We resort to pruning the + # output by reporting only one task per recipe if the set + # of machines matches. + # + # "bitbake-diffsigs -t -s" is intelligent enough to print + # diffs recursively, so often it does not matter that much + # if we don't pick the underlying difference + # here. However, sometimes recursion fails + # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=6428). + # + # To mitigate that a bit, we use a hard-coded ordering of + # tasks that represents how they normally run and prefer + # to print the ones that run first. + for task in sorted(tasks.keys(), key=task_key): + signatures = tasks[task] + # do_build can be ignored: it is know to have + # different signatures in some cases, for example in + # the allarch ca-certificates due to RDEPENDS=openssl. + # That particular dependency is whitelisted via + # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up + # in the sstate signature hash because filtering it + # out would be hard and running do_build multiple + # times doesn't really matter. + if len(signatures.keys()) > 1 and \ + not task.endswith(':do_build'): + # Error! + # + # Sort signatures by machines, because the hex values don't mean anything. + # => all-arch adwaita-icon-theme:do_build: 1234... (beaglebone, qemux86) != abcdf... (qemux86-64) + # + # Skip the line if it is covered already by the predecessor (same pn, same sets of machines). + pn, taskname = task.rsplit(':', 1) + next_line_key = (pn, sorted(signatures.values())) + if next_line_key != last_line_key: + line = ' %s %s: ' % (tune, task) + line += ' != '.join(['%s (%s)' % (signature, ', '.join([m for m in signatures[signature]])) for + signature in sorted(signatures.keys(), key=lambda s: signatures[s])]) + last_line_key = next_line_key + msg.append(line) + # Randomly pick two mismatched signatures and remember how to invoke + # bitbake-diffsigs for them. + iterator = iter(signatures.items()) + a = next(iterator) + b = next(iterator) + diffsig_machines = '(%s) != (%s)' % (', '.join(a[1]), ', '.join(b[1])) + diffsig_params = '-t %s %s -s %s %s' % (pn, taskname, a[0], b[0]) + else: + pruned += 1 + + if msg: + msg.insert(0, 'The machines have conflicting signatures for some shared tasks:') + if pruned > 0: + msg.append('') + msg.append('%d tasks where not listed because some other task of the recipe already differed.' % pruned) + msg.append('It is likely that differences from different recipes also have the same root cause.') + msg.append('') + # Explain how to investigate... + msg.append('To investigate, run bitbake-diffsigs -t recipename taskname -s fromsig tosig.') + cmd = 'bitbake-diffsigs %s' % diffsig_params + msg.append('Example: %s in the last line' % diffsig_machines) + msg.append('Command: %s' % cmd) + # ... and actually do it automatically for that example, but without aborting + # when that fails. + try: + output = check_command('Comparing signatures failed.', cmd).decode('utf-8') + except RuntimeError as ex: + output = str(ex) + msg.extend([' ' + line for line in output.splitlines()]) + self.fail('\n'.join(msg)) diff --git a/scripts/yocto-compat-layer.py b/scripts/yocto-compat-layer.py index daf3a6a32a..0d5700b538 100755 --- a/scripts/yocto-compat-layer.py +++ b/scripts/yocto-compat-layer.py @@ -49,6 +49,8 @@ def main(): help='File to output log (optional)', action='store') parser.add_argument('--dependency', nargs="+", help='Layers to process for dependencies', action='store') + parser.add_argument('--machines', nargs="+", + help='List of MACHINEs to be used during testing', action='store') parser.add_argument('--additional-layers', nargs="+", help='List of additional layers to add during testing', action='store') parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery', @@ -162,6 +164,7 @@ def main(): logger.info('Getting initial signatures ...') td['builddir'] = builddir td['sigs'], td['tunetasks'] = get_signatures(td['builddir']) + td['machines'] = args.machines if not add_layer(bblayersconf, layer, dep_layers, logger): logger.info('Skipping %s ???.' % layer['name']) |