diff options
Diffstat (limited to 'meta/lib')
212 files changed, 18951 insertions, 4030 deletions
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py new file mode 100644 index 0000000000..c5d4c73cf5 --- /dev/null +++ b/meta/lib/buildstats.py @@ -0,0 +1,158 @@ +# Implements system state sampling. Called by buildstats.bbclass. +# Because it is a real Python module, it can hold persistent state, +# like open log files and the time of the last sampling. + +import time +import re +import bb.event + +class SystemStats: + def __init__(self, d): + bn = d.getVar('BUILDNAME') + bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) + bb.utils.mkdirhier(bsdir) + + self.proc_files = [] + for filename, handler in ( + ('diskstats', self._reduce_diskstats), + ('meminfo', self._reduce_meminfo), + ('stat', self._reduce_stat), + ): + # The corresponding /proc files might not exist on the host. + # For example, /proc/diskstats is not available in virtualized + # environments like Linux-VServer. Silently skip collecting + # the data. + if os.path.exists(os.path.join('/proc', filename)): + # In practice, this class gets instantiated only once in + # the bitbake cooker process. Therefore 'append' mode is + # not strictly necessary, but using it makes the class + # more robust should two processes ever write + # concurrently. + destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if handler else '', filename)) + self.proc_files.append((filename, open(destfile, 'ab'), handler)) + self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab') + # Last time that we sampled /proc data resp. recorded disk monitoring data. + self.last_proc = 0 + self.last_disk_monitor = 0 + # Minimum number of seconds between recording a sample. This + # becames relevant when we get called very often while many + # short tasks get started. Sampling during quiet periods + # depends on the heartbeat event, which fires less often. + self.min_seconds = 1 + + self.meminfo_regex = re.compile(b'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)') + self.diskstats_regex = re.compile(b'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') + self.diskstats_ltime = None + self.diskstats_data = None + self.stat_ltimes = None + + def close(self): + self.monitor_disk.close() + for _, output, _ in self.proc_files: + output.close() + + def _reduce_meminfo(self, time, data): + """ + Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' + and writes their values into a single line, in that order. + """ + values = {} + for line in data.split(b'\n'): + m = self.meminfo_regex.match(line) + if m: + values[m.group(1)] = m.group(2) + if len(values) == 6: + return (time, + b' '.join([values[x] for x in + (b'MemTotal', b'MemFree', b'Buffers', b'Cached', b'SwapTotal', b'SwapFree')]) + b'\n') + + def _diskstats_is_relevant_line(self, linetokens): + if len(linetokens) != 14: + return False + disk = linetokens[2] + return self.diskstats_regex.match(disk) + + def _reduce_diskstats(self, time, data): + relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n'))) + diskdata = [0] * 3 + reduced = None + for tokens in relevant_tokens: + # rsect + diskdata[0] += int(tokens[5]) + # wsect + diskdata[1] += int(tokens[9]) + # use + diskdata[2] += int(tokens[12]) + if self.diskstats_ltime: + # We need to compute information about the time interval + # since the last sampling and record the result as sample + # for that point in the past. + interval = time - self.diskstats_ltime + if interval > 0: + sums = [ a - b for a, b in zip(diskdata, self.diskstats_data) ] + readTput = sums[0] / 2.0 * 100.0 / interval + writeTput = sums[1] / 2.0 * 100.0 / interval + util = float( sums[2] ) / 10 / interval + util = max(0.0, min(1.0, util)) + reduced = (self.diskstats_ltime, (readTput, writeTput, util)) + + self.diskstats_ltime = time + self.diskstats_data = diskdata + return reduced + + + def _reduce_nop(self, time, data): + return (time, data) + + def _reduce_stat(self, time, data): + if not data: + return None + # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line + tokens = data.split(b'\n', 1)[0].split() + times = [ int(token) for token in tokens[1:] ] + reduced = None + if self.stat_ltimes: + user = float((times[0] + times[1]) - (self.stat_ltimes[0] + self.stat_ltimes[1])) + system = float((times[2] + times[5] + times[6]) - (self.stat_ltimes[2] + self.stat_ltimes[5] + self.stat_ltimes[6])) + idle = float(times[3] - self.stat_ltimes[3]) + iowait = float(times[4] - self.stat_ltimes[4]) + + aSum = max(user + system + idle + iowait, 1) + reduced = (time, (user/aSum, system/aSum, iowait/aSum)) + + self.stat_ltimes = times + return reduced + + def sample(self, event, force): + now = time.time() + if (now - self.last_proc > self.min_seconds) or force: + for filename, output, handler in self.proc_files: + with open(os.path.join('/proc', filename), 'rb') as input: + data = input.read() + if handler: + reduced = handler(now, data) + else: + reduced = (now, data) + if reduced: + if isinstance(reduced[1], bytes): + # Use as it is. + data = reduced[1] + else: + # Convert to a single line. + data = (' '.join([str(x) for x in reduced[1]]) + '\n').encode('ascii') + # Unbuffered raw write, less overhead and useful + # in case that we end up with concurrent writes. + os.write(output.fileno(), + ('%.0f\n' % reduced[0]).encode('ascii') + + data + + b'\n') + self.last_proc = now + + if isinstance(event, bb.event.MonitorDiskEvent) and \ + ((now - self.last_disk_monitor > self.min_seconds) or force): + os.write(self.monitor_disk.fileno(), + ('%.0f\n' % now).encode('ascii') + + ''.join(['%s: %d\n' % (dev, sample.total_bytes - sample.free_bytes) + for dev, sample in event.disk_usage.items()]).encode('ascii') + + b'\n') + self.last_disk_monitor = now diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py index 5395c768a3..3a5b7b6b44 100644 --- a/meta/lib/oe/buildhistory_analysis.py +++ b/meta/lib/oe/buildhistory_analysis.py @@ -1,6 +1,6 @@ # Report significant differences in the buildhistory repository since a specific revision # -# Copyright (C) 2012 Intel Corporation +# Copyright (C) 2012-2013, 2016-2017 Intel Corporation # Author: Paul Eggleton <paul.eggleton@linux.intel.com> # # Note: requires GitPython 0.3.1+ @@ -13,7 +13,10 @@ import os.path import difflib import git import re +import hashlib +import collections import bb.utils +import bb.tinfoil # How to display fields @@ -62,14 +65,29 @@ class ChangeRecord: def pkglist_combine(depver): pkglist = [] - for k,v in depver.iteritems(): + for k,v in depver.items(): if v: pkglist.append("%s (%s)" % (k,v)) else: pkglist.append(k) return pkglist + def detect_renamed_dirs(aitems, bitems): + adirs = set(map(os.path.dirname, aitems)) + bdirs = set(map(os.path.dirname, bitems)) + files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \ + for name in adirs - bdirs] + files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \ + for name in bdirs - adirs] + renamed_dirs = [(dir1, dir2) for dir1, files1 in files_ab for dir2, files2 in files_ba if files1 == files2] + # remove files that belong to renamed dirs from aitems and bitems + for dir1, dir2 in renamed_dirs: + aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)] + bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)] + return renamed_dirs, aitems, bitems + if self.fieldname in list_fields or self.fieldname in list_order_fields: + renamed_dirs = [] if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue) aitems = pkglist_combine(depvera) @@ -77,16 +95,29 @@ class ChangeRecord: else: aitems = self.oldvalue.split() bitems = self.newvalue.split() + if self.fieldname == 'FILELIST': + renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + removed = list(set(aitems) - set(bitems)) added = list(set(bitems) - set(aitems)) + lines = [] + if renamed_dirs: + for dfrom, dto in renamed_dirs: + lines.append('directory renamed %s -> %s' % (dfrom, dto)) if removed or added: if removed and not bitems: - out = '%s: removed all items "%s"' % (self.fieldname, ' '.join(removed)) + lines.append('removed all items "%s"' % ' '.join(removed)) else: - out = '%s:%s%s' % (self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '') + if removed: + lines.append('removed "%s"' % ' '.join(removed)) + if added: + lines.append('added "%s"' % ' '.join(added)) else: - out = '%s changed order' % self.fieldname + lines.append('changed order') + + out = '%s: %s' % (self.fieldname, ', '.join(lines)) + elif self.fieldname in numeric_fields: aval = int(self.oldvalue or 0) bval = int(self.newvalue or 0) @@ -190,7 +221,7 @@ class FileChange: def blob_to_dict(blob): - alines = blob.data_stream.read().splitlines() + alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] adict = {} for line in alines: splitv = [i.strip() for i in line.split('=',1)] @@ -220,7 +251,7 @@ def compare_file_lists(alines, blines): adict = file_list_to_dict(alines) bdict = file_list_to_dict(blines) filechanges = [] - for path, splitv in adict.iteritems(): + for path, splitv in adict.items(): newsplitv = bdict.pop(path, None) if newsplitv: # Check type @@ -359,18 +390,138 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): if ' '.join(alist) == ' '.join(blist): continue + if key == 'PKGR' and not report_all: + vers = [] + # strip leading 'r' and dots + for ver in (astr.split()[0], bstr.split()[0]): + if ver.startswith('r'): + ver = ver[1:] + vers.append(ver.replace('.', '')) + maxlen = max(len(vers[0]), len(vers[1])) + try: + # pad with '0' and convert to int + vers = [int(ver.ljust(maxlen, '0')) for ver in vers] + except ValueError: + pass + else: + # skip decrements and increments + if abs(vers[0] - vers[1]) == 1: + continue + chg = ChangeRecord(path, key, astr, bstr, monitored) changes.append(chg) return changes -def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False): +def compare_siglists(a_blob, b_blob, taskdiff=False): + # FIXME collapse down a recipe's tasks? + alines = a_blob.data_stream.read().decode('utf-8').splitlines() + blines = b_blob.data_stream.read().decode('utf-8').splitlines() + keys = [] + pnmap = {} + def readsigs(lines): + sigs = {} + for line in lines: + linesplit = line.split() + if len(linesplit) > 2: + sigs[linesplit[0]] = linesplit[2] + if not linesplit[0] in keys: + keys.append(linesplit[0]) + pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0] + return sigs + adict = readsigs(alines) + bdict = readsigs(blines) + out = [] + + changecount = 0 + addcount = 0 + removecount = 0 + if taskdiff: + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + + changes = collections.OrderedDict() + + def compare_hashfiles(pn, taskname, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data) + + if not taskname: + (pn, taskname) = pn.rsplit('.', 1) + pn = pnmap.get(pn, pn) + desc = '%s.%s' % (pn, taskname) + + if len(hashfiles) == 0: + out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2)) + elif not hash1 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1)) + elif not hash2 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2)) + else: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True) + for line in out2: + m = hashlib.sha256() + m.update(line.encode('utf-8')) + entry = changes.get(m.hexdigest(), (line, [])) + if desc not in entry[1]: + changes[m.hexdigest()] = (line, entry[1] + [desc]) + + # Define recursion callback + def recursecb(key, hash1, hash2): + compare_hashfiles(key, None, hash1, hash2) + return [] + + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + changecount += 1 + (pn, taskname) = key.rsplit('.', 1) + compare_hashfiles(pn, taskname, siga, sigb) + elif siga is None: + addcount += 1 + elif sigb is None: + removecount += 1 + for key, item in changes.items(): + line, tasks = item + if len(tasks) == 1: + desc = tasks[0] + elif len(tasks) == 2: + desc = '%s and %s' % (tasks[0], tasks[1]) + else: + desc = '%s and %d others' % (tasks[-1], len(tasks)-1) + out.append('%s: %s' % (desc, line)) + else: + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + out.append('%s changed from %s to %s' % (key, siga, sigb)) + changecount += 1 + elif siga is None: + out.append('%s was added' % key) + addcount += 1 + elif sigb is None: + out.append('%s was removed' % key) + removecount += 1 + out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100))) + return '\n'.join(out) + + +def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False): repo = git.Repo(repopath) assert repo.bare == False commit = repo.commit(revision1) diff = commit.diff(revision2) changes = [] + + if sigs or sigsdiff: + for d in diff.iter_change_type('M'): + if d.a_blob.path == 'siglist.txt': + changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff)) + return changes + for d in diff.iter_change_type('M'): path = os.path.dirname(d.a_blob.path) if path.startswith('packages/'): @@ -378,34 +529,34 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif path.startswith('images/'): filename = os.path.basename(d.a_blob.path) if filename in img_monitor_files: if filename == 'files-in-image.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_file_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) elif filename == 'installed-package-names.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) else: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif filename == 'image-info.txt': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif '/image-files/' in path: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) # Look for added preinst/postinst/prerm/postrm @@ -419,7 +570,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': addedpkgs.append(path) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True) addedchanges.append(chg) for chg in addedchanges: found = False @@ -436,7 +587,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if path.startswith('packages/'): filename = os.path.basename(d.a_blob.path) if filename != 'latest' and filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read(), '', True) + chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) # Link related changes diff --git a/meta/lib/oe/classextend.py b/meta/lib/oe/classextend.py index 68efca32d0..d2eeaf0e5c 100644 --- a/meta/lib/oe/classextend.py +++ b/meta/lib/oe/classextend.py @@ -1,3 +1,5 @@ +import collections + class ClassExtender(object): def __init__(self, extname, d): self.extname = extname @@ -23,7 +25,7 @@ class ClassExtender(object): return name def map_variable(self, varname, setvar = True): - var = self.d.getVar(varname, True) + var = self.d.getVar(varname) if not var: return "" var = var.split() @@ -36,7 +38,7 @@ class ClassExtender(object): return newdata def map_regexp_variable(self, varname, setvar = True): - var = self.d.getVar(varname, True) + var = self.d.getVar(varname) if not var: return "" var = var.split() @@ -57,6 +59,13 @@ class ClassExtender(object): if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('cross-canadian' in dep) or ('-crosssdk-' in dep): return dep else: + # Do not extend for that already have multilib prefix + var = self.d.getVar("MULTILIB_VARIANTS") + if var: + var = var.split() + for v in var: + if dep.startswith(v): + return dep return self.extend_name(dep) def map_depends_variable(self, varname, suffix = ""): @@ -65,12 +74,12 @@ class ClassExtender(object): varname = varname + "_" + suffix orig = self.d.getVar("EXTENDPKGV", False) self.d.setVar("EXTENDPKGV", "EXTENDPKGV") - deps = self.d.getVar(varname, True) + deps = self.d.getVar(varname) if not deps: self.d.setVar("EXTENDPKGV", orig) return deps = bb.utils.explode_dep_versions2(deps) - newdeps = {} + newdeps = collections.OrderedDict() for dep in deps: newdeps[self.map_depends(dep)] = deps[dep] @@ -78,7 +87,7 @@ class ClassExtender(object): self.d.setVar("EXTENDPKGV", orig) def map_packagevars(self): - for pkg in (self.d.getVar("PACKAGES", True).split() + [""]): + for pkg in (self.d.getVar("PACKAGES").split() + [""]): self.map_depends_variable("RDEPENDS", pkg) self.map_depends_variable("RRECOMMENDS", pkg) self.map_depends_variable("RSUGGESTS", pkg) @@ -88,7 +97,7 @@ class ClassExtender(object): self.map_depends_variable("PKG", pkg) def rename_packages(self): - for pkg in (self.d.getVar("PACKAGES", True) or "").split(): + for pkg in (self.d.getVar("PACKAGES") or "").split(): if pkg.startswith(self.extname): self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg]) continue @@ -103,6 +112,8 @@ class ClassExtender(object): class NativesdkClassExtender(ClassExtender): def map_depends(self, dep): + if dep.startswith(self.extname): + return dep if dep.endswith(("-gcc-initial", "-gcc", "-g++")): return dep + "-crosssdk" elif dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep): diff --git a/meta/lib/oe/classutils.py b/meta/lib/oe/classutils.py index 58188fdd6e..45cd5249be 100644 --- a/meta/lib/oe/classutils.py +++ b/meta/lib/oe/classutils.py @@ -1,4 +1,11 @@ -class ClassRegistry(type): + +class ClassRegistryMeta(type): + """Give each ClassRegistry their own registry""" + def __init__(cls, name, bases, attrs): + cls.registry = {} + type.__init__(cls, name, bases, attrs) + +class ClassRegistry(type, metaclass=ClassRegistryMeta): """Maintain a registry of classes, indexed by name. Note that this implementation requires that the names be unique, as it uses @@ -12,12 +19,6 @@ Subclasses of ClassRegistry may define an 'implemented' property to exert control over whether the class will be added to the registry (e.g. to keep abstract base classes out of the registry).""" priority = 0 - class __metaclass__(type): - """Give each ClassRegistry their own registry""" - def __init__(cls, name, bases, attrs): - cls.registry = {} - type.__init__(cls, name, bases, attrs) - def __init__(cls, name, bases, attrs): super(ClassRegistry, cls).__init__(name, bases, attrs) try: @@ -34,8 +35,8 @@ abstract base classes out of the registry).""" @classmethod def prioritized(tcls): - return sorted(tcls.registry.values(), - key=lambda v: v.priority, reverse=True) + return sorted(list(tcls.registry.values()), + key=lambda v: (v.priority, v.name), reverse=True) def unregister(cls): for key in cls.registry.keys(): diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py new file mode 100644 index 0000000000..a372904183 --- /dev/null +++ b/meta/lib/oe/copy_buildsystem.py @@ -0,0 +1,244 @@ +# This class should provide easy access to the different aspects of the +# buildsystem such as layers, bitbake location, etc. +import stat +import shutil + +def _smart_copy(src, dest): + import subprocess + # smart_copy will choose the correct function depending on whether the + # source is a file or a directory. + mode = os.stat(src).st_mode + if stat.S_ISDIR(mode): + bb.utils.mkdirhier(dest) + cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + else: + shutil.copyfile(src, dest) + shutil.copymode(src, dest) + +class BuildSystem(object): + def __init__(self, context, d): + self.d = d + self.context = context + self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] + self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() + + def copy_bitbake_and_layers(self, destdir, workspace_name=None): + # Copy in all metadata layers + bitbake (as repositories) + layers_copied = [] + bb.utils.mkdirhier(destdir) + layers = list(self.layerdirs) + + corebase = os.path.abspath(self.d.getVar('COREBASE')) + layers.append(corebase) + + # Exclude layers + for layer_exclude in self.layers_exclude: + if layer_exclude in layers: + layers.remove(layer_exclude) + + workspace_newname = workspace_name + if workspace_newname: + layernames = [os.path.basename(layer) for layer in layers] + extranum = 0 + while workspace_newname in layernames: + extranum += 1 + workspace_newname = '%s-%d' % (workspace_name, extranum) + + corebase_files = self.d.getVar('COREBASE_FILES').split() + corebase_files = [corebase + '/' +x for x in corebase_files] + # Make sure bitbake goes in + bitbake_dir = bb.__file__.rsplit('/', 3)[0] + corebase_files.append(bitbake_dir) + + for layer in layers: + layerconf = os.path.join(layer, 'conf', 'layer.conf') + layernewname = os.path.basename(layer) + workspace = False + if os.path.exists(layerconf): + with open(layerconf, 'r') as f: + if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): + if workspace_newname: + layernewname = workspace_newname + workspace = True + else: + bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) + continue + + # If the layer was already under corebase, leave it there + # since layers such as meta have issues when moved. + layerdestpath = destdir + if corebase == os.path.dirname(layer): + layerdestpath += '/' + os.path.basename(corebase) + layerdestpath += '/' + layernewname + + layer_relative = os.path.relpath(layerdestpath, + destdir) + layers_copied.append(layer_relative) + + # Treat corebase as special since it typically will contain + # build directories or other custom items. + if corebase == layer: + bb.utils.mkdirhier(layerdestpath) + for f in corebase_files: + f_basename = os.path.basename(f) + destname = os.path.join(layerdestpath, f_basename) + _smart_copy(f, destname) + else: + if os.path.exists(layerdestpath): + bb.note("Skipping layer %s, already handled" % layer) + else: + _smart_copy(layer, layerdestpath) + + if workspace: + # Make some adjustments original workspace layer + # Drop sources (recipe tasks will be locked, so we don't need them) + srcdir = os.path.join(layerdestpath, 'sources') + if os.path.isdir(srcdir): + shutil.rmtree(srcdir) + # Drop all bbappends except the one for the image the SDK is being built for + # (because of externalsrc, the workspace bbappends will interfere with the + # locked signatures if present, and we don't need them anyway) + image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend' + appenddir = os.path.join(layerdestpath, 'appends') + if os.path.isdir(appenddir): + for fn in os.listdir(appenddir): + if fn == image_bbappend: + continue + else: + os.remove(os.path.join(appenddir, fn)) + # Drop README + readme = os.path.join(layerdestpath, 'README') + if os.path.exists(readme): + os.remove(readme) + # Filter out comments in layer.conf and change layer name + layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') + with open(layerconf, 'r') as f: + origlines = f.readlines() + with open(layerconf, 'w') as f: + for line in origlines: + if line.startswith('#'): + continue + line = line.replace('workspacelayer', workspace_newname) + f.write(line) + + return layers_copied + +def generate_locked_sigs(sigfile, d): + bb.utils.mkdirhier(os.path.dirname(sigfile)) + depd = d.getVar('BB_TASKDEPDATA', False) + tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()] + bb.parse.siggen.dump_lockedsigs(sigfile, tasks) + +def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output): + with open(lockedsigs, 'r') as infile: + bb.utils.mkdirhier(os.path.dirname(pruned_output)) + with open(pruned_output, 'w') as f: + invalue = False + for line in infile: + if invalue: + if line.endswith('\\\n'): + splitval = line.strip().split(':') + if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets: + f.write(line) + else: + f.write(line) + invalue = False + elif line.startswith('SIGGEN_LOCKEDSIGS'): + invalue = True + f.write(line) + +def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): + merged = {} + arch_order = [] + with open(lockedsigs_main, 'r') as f: + invalue = None + for line in f: + if invalue: + if line.endswith('\\\n'): + merged[invalue].append(line) + else: + invalue = None + elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): + invalue = line[18:].split('=', 1)[0].rstrip() + merged[invalue] = [] + arch_order.append(invalue) + + with open(lockedsigs_extra, 'r') as f: + invalue = None + tocopy = {} + for line in f: + if invalue: + if line.endswith('\\\n'): + if not line in merged[invalue]: + target, task = line.strip().split(':')[:2] + if not copy_tasks or task in copy_tasks: + tocopy[invalue].append(line) + merged[invalue].append(line) + else: + invalue = None + elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): + invalue = line[18:].split('=', 1)[0].rstrip() + if not invalue in merged: + merged[invalue] = [] + arch_order.append(invalue) + tocopy[invalue] = [] + + def write_sigs_file(fn, types, sigs): + fulltypes = [] + bb.utils.mkdirhier(os.path.dirname(fn)) + with open(fn, 'w') as f: + for typename in types: + lines = sigs[typename] + if lines: + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename) + for line in lines: + f.write(line) + f.write(' "\n') + fulltypes.append(typename) + f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) + + if copy_output: + write_sigs_file(copy_output, list(tocopy.keys()), tocopy) + if merged_output: + write_sigs_file(merged_output, arch_order, merged) + +def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): + import shutil + bb.note('Generating sstate-cache...') + + nativelsbstring = d.getVar('NATIVELSBSTRING') + bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) + if fixedlsbstring and nativelsbstring != fixedlsbstring: + nativedir = output_sstate_cache + '/' + nativelsbstring + if os.path.isdir(nativedir): + destdir = os.path.join(output_sstate_cache, fixedlsbstring) + for root, _, files in os.walk(nativedir): + for fn in files: + src = os.path.join(root, fn) + dest = os.path.join(destdir, os.path.relpath(src, nativedir)) + if os.path.exists(dest): + # Already exists, and it'll be the same file, so just delete it + os.unlink(src) + else: + bb.utils.mkdirhier(os.path.dirname(dest)) + shutil.move(src, dest) + +def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): + import subprocess + + bb.note('Generating sstate task list...') + + if not cwd: + cwd = os.getcwd() + if logfile: + logparam = '-l %s' % logfile + else: + logparam = '' + cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + env = dict(d.getVar('BB_ORIGENV', False)) + env.pop('BUILDDIR', '') + pathitems = env['PATH'].split(':') + env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) + bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py index 4cc0e02968..b8901e63f5 100644 --- a/meta/lib/oe/data.py +++ b/meta/lib/oe/data.py @@ -1,3 +1,4 @@ +import json import oe.maketype def typed_value(key, d): @@ -7,11 +8,40 @@ def typed_value(key, d): flags = d.getVarFlags(key) if flags is not None: flags = dict((flag, d.expand(value)) - for flag, value in flags.iteritems()) + for flag, value in list(flags.items())) else: flags = {} try: - return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags) - except (TypeError, ValueError), exc: + return oe.maketype.create(d.getVar(key) or '', var_type, **flags) + except (TypeError, ValueError) as exc: bb.msg.fatal("Data", "%s: %s" % (key, str(exc))) + +def export2json(d, json_file, expand=True, searchString="",replaceString=""): + data2export = {} + keys2export = [] + + for key in d.keys(): + if key.startswith("_"): + continue + elif key.startswith("BB"): + continue + elif key.startswith("B_pn"): + continue + elif key.startswith("do_"): + continue + elif d.getVarFlag(key, "func"): + continue + + keys2export.append(key) + + for key in keys2export: + try: + data2export[key] = d.getVar(key, expand).replace(searchString,replaceString) + except bb.data_smart.ExpansionError: + data2export[key] = '' + except AttributeError: + pass + + with open(json_file, "w") as f: + json.dump(data2export, f, skipkeys=True, indent=4, sort_keys=True) diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py index 8ed5b0ec80..37f04ed359 100644 --- a/meta/lib/oe/distro_check.py +++ b/meta/lib/oe/distro_check.py @@ -1,43 +1,28 @@ -def get_links_from_url(url): +def create_socket(url, d): + import urllib + from bb.utils import export_proxies + + export_proxies(d) + return urllib.request.urlopen(url) + +def get_links_from_url(url, d): "Return all the href links found on the web location" - import urllib, sgmllib - - class LinksParser(sgmllib.SGMLParser): - def parse(self, s): - "Parse the given string 's'." - self.feed(s) - self.close() - - def __init__(self, verbose=0): - "Initialise an object passing 'verbose' to the superclass." - sgmllib.SGMLParser.__init__(self, verbose) - self.hyperlinks = [] - - def start_a(self, attributes): - "Process a hyperlink and its 'attributes'." - for name, value in attributes: - if name == "href": - self.hyperlinks.append(value.strip('/')) - - def get_hyperlinks(self): - "Return the list of hyperlinks." - return self.hyperlinks - - sock = urllib.urlopen(url) - webpage = sock.read() - sock.close() - - linksparser = LinksParser() - linksparser.parse(webpage) - return linksparser.get_hyperlinks() - -def find_latest_numeric_release(url): + from bs4 import BeautifulSoup, SoupStrainer + + soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) + hyperlinks = [] + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks + +def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" max=0 maxstr="" - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): try: + # TODO use LooseVersion release = float(link) except: release = 0 @@ -48,194 +33,158 @@ def find_latest_numeric_release(url): def is_src_rpm(name): "Check if the link is pointing to a src.rpm file" - if name[-8:] == ".src.rpm": - return True - else: - return False + return name.endswith(".src.rpm") def package_name_from_srpm(srpm): "Strip out the package name from the src.rpm filename" - strings = srpm.split('-') - package_name = strings[0] - for i in range(1, len (strings) - 1): - str = strings[i] - if not str[0].isdigit(): - package_name += '-' + str - return package_name -def clean_package_list(package_list): - "Removes multiple entries of packages and sorts the list" - set = {} - map(set.__setitem__, package_list, []) - return set.keys() - - -def get_latest_released_meego_source_package_list(): - "Returns list of all the name os packages in the latest meego distro" - - package_names = [] - try: - f = open("/tmp/Meego-1.1", "r") - for line in f: - package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end - except IOError: pass - package_list=clean_package_list(package_names) - return "1.0", package_list + # ca-certificates-2016.2.7-1.0.fc24.src.rpm + # ^name ^ver ^release^removed + (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) + return name -def get_source_package_list_from_url(url, section): +def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) - links = get_links_from_url(url) + links = get_links_from_url(url, d) srpms = filter(is_src_rpm, links) names_list = map(package_name_from_srpm, srpms) - new_pkgs = [] + new_pkgs = set() for pkgs in names_list: - new_pkgs.append(pkgs + ":" + section) - + new_pkgs.add(pkgs + ":" + section) return new_pkgs -def get_latest_released_fedora_source_package_list(): - "Returns list of all the name os packages in the latest fedora distro" - latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/") - - package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main") - -# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") - package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates") +def get_source_package_list_from_url_by_letter(url, section, d): + import string + from urllib.error import HTTPError + packages = set() + for letter in (string.ascii_lowercase + string.digits): + # Not all subfolders may exist, so silently handle 404 + try: + packages |= get_source_package_list_from_url(url + "/" + letter, section, d) + except HTTPError as e: + if e.code != 404: raise + return packages - package_list=clean_package_list(package_names) - - return latest, package_list +def get_latest_released_fedora_source_package_list(d): + "Returns list of all the name os packages in the latest fedora distro" + latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) + package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) + package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) + return latest, package_names -def get_latest_released_opensuse_source_package_list(): +def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main") - package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates") + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d) + return latest, package_names - package_list=clean_package_list(package_names) - return latest, package_list - -def get_latest_released_mandriva_source_package_list(): +def get_latest_released_mandriva_source_package_list(d): "Returns list of all the name os packages in the latest mandriva distro" - latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") - package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main") -# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") - package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates") + latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) + package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) + return latest, package_names - package_list=clean_package_list(package_names) - return latest, package_list +def get_latest_released_clear_source_package_list(d): + latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) + package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) + return latest, package_names -def find_latest_debian_release(url): +def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" - releases = [] - for link in get_links_from_url(url): - if link[:6] == "Debian": - if ';' not in link: - releases.append(link) + releases = [link.replace("Debian", "") + for link in get_links_from_url(url, d) + if link.startswith("Debian")] releases.sort() try: - return releases.pop()[6:] + return releases[-1] except: return "_NotFound_" -def get_debian_style_source_package_list(url, section): +def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - import urllib - sock = urllib.urlopen(url) - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - sock.close() - tmpfile.close() import gzip - bb.note("Reading %s: %s" % (url, section)) - - f = gzip.open(tmpfilename) - package_names = [] - for line in f: - if line[:9] == "Package: ": - package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end - os.unlink(tmpfilename) + package_names = set() + for line in gzip.open(create_socket(url, d), mode="rt"): + if line.startswith("Package:"): + pkg = line.split(":", 1)[1].strip() + package_names.add(pkg + ":" + section) return package_names -def get_latest_released_debian_source_package_list(): - "Returns list of all the name os packages in the latest debian distro" - latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/") - url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" - package_names = get_debian_style_source_package_list(url, "main") -# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" -# package_names += get_debian_style_source_package_list(url, "contrib") - url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" - package_names += get_debian_style_source_package_list(url, "updates") - package_list=clean_package_list(package_names) - return latest, package_list - -def find_latest_ubuntu_release(url): - "Find the latest listed ubuntu release on the given url" +def get_latest_released_debian_source_package_list(d): + "Returns list of all the name of packages in the latest debian distro" + latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) + url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" + package_names = get_debian_style_source_package_list(url, "main", d) + url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def find_latest_ubuntu_release(url, d): + """ + Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. + + To avoid matching development releases look for distributions that have + updates, so the resulting distro could be any supported release. + """ url += "?C=M;O=D" # Descending Sort by Last Modified - for link in get_links_from_url(url): - if link[-8:] == "-updates": - return link[:-8] + for link in get_links_from_url(url, d): + if "-updates" in link: + distro = link.replace("-updates", "") + return distro return "_NotFound_" -def get_latest_released_ubuntu_source_package_list(): +def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" - latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") + latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest - package_names = get_debian_style_source_package_list(url, "main") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "multiverse") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "universe") + package_names = get_debian_style_source_package_list(url, "main", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest - package_names += get_debian_style_source_package_list(url, "updates") - package_list=clean_package_list(package_names) - return latest, package_list + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def create_distro_packages_list(distro_check_dir, d): + import shutil -def create_distro_packages_list(distro_check_dir): pkglst_dir = os.path.join(distro_check_dir, "package_lists") - if not os.path.isdir (pkglst_dir): - os.makedirs(pkglst_dir) - # first clear old stuff - for file in os.listdir(pkglst_dir): - os.unlink(os.path.join(pkglst_dir, file)) - - per_distro_functions = [ - ["Debian", get_latest_released_debian_source_package_list], - ["Ubuntu", get_latest_released_ubuntu_source_package_list], - ["Fedora", get_latest_released_fedora_source_package_list], - ["OpenSuSE", get_latest_released_opensuse_source_package_list], - ["Mandriva", get_latest_released_mandriva_source_package_list], - ["Meego", get_latest_released_meego_source_package_list] - ] - - from datetime import datetime - begin = datetime.now() - for distro in per_distro_functions: - name = distro[0] - release, package_list = distro[1]() + bb.utils.remove(pkglst_dir, True) + bb.utils.mkdirhier(pkglst_dir) + + per_distro_functions = ( + ("Debian", get_latest_released_debian_source_package_list), + ("Ubuntu", get_latest_released_ubuntu_source_package_list), + ("Fedora", get_latest_released_fedora_source_package_list), + ("OpenSuSE", get_latest_released_opensuse_source_package_list), + ("Mandriva", get_latest_released_mandriva_source_package_list), + ("Clear", get_latest_released_clear_source_package_list), + ) + + for name, fetcher_func in per_distro_functions: + try: + release, package_list = fetcher_func(d) + except Exception as e: + bb.warn("Cannot fetch packages for %s: %s" % (name, e)) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) + if len(package_list) == 0: + bb.error("Didn't fetch any packages for %s %s" % (name, release)) + package_list_file = os.path.join(pkglst_dir, name + "-" + release) - f = open(package_list_file, "w+b") - for pkg in package_list: - f.write(pkg + "\n") - f.close() - end = datetime.now() - delta = end - begin - bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) + with open(package_list_file, 'w') as f: + for pkg in sorted(package_list): + f.write(pkg + "\n") -def update_distro_data(distro_check_dir, datetime): +def update_distro_data(distro_check_dir, datetime, d): """ - If distro packages list data is old then rebuild it. - The operations has to be protected by a lock so that - only one thread performes it at a time. + If distro packages list data is old then rebuild it. + The operations has to be protected by a lock so that + only one thread performes it at a time. """ if not os.path.isdir (distro_check_dir): try: @@ -250,83 +199,71 @@ def update_distro_data(distro_check_dir, datetime): import fcntl try: if not os.path.exists(datetime_file): - open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail - f = open(datetime_file, "r+b") + f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") - create_distro_packages_list(distro_check_dir) + create_distro_packages_list(distro_check_dir, d) f.seek(0) f.write(datetime) - except OSError: - raise Exception('Unable to read/write this file: %s' % (datetime_file)) + except OSError as e: + raise Exception('Unable to open timestamp: %s' % e) finally: fcntl.lockf(f, fcntl.LOCK_UN) f.close() - + def compare_in_distro_packages_list(distro_check_dir, d): if not os.path.isdir(distro_check_dir): raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") - + localdata = bb.data.createCopy(d) pkglst_dir = os.path.join(distro_check_dir, "package_lists") matching_distros = [] - pn = d.getVar('PN', True) - recipe_name = d.getVar('PN', True) + pn = recipe_name = d.getVar('PN') bb.note("Checking: %s" % pn) - trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"}) - if pn.find("-native") != -1: pnstripped = pn.split("-native") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.startswith("nativesdk-"): pnstripped = pn.split("nativesdk-") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[1] if pn.find("-cross") != -1: pnstripped = pn.split("-cross") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.find("-initial") != -1: pnstripped = pn.split("-initial") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] bb.note("Recipe: %s" % recipe_name) - tmp = localdata.getVar('DISTRO_PN_ALIAS', True) distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) - - if tmp: - list = tmp.split(' ') - for str in list: - if str and str.find("=") == -1 and distro_exceptions[str]: - matching_distros.append(str) + tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" + for str in tmp.split(): + if str and str.find("=") == -1 and distro_exceptions[str]: + matching_distros.append(str) distro_pn_aliases = {} - if tmp: - list = tmp.split(' ') - for str in list: - if str.find("=") != -1: - (dist, pn_alias) = str.split('=') - distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() - + for str in tmp.split(): + if "=" in str: + (dist, pn_alias) = str.split('=') + distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() + for file in os.listdir(pkglst_dir): (distro, distro_release) = file.split("-") - f = open(os.path.join(pkglst_dir, file), "rb") + f = open(os.path.join(pkglst_dir, file), "r") for line in f: (pkg, section) = line.split(":") if distro.lower() in distro_pn_aliases: @@ -339,38 +276,34 @@ def compare_in_distro_packages_list(distro_check_dir, d): break f.close() - - if tmp != None: - list = tmp.split(' ') - for item in list: - matching_distros.append(item) + for item in tmp.split(): + matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros def create_log_file(d, logname): - import subprocess - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) logfn, logsuffix = os.path.splitext(logname) - logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix)) + logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) if not os.path.exists(logfile): slogfile = os.path.join(logpath, logname) if os.path.exists(slogfile): os.remove(slogfile) - subprocess.call("touch %s" % logfile, shell=True) + open(logfile, 'w+').close() os.symlink(logfile, slogfile) d.setVar('LOG_FILE', logfile) return logfile def save_distro_check_result(result, datetime, result_file, d): - pn = d.getVar('PN', True) - logdir = d.getVar('LOG_DIR', True) + pn = d.getVar('PN') + logdir = d.getVar('LOG_DIR') if not logdir: bb.error("LOG_DIR variable is not defined, can't write the distro_check results") return - if not os.path.isdir(logdir): - os.makedirs(logdir) + bb.utils.mkdirhier(logdir) + line = pn for i in result: line = line + "," + i diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py new file mode 100644 index 0000000000..7ce767ee0a --- /dev/null +++ b/meta/lib/oe/gpg_sign.py @@ -0,0 +1,119 @@ +"""Helper module for GPG signing""" +import os + +import bb +import oe.utils + +class LocalSigner(object): + """Class for handling local (on the build host) signing""" + def __init__(self, d): + self.gpg_bin = d.getVar('GPG_BIN') or \ + bb.utils.which(os.getenv('PATH'), 'gpg') + self.gpg_path = d.getVar('GPG_PATH') + self.gpg_version = self.get_gpg_version() + self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign") + + def export_pubkey(self, output_file, keyid, armor=True): + """Export GPG public key to a file""" + cmd = '%s --batch --yes --export -o %s ' % \ + (self.gpg_bin, output_file) + if self.gpg_path: + cmd += "--homedir %s " % self.gpg_path + if armor: + cmd += "--armor " + cmd += keyid + status, output = oe.utils.getstatusoutput(cmd) + if status: + raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % + (keyid, output)) + + def sign_rpms(self, files, keyid, passphrase): + """Sign RPM files""" + + cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid + gpg_args = '--batch --passphrase=%s' % passphrase + if self.gpg_version > (2,1,): + gpg_args += ' --pinentry-mode=loopback' + cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args + if self.gpg_bin: + cmd += "--define '%%__gpg %s' " % self.gpg_bin + if self.gpg_path: + cmd += "--define '_gpg_path %s' " % self.gpg_path + + # Sign in chunks of 100 packages + for i in range(0, len(files), 100): + status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100])) + if status: + raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) + + def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): + """Create a detached signature of a file""" + import subprocess + + if passphrase_file and passphrase: + raise Exception("You should use either passphrase_file of passphrase, not both") + + cmd = [self.gpg_bin, '--detach-sign', '--batch', '--no-tty', '--yes', + '--passphrase-fd', '0', '-u', keyid] + + if self.gpg_path: + cmd += ['--homedir', self.gpg_path] + if armor: + cmd += ['--armor'] + + #gpg > 2.1 supports password pipes only through the loopback interface + #gpg < 2.1 errors out if given unknown parameters + if self.gpg_version > (2,1,): + cmd += ['--pinentry-mode', 'loopback'] + + cmd += [input_file] + + try: + if passphrase_file: + with open(passphrase_file) as fobj: + passphrase = fobj.readline(); + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) + + if job.returncode: + raise bb.build.FuncFailed("GPG exited with code %d: %s" % + (job.returncode, stderr.decode("utf-8"))) + + except IOError as e: + bb.error("IO error (%s): %s" % (e.errno, e.strerror)) + raise Exception("Failed to sign '%s'" % input_file) + + except OSError as e: + bb.error("OS error (%s): %s" % (e.errno, e.strerror)) + raise Exception("Failed to sign '%s" % input_file) + + + def get_gpg_version(self): + """Return the gpg version as a tuple of ints""" + import subprocess + try: + ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + return tuple([int(i) for i in ver_str.split('.')]) + except subprocess.CalledProcessError as e: + raise bb.build.FuncFailed("Could not get gpg version: %s" % e) + + + def verify(self, sig_file): + """Verify signature""" + cmd = self.gpg_bin + " --verify " + if self.gpg_path: + cmd += "--homedir %s " % self.gpg_path + cmd += sig_file + status, _ = oe.utils.getstatusoutput(cmd) + ret = False if status else True + return ret + + +def get_signer(d, backend): + """Get signer object for the specified backend""" + # Use local signing by default + if backend == 'local': + return LocalSigner(d) + else: + bb.fatal("Unsupported signing backend '%s'" % backend) diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py deleted file mode 100644 index 354a676f42..0000000000 --- a/meta/lib/oe/image.py +++ /dev/null @@ -1,345 +0,0 @@ -from oe.utils import execute_pre_post_process -import os -import subprocess -import multiprocessing - - -def generate_image(arg): - (type, subimages, create_img_cmd) = arg - - bb.note("Running image creation script for %s: %s ..." % - (type, create_img_cmd)) - - try: - subprocess.check_output(create_img_cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - return("Error: The image creation script '%s' returned %d:\n%s" % - (e.cmd, e.returncode, e.output)) - - return None - - -""" -This class will help compute IMAGE_FSTYPE dependencies and group them in batches -that can be executed in parallel. - -The next example is for illustration purposes, highly unlikely to happen in real life. -It's just one of the test cases I used to test the algorithm: - -For: -IMAGE_FSTYPES = "i1 i2 i3 i4 i5" -IMAGE_TYPEDEP_i4 = "i2" -IMAGE_TYPEDEP_i5 = "i6 i4" -IMAGE_TYPEDEP_i6 = "i7" -IMAGE_TYPEDEP_i7 = "i2" - -We get the following list of batches that can be executed in parallel, having the -dependencies satisfied: - -[['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']] -""" -class ImageDepGraph(object): - def __init__(self, d): - self.d = d - self.graph = dict() - self.deps_array = dict() - - def _construct_dep_graph(self, image_fstypes): - graph = dict() - - def add_node(node): - deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "") - if deps != "": - graph[node] = deps - - for dep in deps.split(): - if not dep in graph: - add_node(dep) - else: - graph[node] = "" - - for fstype in image_fstypes: - add_node(fstype) - - return graph - - def _clean_graph(self): - # Live and VMDK images will be processed via inheriting - # bbclass and does not get processed here. Remove them from the fstypes - # graph. Their dependencies are already added, so no worries here. - remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split() - - for item in remove_list: - self.graph.pop(item, None) - - def _compute_dependencies(self): - """ - returns dict object of nodes with [no_of_depends_on, no_of_depended_by] - for each node - """ - deps_array = dict() - for node in self.graph: - deps_array[node] = [0, 0] - - for node in self.graph: - deps = self.graph[node].split() - deps_array[node][0] += len(deps) - for dep in deps: - deps_array[dep][1] += 1 - - return deps_array - - def _sort_graph(self): - sorted_list = [] - group = [] - for node in self.graph: - if node not in self.deps_array: - continue - - depends_on = self.deps_array[node][0] - - if depends_on == 0: - group.append(node) - - if len(group) == 0 and len(self.deps_array) != 0: - bb.fatal("possible fstype circular dependency...") - - sorted_list.append(group) - - # remove added nodes from deps_array - for item in group: - for node in self.graph: - if item in self.graph[node]: - self.deps_array[node][0] -= 1 - - self.deps_array.pop(item, None) - - if len(self.deps_array): - # recursive call, to find the next group - sorted_list += self._sort_graph() - - return sorted_list - - def group_fstypes(self, image_fstypes): - self.graph = self._construct_dep_graph(image_fstypes) - - self._clean_graph() - - self.deps_array = self._compute_dependencies() - - alltypes = [node for node in self.graph] - - return (alltypes, self._sort_graph()) - - -class Image(ImageDepGraph): - def __init__(self, d): - self.d = d - - super(Image, self).__init__(d) - - def _get_rootfs_size(self): - """compute the rootfs size""" - rootfs_alignment = int(self.d.getVar('IMAGE_ROOTFS_ALIGNMENT', True)) - overhead_factor = float(self.d.getVar('IMAGE_OVERHEAD_FACTOR', True)) - rootfs_req_size = int(self.d.getVar('IMAGE_ROOTFS_SIZE', True)) - rootfs_extra_space = eval(self.d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True)) - rootfs_maxsize = self.d.getVar('IMAGE_ROOTFS_MAXSIZE', True) - - output = subprocess.check_output(['du', '-ks', - self.d.getVar('IMAGE_ROOTFS', True)]) - size_kb = int(output.split()[0]) - base_size = size_kb * overhead_factor - base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \ - rootfs_extra_space - - if base_size != int(base_size): - base_size = int(base_size + 1) - - base_size += rootfs_alignment - 1 - base_size -= base_size % rootfs_alignment - - # Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set) - if rootfs_maxsize: - rootfs_maxsize_int = int(rootfs_maxsize) - if base_size > rootfs_maxsize_int: - bb.fatal("The rootfs size %d(K) overrides the max size %d(K)" % \ - (base_size, rootfs_maxsize_int)) - - return base_size - - def _create_symlinks(self, subimages): - """create symlinks to the newly created image""" - deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) - img_name = self.d.getVar('IMAGE_NAME', True) - link_name = self.d.getVar('IMAGE_LINK_NAME', True) - manifest_name = self.d.getVar('IMAGE_MANIFEST', True) - - os.chdir(deploy_dir) - - if link_name is not None: - for type in subimages: - if os.path.exists(img_name + ".rootfs." + type): - dst = link_name + "." + type - src = img_name + ".rootfs." + type - bb.note("Creating symlink: %s -> %s" % (dst, src)) - os.symlink(src, dst) - - if manifest_name is not None and \ - os.path.exists(manifest_name) and \ - not os.path.exists(link_name + ".manifest"): - os.symlink(os.path.basename(manifest_name), - link_name + ".manifest") - - def _remove_old_symlinks(self): - """remove the symlinks to old binaries""" - - if self.d.getVar('IMAGE_LINK_NAME', True): - deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) - for img in os.listdir(deploy_dir): - if img.find(self.d.getVar('IMAGE_LINK_NAME', True)) == 0: - img = os.path.join(deploy_dir, img) - if os.path.islink(img): - if self.d.getVar('RM_OLD_IMAGE', True) == "1" and \ - os.path.exists(os.path.realpath(img)): - os.remove(os.path.realpath(img)) - - os.remove(img) - - """ - This function will just filter out the compressed image types from the - fstype groups returning a (filtered_fstype_groups, cimages) tuple. - """ - def _filter_out_commpressed(self, fstype_groups): - ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() - cimages = {} - - filtered_groups = [] - for group in fstype_groups: - filtered_group = [] - for type in group: - basetype = None - for ctype in ctypes: - if type.endswith("." + ctype): - basetype = type[:-len("." + ctype)] - if basetype not in filtered_group: - filtered_group.append(basetype) - if basetype not in cimages: - cimages[basetype] = [] - if ctype not in cimages[basetype]: - cimages[basetype].append(ctype) - break - if not basetype and type not in filtered_group: - filtered_group.append(type) - - filtered_groups.append(filtered_group) - - return (filtered_groups, cimages) - - def _get_image_types(self): - """returns a (types, cimages) tuple""" - - alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split()) - - filtered_groups, cimages = self._filter_out_commpressed(fstype_groups) - - return (alltypes, filtered_groups, cimages) - - def _write_script(self, type, cmds): - tempdir = self.d.getVar('T', True) - script_name = os.path.join(tempdir, "create_image." + type) - - self.d.setVar('img_creation_func', '\n'.join(cmds)) - self.d.setVarFlag('img_creation_func', 'func', 1) - self.d.setVarFlag('img_creation_func', 'fakeroot', 1) - - with open(script_name, "w+") as script: - script.write("%s" % bb.build.shell_trap_code()) - script.write("export ROOTFS_SIZE=%d\n" % self._get_rootfs_size()) - bb.data.emit_func('img_creation_func', script, self.d) - script.write("img_creation_func\n") - - os.chmod(script_name, 0775) - - return script_name - - def _get_imagecmds(self): - old_overrides = self.d.getVar('OVERRIDES', 0) - - alltypes, fstype_groups, cimages = self._get_image_types() - - image_cmd_groups = [] - - bb.note("The image creation groups are: %s" % str(fstype_groups)) - for fstype_group in fstype_groups: - image_cmds = [] - for type in fstype_group: - cmds = [] - subimages = [] - - localdata = bb.data.createCopy(self.d) - localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides)) - bb.data.update_data(localdata) - localdata.setVar('type', type) - - cmds.append("\t" + localdata.getVar("IMAGE_CMD", True)) - cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) - - if type in cimages: - for ctype in cimages[type]: - cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True)) - subimages.append(type + "." + ctype) - - if type not in alltypes: - cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) - else: - subimages.append(type) - - script_name = self._write_script(type, cmds) - - image_cmds.append((type, subimages, script_name)) - - image_cmd_groups.append(image_cmds) - - return image_cmd_groups - - def create(self): - bb.note("###### Generate images #######") - pre_process_cmds = self.d.getVar("IMAGE_PREPROCESS_COMMAND", True) - post_process_cmds = self.d.getVar("IMAGE_POSTPROCESS_COMMAND", True) - - execute_pre_post_process(self.d, pre_process_cmds) - - self._remove_old_symlinks() - - image_cmd_groups = self._get_imagecmds() - - for image_cmds in image_cmd_groups: - # create the images in parallel - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(generate_image, image_cmds)) - pool.close() - pool.join() - - for result in results: - if result is not None: - bb.fatal(result) - - for image_type, subimages, script in image_cmds: - bb.note("Creating symlinks for %s image ..." % image_type) - self._create_symlinks(subimages) - - execute_pre_post_process(self.d, post_process_cmds) - - -def create_image(d): - Image(d).create() - -if __name__ == "__main__": - """ - Image creation can be called independent from bitbake environment. - """ - """ - TBD - """ diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index 340da61102..8d2fd1709c 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -5,6 +5,20 @@ import ast import re from fnmatch import fnmatchcase as fnmatch +def license_ok(license, dont_want_licenses): + """ Return False if License exist in dont_want_licenses else True """ + for dwl in dont_want_licenses: + # If you want to exclude license named generically 'X', we + # surely want to exclude 'X+' as well. In consequence, we + # will exclude a trailing '+' character from LICENSE in + # case INCOMPATIBLE_LICENSE is not a 'X+' license. + lic = license + if not re.search('\+$', dwl): + lic = re.sub('\+', '', license) + if fnmatch(lic, dwl): + return False + return True + class LicenseError(Exception): pass @@ -25,14 +39,15 @@ class InvalidLicense(LicenseError): def __str__(self): return "invalid characters in license '%s'" % self.license -license_operator = re.compile('([&|() ])') +license_operator_chars = '&|() ' +license_operator = re.compile('([' + license_operator_chars + '])') license_pattern = re.compile('[a-zA-Z0-9.+_\-]+$') class LicenseVisitor(ast.NodeVisitor): - """Syntax tree visitor which can accept OpenEmbedded license strings""" - def visit_string(self, licensestr): + """Get elements based on OpenEmbedded license strings""" + def get_elements(self, licensestr): new_elements = [] - elements = filter(lambda x: x.strip(), license_operator.split(licensestr)) + elements = list([x for x in license_operator.split(licensestr) if x.strip()]) for pos, element in enumerate(elements): if license_pattern.match(element): if pos > 0 and license_pattern.match(elements[pos-1]): @@ -42,7 +57,16 @@ class LicenseVisitor(ast.NodeVisitor): raise InvalidLicense(element) new_elements.append(element) - self.visit(ast.parse(' '.join(new_elements))) + return new_elements + + """Syntax tree visitor which can accept elements previously generated with + OpenEmbedded license string""" + def visit_elements(self, elements): + self.visit(ast.parse(' '.join(elements))) + + """Syntax tree visitor which can accept OpenEmbedded license strings""" + def visit_string(self, licensestr): + self.visit_elements(self.get_elements(licensestr)) class FlattenVisitor(LicenseVisitor): """Flatten a license tree (parsed from a string) by selecting one of each @@ -94,8 +118,8 @@ def is_included(licensestr, whitelist=None, blacklist=None): def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most included licenses).""" - alpha_weight = len(filter(include_license, alpha)) - beta_weight = len(filter(include_license, beta)) + alpha_weight = len(list(filter(include_license, alpha))) + beta_weight = len(list(filter(include_license, beta))) if alpha_weight > beta_weight: return alpha else: @@ -108,9 +132,104 @@ def is_included(licensestr, whitelist=None, blacklist=None): blacklist = [] licenses = flattened_licenses(licensestr, choose_licenses) - excluded = filter(lambda lic: exclude_license(lic), licenses) - included = filter(lambda lic: include_license(lic), licenses) + excluded = [lic for lic in licenses if exclude_license(lic)] + included = [lic for lic in licenses if include_license(lic)] if excluded: return False, excluded else: return True, included + +class ManifestVisitor(LicenseVisitor): + """Walk license tree (parsed from a string) removing the incompatible + licenses specified""" + def __init__(self, dont_want_licenses, canonical_license, d): + self._dont_want_licenses = dont_want_licenses + self._canonical_license = canonical_license + self._d = d + self._operators = [] + + self.licenses = [] + self.licensestr = '' + + LicenseVisitor.__init__(self) + + def visit(self, node): + if isinstance(node, ast.Str): + lic = node.s + + if license_ok(self._canonical_license(self._d, lic), + self._dont_want_licenses) == True: + if self._operators: + ops = [] + for op in self._operators: + if op == '[': + ops.append(op) + elif op == ']': + ops.append(op) + else: + if not ops: + ops.append(op) + elif ops[-1] in ['[', ']']: + ops.append(op) + else: + ops[-1] = op + + for op in ops: + if op == '[' or op == ']': + self.licensestr += op + elif self.licenses: + self.licensestr += ' ' + op + ' ' + + self._operators = [] + + self.licensestr += lic + self.licenses.append(lic) + elif isinstance(node, ast.BitAnd): + self._operators.append("&") + elif isinstance(node, ast.BitOr): + self._operators.append("|") + elif isinstance(node, ast.List): + self._operators.append("[") + elif isinstance(node, ast.Load): + self.licensestr += "]" + + self.generic_visit(node) + +def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): + """Given a license string and dont_want_licenses list, + return license string filtered and a list of licenses""" + manifest = ManifestVisitor(dont_want_licenses, canonical_license, d) + + try: + elements = manifest.get_elements(licensestr) + + # Replace '()' to '[]' for handle in ast as List and Load types. + elements = ['[' if e == '(' else e for e in elements] + elements = [']' if e == ')' else e for e in elements] + + manifest.visit_elements(elements) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + + # Replace '[]' to '()' for output correct license. + manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') + + return (manifest.licensestr, manifest.licenses) + +class ListVisitor(LicenseVisitor): + """Record all different licenses found in the license string""" + def __init__(self): + self.licenses = set() + + def visit_Str(self, node): + self.licenses.add(node.s) + +def list_licenses(licensestr): + """Simply get a list of all licenses mentioned in a license string. + Binary operators are not applied or taken into account in any way""" + visitor = ListVisitor() + try: + visitor.visit_string(licensestr) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + return visitor.licenses diff --git a/meta/lib/oe/lsb.py b/meta/lib/oe/lsb.py index b53f361035..3a945e0fce 100644 --- a/meta/lib/oe/lsb.py +++ b/meta/lib/oe/lsb.py @@ -1,25 +1,54 @@ -def release_dict(): - """Return the output of lsb_release -ir as a dictionary""" +def release_dict_osr(): + """ Populate a dict with pertinent values from /etc/os-release """ + if not os.path.exists('/etc/os-release'): + return None + + data = {} + with open('/etc/os-release') as f: + for line in f: + try: + key, val = line.rstrip().split('=', 1) + except ValueError: + continue + if key == 'ID': + data['DISTRIB_ID'] = val.strip('"') + if key == 'VERSION_ID': + data['DISTRIB_RELEASE'] = val.strip('"') + + return data + +def release_dict_lsb(): + """ Return the output of lsb_release -ir as a dictionary """ from subprocess import PIPE try: output, err = bb.process.run(['lsb_release', '-ir'], stderr=PIPE) except bb.process.CmdError as exc: - return None + return {} + + lsb_map = { 'Distributor ID': 'DISTRIB_ID', + 'Release': 'DISTRIB_RELEASE'} + lsb_keys = lsb_map.keys() data = {} for line in output.splitlines(): + if line.startswith("-e"): + line = line[3:] try: key, value = line.split(":\t", 1) except ValueError: continue - else: - data[key] = value + if key in lsb_keys: + data[lsb_map[key]] = value + + if len(data.keys()) != 2: + return None + return data def release_dict_file(): - """ Try to gather LSB release information manually when lsb_release tool is unavailable """ - data = None + """ Try to gather release information manually when other methods fail """ + data = {} try: if os.path.exists('/etc/lsb-release'): data = {} @@ -44,38 +73,38 @@ def release_dict_file(): if line.startswith('VERSION = '): data['DISTRIB_RELEASE'] = line[10:].rstrip() break - elif os.path.exists('/etc/os-release'): - data = {} - with open('/etc/os-release') as f: - for line in f: - if line.startswith('NAME='): - data['DISTRIB_ID'] = line[5:].rstrip().strip('"') - if line.startswith('VERSION_ID='): - data['DISTRIB_RELEASE'] = line[11:].rstrip().strip('"') + except IOError: - return None + return {} return data def distro_identifier(adjust_hook=None): """Return a distro identifier string based upon lsb_release -ri, with optional adjustment via a hook""" - lsb_data = release_dict() - if lsb_data: - distro_id, release = lsb_data['Distributor ID'], lsb_data['Release'] - else: - lsb_data_file = release_dict_file() - if lsb_data_file: - distro_id, release = lsb_data_file['DISTRIB_ID'], lsb_data_file.get('DISTRIB_RELEASE', None) - else: - distro_id, release = None, None + import re + + # Try /etc/os-release first, then the output of `lsb_release -ir` and + # finally fall back on parsing various release files in order to determine + # host distro name and version. + distro_data = release_dict_osr() + if not distro_data: + distro_data = release_dict_lsb() + if not distro_data: + distro_data = release_dict_file() + + distro_id = distro_data.get('DISTRIB_ID', '') + release = distro_data.get('DISTRIB_RELEASE', '') if adjust_hook: distro_id, release = adjust_hook(distro_id, release) if not distro_id: return "Unknown" + # Filter out any non-alphanumerics + distro_id = re.sub(r'\W', '', distro_id) + if release: - id_str = '{0}-{1}'.format(distro_id, release) + id_str = '{0}-{1}'.format(distro_id.lower(), release) else: id_str = distro_id return id_str.replace(' ','-').replace('/','-') diff --git a/meta/lib/oe/maketype.py b/meta/lib/oe/maketype.py index 139f333691..f88981dd90 100644 --- a/meta/lib/oe/maketype.py +++ b/meta/lib/oe/maketype.py @@ -6,7 +6,8 @@ the arguments of the type's factory for details. """ import inspect -import types +import oe.types as types +import collections available_types = {} @@ -53,7 +54,9 @@ def get_callable_args(obj): if type(obj) is type: obj = obj.__init__ - args, varargs, keywords, defaults = inspect.getargspec(obj) + sig = inspect.signature(obj) + args = list(sig.parameters.keys()) + defaults = list(s for s in sig.parameters.keys() if sig.parameters[s].default != inspect.Parameter.empty) flaglist = [] if args: if len(args) > 1 and args[0] == 'self': @@ -93,7 +96,7 @@ for name in dir(types): continue obj = getattr(types, name) - if not callable(obj): + if not isinstance(obj, collections.Callable): continue register(name, obj) diff --git a/meta/lib/oe/manifest.py b/meta/lib/oe/manifest.py index 42832f15d2..60c49be0e9 100644 --- a/meta/lib/oe/manifest.py +++ b/meta/lib/oe/manifest.py @@ -4,11 +4,10 @@ import re import bb -class Manifest(object): +class Manifest(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta PKG_TYPE_MUST_INSTALL = "mip" PKG_TYPE_MULTILIB = "mlp" @@ -60,9 +59,9 @@ class Manifest(object): if manifest_dir is None: if manifest_type != self.MANIFEST_TYPE_IMAGE: - self.manifest_dir = self.d.getVar('SDK_DIR', True) + self.manifest_dir = self.d.getVar('SDK_DIR') else: - self.manifest_dir = self.d.getVar('WORKDIR', True) + self.manifest_dir = self.d.getVar('WORKDIR') else: self.manifest_dir = manifest_dir @@ -83,7 +82,7 @@ class Manifest(object): This will be used for testing until the class is implemented properly! """ def _create_dummy_initial(self): - image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) + image_rootfs = self.d.getVar('IMAGE_ROOTFS') pkg_list = dict() if image_rootfs.find("core-image-sato-sdk") > 0: pkg_list[self.PKG_TYPE_MUST_INSTALL] = \ @@ -105,7 +104,7 @@ class Manifest(object): pkg_list['lgp'] = \ "locale-base-en-us locale-base-en-gb" elif image_rootfs.find("core-image-minimal") > 0: - pkg_list[self.PKG_TYPE_MUST_INSTALL] = "run-postinsts packagegroup-core-boot" + pkg_list[self.PKG_TYPE_MUST_INSTALL] = "packagegroup-core-boot" with open(self.initial_manifest, "w+") as manifest: manifest.write(self.initial_manifest_file_header) @@ -196,7 +195,7 @@ class RpmManifest(Manifest): for pkg in pkg_list.split(): pkg_type = self.PKG_TYPE_MUST_INSTALL - ml_variants = self.d.getVar('MULTILIB_VARIANTS', True).split() + ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() for ml_variant in ml_variants: if pkg.startswith(ml_variant + '-'): @@ -217,13 +216,13 @@ class RpmManifest(Manifest): for var in self.var_maps[self.manifest_type]: if var in self.vars_to_split: - split_pkgs = self._split_multilib(self.d.getVar(var, True)) + split_pkgs = self._split_multilib(self.d.getVar(var)) if split_pkgs is not None: - pkgs = dict(pkgs.items() + split_pkgs.items()) + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: - pkg_list = self.d.getVar(var, True) + pkg_list = self.d.getVar(var) if pkg_list is not None: - pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var, True) + pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) for pkg_type in pkgs: for pkg in pkgs[pkg_type].split(): @@ -246,7 +245,7 @@ class OpkgManifest(Manifest): for pkg in pkg_list.split(): pkg_type = self.PKG_TYPE_MUST_INSTALL - ml_variants = self.d.getVar('MULTILIB_VARIANTS', True).split() + ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() for ml_variant in ml_variants: if pkg.startswith(ml_variant + '-'): @@ -267,13 +266,13 @@ class OpkgManifest(Manifest): for var in self.var_maps[self.manifest_type]: if var in self.vars_to_split: - split_pkgs = self._split_multilib(self.d.getVar(var, True)) + split_pkgs = self._split_multilib(self.d.getVar(var)) if split_pkgs is not None: - pkgs = dict(pkgs.items() + split_pkgs.items()) + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: - pkg_list = self.d.getVar(var, True) + pkg_list = self.d.getVar(var) if pkg_list is not None: - pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var, True) + pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) for pkg_type in pkgs: for pkg in pkgs[pkg_type].split(): @@ -311,7 +310,7 @@ class DpkgManifest(Manifest): manifest.write(self.initial_manifest_file_header) for var in self.var_maps[self.manifest_type]: - pkg_list = self.d.getVar(var, True) + pkg_list = self.d.getVar(var) if pkg_list is None: continue @@ -333,7 +332,7 @@ def create_manifest(d, final_manifest=False, manifest_dir=None, 'ipk': OpkgManifest, 'deb': DpkgManifest} - manifest = manifest_map[d.getVar('IMAGE_PKGTYPE', True)](d, manifest_dir, manifest_type) + manifest = manifest_map[d.getVar('IMAGE_PKGTYPE')](d, manifest_dir, manifest_type) if final_manifest: manifest.create_final() diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index f8b532220a..52c5f16cf8 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -8,7 +8,7 @@ def runstrip(arg): # 8 - shared library # 16 - kernel module - import commands, stat, subprocess + import stat, subprocess (file, elftype, strip) = arg @@ -18,29 +18,30 @@ def runstrip(arg): newmode = origmode | stat.S_IWRITE | stat.S_IREAD os.chmod(file, newmode) - extraflags = "" + stripcmd = [strip] # kernel module if elftype & 16: - extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates" + stripcmd.extend(["--strip-debug", "--remove-section=.comment", + "--remove-section=.note", "--preserve-dates"]) # .so and shared library elif ".so" in file and elftype & 8: - extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded" + stripcmd.extend(["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"]) # shared or executable: elif elftype & 8 or elftype & 4: - extraflags = "--remove-section=.comment --remove-section=.note" + stripcmd.extend(["--remove-section=.comment", "--remove-section=.note"]) - stripcmd = "'%s' %s '%s'" % (strip, extraflags, file) + stripcmd.append(file) bb.debug(1, "runstrip: %s" % stripcmd) - ret = subprocess.call(stripcmd, shell=True) + try: + output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.error("runstrip: '%s' strip command failed with %s (%s)" % (stripcmd, e.returncode, e.output)) if newmode: os.chmod(file, origmode) - if ret: - bb.error("runstrip: '%s' strip command failed" % stripcmd) - return @@ -56,7 +57,7 @@ def file_translate(file): def filedeprunner(arg): import re, subprocess, shlex - (pkg, pkgfiles, rpmdeps, pkgdest) = arg + (pkg, pkgfiles, rpmdeps, pkgdest, magic) = arg provides = {} requires = {} @@ -64,8 +65,8 @@ def filedeprunner(arg): def process_deps(pipe, pkg, pkgdest, provides, requires): for line in pipe: - f = line.split(" ", 1)[0].strip() - line = line.split(" ", 1)[1].strip() + f = line.decode("utf-8").split(" ", 1)[0].strip() + line = line.decode("utf-8").split(" ", 1)[1].strip() if line.startswith("Requires:"): i = requires @@ -89,11 +90,78 @@ def filedeprunner(arg): return provides, requires + env = os.environ.copy() + env["MAGIC"] = magic + try: - dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE) + dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE, env=env) provides, requires = process_deps(dep_popen.stdout, pkg, pkgdest, provides, requires) except OSError as e: bb.error("rpmdeps: '%s' command failed, '%s'" % (shlex.split(rpmdeps) + pkgfiles, e)) raise e return (pkg, provides, requires) + + +def read_shlib_providers(d): + import re + + shlib_provider = {} + shlibs_dirs = d.getVar('SHLIBSDIRS').split() + list_re = re.compile('^(.*)\.list$') + # Go from least to most specific since the last one found wins + for dir in reversed(shlibs_dirs): + bb.debug(2, "Reading shlib providers in %s" % (dir)) + if not os.path.exists(dir): + continue + for file in os.listdir(dir): + m = list_re.match(file) + if m: + dep_pkg = m.group(1) + try: + fd = open(os.path.join(dir, file)) + except IOError: + # During a build unrelated shlib files may be deleted, so + # handle files disappearing between the listdirs and open. + continue + lines = fd.readlines() + fd.close() + for l in lines: + s = l.strip().split(":") + if s[0] not in shlib_provider: + shlib_provider[s[0]] = {} + shlib_provider[s[0]][s[1]] = (dep_pkg, s[2]) + return shlib_provider + + +def npm_split_package_dirs(pkgdir): + """ + Work out the packages fetched and unpacked by BitBake's npm fetcher + Returns a dict of packagename -> (relpath, package.json) ordered + such that it is suitable for use in PACKAGES and FILES + """ + from collections import OrderedDict + import json + packages = {} + for root, dirs, files in os.walk(pkgdir): + if os.path.basename(root) == 'node_modules': + for dn in dirs: + relpth = os.path.relpath(os.path.join(root, dn), pkgdir) + pkgitems = ['${PN}'] + for pathitem in relpth.split('/'): + if pathitem == 'node_modules': + continue + pkgitems.append(pathitem) + pkgname = '-'.join(pkgitems).replace('_', '-') + pkgname = pkgname.replace('@', '') + pkgfile = os.path.join(root, dn, 'package.json') + data = None + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + data = json.loads(f.read()) + packages[pkgname] = (relpth, data) + # We want the main package for a module sorted *after* its subpackages + # (so that it doesn't otherwise steal the files for the subpackage), so + # this is a cheap way to do that whilst still having an otherwise + # alphabetical sort + return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~')) diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py index a0984c404b..f1b65bdbbc 100644 --- a/meta/lib/oe/package_manager.py +++ b/meta/lib/oe/package_manager.py @@ -5,9 +5,13 @@ import subprocess import shutil import multiprocessing import re +import collections import bb import tempfile - +import oe.utils +import oe.path +import string +from oe.gpg_sign import get_signer # this can be used by all PM backends to create the index files in parallel def create_index(arg): @@ -15,17 +19,79 @@ def create_index(arg): try: bb.note("Executing '%s' ..." % index_cmd) - subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: return("Index creation command '%s' failed with return code %d:\n%s" % - (e.cmd, e.returncode, e.output)) - - return None + (e.cmd, e.returncode, e.output.decode("utf-8"))) + if result: + bb.note(result) -class Indexer(object): - __metaclass__ = ABCMeta + return None +""" +This method parse the output from the package managerand return +a dictionary with the information of the packages. This is used +when the packages are in deb or ipk format. +""" +def opkg_query(cmd_output): + verregex = re.compile(' \([=<>]* [^ )]*\)') + output = dict() + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + for line in cmd_output.splitlines(): + line = line.rstrip() + if ':' in line: + if line.startswith("Package: "): + pkg = line.split(": ")[1] + elif line.startswith("Architecture: "): + arch = line.split(": ")[1] + elif line.startswith("Version: "): + ver = line.split(": ")[1] + elif line.startswith("File: ") or line.startswith("Filename:"): + filename = line.split(": ")[1] + if "/" in filename: + filename = os.path.basename(filename) + elif line.startswith("Depends: "): + depends = verregex.sub('', line.split(": ")[1]) + for depend in depends.split(", "): + dep.append(depend) + elif line.startswith("Recommends: "): + recommends = verregex.sub('', line.split(": ")[1]) + for recommend in recommends.split(", "): + dep.append("%s [REC]" % recommend) + elif line.startswith("PackageArch: "): + pkgarch = line.split(": ")[1] + + # When there is a blank line save the package information + elif not line: + # IPK doesn't include the filename + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + if pkg: + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep, "pkgarch":pkgarch } + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + + if pkg: + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + + return output + + +class Indexer(object, metaclass=ABCMeta): def __init__(self, d, deploy_dir): self.d = d self.deploy_dir = deploy_dir @@ -36,96 +102,14 @@ class Indexer(object): class RpmIndexer(Indexer): - def get_ml_prefix_and_os_list(self, arch_var=None, os_var=None): - package_archs = { - 'default': [], - } - - target_os = { - 'default': "", - } - - if arch_var is not None and os_var is not None: - package_archs['default'] = self.d.getVar(arch_var, True).split() - package_archs['default'].reverse() - target_os['default'] = self.d.getVar(os_var, True).strip() - else: - package_archs['default'] = self.d.getVar("PACKAGE_ARCHS", True).split() - # arch order is reversed. This ensures the -best- match is - # listed first! - package_archs['default'].reverse() - target_os['default'] = self.d.getVar("TARGET_OS", True).strip() - multilibs = self.d.getVar('MULTILIBS', True) or "" - for ext in multilibs.split(): - eext = ext.split(':') - if len(eext) > 1 and eext[0] == 'multilib': - localdata = bb.data.createCopy(self.d) - default_tune_key = "DEFAULTTUNE_virtclass-multilib-" + eext[1] - default_tune = localdata.getVar(default_tune_key, False) - if default_tune: - localdata.setVar("DEFAULTTUNE", default_tune) - bb.data.update_data(localdata) - package_archs[eext[1]] = localdata.getVar('PACKAGE_ARCHS', - True).split() - package_archs[eext[1]].reverse() - target_os[eext[1]] = localdata.getVar("TARGET_OS", - True).strip() - - ml_prefix_list = dict() - for mlib in package_archs: - if mlib == 'default': - ml_prefix_list[mlib] = package_archs[mlib] - else: - ml_prefix_list[mlib] = list() - for arch in package_archs[mlib]: - if arch in ['all', 'noarch', 'any']: - ml_prefix_list[mlib].append(arch) - else: - ml_prefix_list[mlib].append(mlib + "_" + arch) - - return (ml_prefix_list, target_os) - def write_index(self): - sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS', True) or "").replace('-', '_').split() - all_mlb_pkg_archs = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split() - - mlb_prefix_list = self.get_ml_prefix_and_os_list()[0] - - archs = set() - for item in mlb_prefix_list: - archs = archs.union(set(i.replace('-', '_') for i in mlb_prefix_list[item])) - - if len(archs) == 0: - archs = archs.union(set(all_mlb_pkg_archs)) - - archs = archs.union(set(sdk_pkg_archs)) - - rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo") - index_cmds = [] - rpm_dirs_found = False - for arch in archs: - arch_dir = os.path.join(self.deploy_dir, arch) - if not os.path.isdir(arch_dir): - continue - - index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir)) - - rpm_dirs_found = True - - if not rpm_dirs_found: - bb.note("There are no packages in %s" % self.deploy_dir) - return - - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(create_index, index_cmds)) - pool.close() - pool.join() - - for result in results: - if result is not None: - return(result) + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + raise NotImplementedError('Package feed signing not yet implementd for rpm') + createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c") + result = create_index("%s --update -q %s" % (createrepo_c, self.deploy_dir)) + if result: + bb.fatal(result) class OpkgIndexer(Indexer): def write_index(self): @@ -134,13 +118,18 @@ class OpkgIndexer(Indexer): "MULTILIB_ARCHS"] opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) + else: + signer = None if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): open(os.path.join(self.deploy_dir, "Packages"), "w").close() - index_cmds = [] + index_cmds = set() + index_sign_files = set() for arch_var in arch_vars: - archs = self.d.getVar(arch_var, True) + archs = self.d.getVar(arch_var) if archs is None: continue @@ -154,35 +143,71 @@ class OpkgIndexer(Indexer): if not os.path.exists(pkgs_file): open(pkgs_file, "w").close() - index_cmds.append('%s -r %s -p %s -m %s' % + index_cmds.add('%s -r %s -p %s -m %s' % (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) + index_sign_files.add(pkgs_file) + if len(index_cmds) == 0: bb.note("There are no packages in %s!" % self.deploy_dir) return - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(create_index, index_cmds)) - pool.close() - pool.join() + result = oe.utils.multiprocess_exec(index_cmds, create_index) + if result: + bb.fatal('%s' % ('\n'.join(result))) - for result in results: - if result is not None: - return(result) + if signer: + feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') + is_ascii_sig = (feed_sig_type.upper() != "BIN") + for f in index_sign_files: + signer.detach_sign(f, + self.d.getVar('PACKAGE_FEED_GPG_NAME'), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), + armor=is_ascii_sig) class DpkgIndexer(Indexer): + def _create_configs(self): + bb.utils.mkdirhier(self.apt_conf_dir) + bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "lists", "partial")) + bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "apt.conf.d")) + bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "preferences.d")) + + with open(os.path.join(self.apt_conf_dir, "preferences"), + "w") as prefs_file: + pass + with open(os.path.join(self.apt_conf_dir, "sources.list"), + "w+") as sources_file: + pass + + with open(self.apt_conf_file, "w") as apt_conf: + with open(os.path.join(self.d.expand("${STAGING_ETCDIR_NATIVE}"), + "apt", "apt.conf.sample")) as apt_conf_sample: + for line in apt_conf_sample.read().split("\n"): + line = re.sub("#ROOTFS#", "/dev/null", line) + line = re.sub("#APTCONF#", self.apt_conf_dir, line) + apt_conf.write(line + "\n") + def write_index(self): - pkg_archs = self.d.getVar('PACKAGE_ARCHS', True) + self.apt_conf_dir = os.path.join(self.d.expand("${APTCONF_TARGET}"), + "apt-ftparchive") + self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") + self._create_configs() + + os.environ['APT_CONFIG'] = self.apt_conf_file + + pkg_archs = self.d.getVar('PACKAGE_ARCHS') if pkg_archs is not None: arch_list = pkg_archs.split() - sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS', True) + sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS') if sdk_pkg_archs is not None: for a in sdk_pkg_archs.split(): if a not in pkg_archs: arch_list.append(a) + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() + arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list) + apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive") gzip = bb.utils.which(os.getenv('PATH'), "gzip") @@ -210,239 +235,81 @@ class DpkgIndexer(Indexer): bb.note("There are no packages in %s" % self.deploy_dir) return - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(create_index, index_cmds)) - pool.close() - pool.join() - - for result in results: - if result is not None: - return(result) + result = oe.utils.multiprocess_exec(index_cmds, create_index) + if result: + bb.fatal('%s' % ('\n'.join(result))) + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + raise NotImplementedError('Package feed signing not implementd for dpkg') -class PkgsList(object): - __metaclass__ = ABCMeta +class PkgsList(object, metaclass=ABCMeta): def __init__(self, d, rootfs_dir): self.d = d self.rootfs_dir = rootfs_dir @abstractmethod - def list(self, format=None): + def list_pkgs(self): pass - class RpmPkgsList(PkgsList): - def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): - super(RpmPkgsList, self).__init__(d, rootfs_dir) - - self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") - self.image_rpmlib = os.path.join(self.rootfs_dir, 'var/lib/rpm') - - self.ml_prefix_list, self.ml_os_list = \ - RpmIndexer(d, rootfs_dir).get_ml_prefix_and_os_list(arch_var, os_var) - - ''' - Translate the RPM/Smart format names to the OE multilib format names - ''' - def _pkg_translate_smart_to_oe(self, pkg, arch): - new_pkg = pkg - new_arch = arch - fixed_arch = arch.replace('_', '-') - found = 0 - for mlib in self.ml_prefix_list: - for cmp_arch in self.ml_prefix_list[mlib]: - fixed_cmp_arch = cmp_arch.replace('_', '-') - if fixed_arch == fixed_cmp_arch: - if mlib == 'default': - new_pkg = pkg - new_arch = cmp_arch - else: - new_pkg = mlib + '-' + pkg - # We need to strip off the ${mlib}_ prefix on the arch - new_arch = cmp_arch.replace(mlib + '_', '') - - # Workaround for bug 3565. Simply look to see if we - # know of a package with that name, if not try again! - filename = os.path.join(self.d.getVar('PKGDATA_DIR', True), - 'runtime-reverse', - new_pkg) - if os.path.exists(filename): - found = 1 - break - - if found == 1 and fixed_arch == fixed_cmp_arch: - break - #bb.note('%s, %s -> %s, %s' % (pkg, arch, new_pkg, new_arch)) - return new_pkg, new_arch - - def _list_pkg_deps(self): - cmd = [bb.utils.which(os.getenv('PATH'), "rpmresolve"), - "-t", self.image_rpmlib] - - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() - except subprocess.CalledProcessError as e: - bb.fatal("Cannot get the package dependencies. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) - - return output - - def list(self, format=None): - if format == "deps": - return self._list_pkg_deps() - - cmd = self.rpm_cmd + ' --root ' + self.rootfs_dir - cmd += ' -D "_dbpath /var/lib/rpm" -qa' - cmd += " --qf '[%{NAME} %{ARCH} %{VERSION} %{PACKAGEORIGIN}\n]'" - - try: - # bb.note(cmd) - tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() - - except subprocess.CalledProcessError as e: - bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) - - output = list() - for line in tmp_output.split('\n'): - if len(line.strip()) == 0: - continue - pkg = line.split()[0] - arch = line.split()[1] - ver = line.split()[2] - pkgorigin = line.split()[3] - new_pkg, new_arch = self._pkg_translate_smart_to_oe(pkg, arch) - - if format == "arch": - output.append('%s %s' % (new_pkg, new_arch)) - elif format == "file": - output.append('%s %s %s' % (new_pkg, pkgorigin, new_arch)) - elif format == "ver": - output.append('%s %s %s' % (new_pkg, new_arch, ver)) - else: - output.append('%s' % (new_pkg)) - - output.sort() - - return '\n'.join(output) - + def list_pkgs(self): + return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR')).list_installed() class OpkgPkgsList(PkgsList): def __init__(self, d, rootfs_dir, config_file): super(OpkgPkgsList, self).__init__(d, rootfs_dir) - self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl") + self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) - self.opkg_args += self.d.getVar("OPKG_ARGS", True) - - def list(self, format=None): - opkg_query_cmd = bb.utils.which(os.getenv('PATH'), "opkg-query-helper.py") - - if format == "arch": - cmd = "%s %s status | %s -a" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - elif format == "file": - cmd = "%s %s status | %s -f" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - elif format == "ver": - cmd = "%s %s status | %s -v" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - elif format == "deps": - cmd = "%s %s status | %s" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - else: - cmd = "%s %s list_installed | cut -d' ' -f1" % \ - (self.opkg_cmd, self.opkg_args) - - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() - except subprocess.CalledProcessError as e: + self.opkg_args += self.d.getVar("OPKG_ARGS") + + def list_pkgs(self, format=None): + cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) + + # opkg returns success even when it printed some + # "Collected errors:" report to stderr. Mixing stderr into + # stdout then leads to random failures later on when + # parsing the output. To avoid this we need to collect both + # output streams separately and check for empty stderr. + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + cmd_output, cmd_stderr = p.communicate() + cmd_output = cmd_output.decode("utf-8") + cmd_stderr = cmd_stderr.decode("utf-8") + if p.returncode or cmd_stderr: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) - - if output and format == "file": - tmp_output = "" - for line in output.split('\n'): - pkg, pkg_file, pkg_arch = line.split() - full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file) - if os.path.exists(full_path): - tmp_output += "%s %s %s\n" % (pkg, full_path, pkg_arch) - else: - tmp_output += "%s %s %s\n" % (pkg, pkg_file, pkg_arch) + "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) - output = tmp_output - - return output + return opkg_query(cmd_output) class DpkgPkgsList(PkgsList): - def list(self, format=None): + + def list_pkgs(self): cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"), "--admindir=%s/var/lib/dpkg" % self.rootfs_dir, "-W"] - if format == "arch": - cmd.append("-f=${Package} ${PackageArch}\n") - elif format == "file": - cmd.append("-f=${Package} ${Package}_${Version}_${Architecture}.deb ${PackageArch}\n") - elif format == "ver": - cmd.append("-f=${Package} ${PackageArch} ${Version}\n") - elif format == "deps": - cmd.append("-f=Package: ${Package}\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") - else: - cmd.append("-f=${Package}\n") + cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) - - if format == "file": - tmp_output = "" - for line in tuple(output.split('\n')): - pkg, pkg_file, pkg_arch = line.split() - full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file) - if os.path.exists(full_path): - tmp_output += "%s %s %s\n" % (pkg, full_path, pkg_arch) - else: - tmp_output += "%s %s %s\n" % (pkg, pkg_file, pkg_arch) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - output = tmp_output - elif format == "deps": - opkg_query_cmd = bb.utils.which(os.getenv('PATH'), "opkg-query-helper.py") - file_out = tempfile.NamedTemporaryFile() - file_out.write(output) - file_out.flush() + return opkg_query(cmd_output) - try: - output = subprocess.check_output("cat %s | %s" % - (file_out.name, opkg_query_cmd), - stderr=subprocess.STDOUT, - shell=True) - except subprocess.CalledProcessError as e: - file_out.close() - bb.fatal("Cannot compute packages dependencies. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) - file_out.close() - - return output - - -class PackageManager(object): +class PackageManager(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta def __init__(self, d): self.d = d self.deploy_dir = None self.deploy_lock = None - self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or "" """ Update the package manager package database. @@ -479,11 +346,27 @@ class PackageManager(object): pass @abstractmethod - def list_installed(self, format=None): + def list_installed(self): + pass + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + """ + @abstractmethod + def extract(self, pkg): pass + """ + Add remote package feeds into repository manager configuration. The parameters + for the feeds are set by feed_uris, feed_base_paths and feed_archs. + See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS + for their description. + """ @abstractmethod - def insert_feeds_uris(self): + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): pass """ @@ -494,18 +377,11 @@ class PackageManager(object): installation """ def install_complementary(self, globs=None): - # we need to write the list of installed packages to a file because the - # oe-pkgdata-util reads it from a file - installed_pkgs_file = os.path.join(self.d.getVar('WORKDIR', True), - "installed_pkgs.txt") - with open(installed_pkgs_file, "w+") as installed_pkgs: - installed_pkgs.write(self.list_installed("arch")) - if globs is None: - globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY', True) + globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY') split_linguas = set() - for translation in self.d.getVar('IMAGE_LINGUAS', True).split(): + for translation in self.d.getVar('IMAGE_LINGUAS').split(): split_linguas.add(translation) split_linguas.add(translation.split('-')[0]) @@ -517,18 +393,28 @@ class PackageManager(object): if globs is None: return - cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), - "glob", self.d.getVar('PKGDATA_DIR', True), installed_pkgs_file, - globs] - try: - bb.note("Installing complementary packages ...") - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - bb.fatal("Could not compute complementary packages list. Command " - "'%s' returned %d:\n%s" % - (' '.join(cmd), e.returncode, e.output)) - - self.install(complementary_pkgs.split(), attempt_only=True) + # we need to write the list of installed packages to a file because the + # oe-pkgdata-util reads it from a file + with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs: + pkgs = self.list_installed() + output = oe.utils.format_pkg_list(pkgs, "arch") + installed_pkgs.write(output) + + cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), + "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name, + globs] + exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY') + if exclude: + cmd.extend(['--exclude=' + '|'.join(exclude.split())]) + try: + bb.note("Installing complementary packages ...") + bb.note('Running %s' % cmd) + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not compute complementary packages list. Command " + "'%s' returned %d:\n%s" % + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + self.install(complementary_pkgs.split(), attempt_only=True) def deploy_dir_lock(self): if self.deploy_dir is None: @@ -546,6 +432,25 @@ class PackageManager(object): self.deploy_lock = None + """ + Construct URIs based on the following pattern: uri/base_path where 'uri' + and 'base_path' correspond to each element of the corresponding array + argument leading to len(uris) x len(base_paths) elements on the returned + array + """ + def construct_uris(self, uris, base_paths): + def _append(arr1, arr2, sep='/'): + res = [] + narr1 = [a.rstrip(sep) for a in arr1] + narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2] + for a1 in narr1: + if arr2: + for a2 in narr2: + res.append("%s%s%s" % (a1, sep, a2)) + else: + res.append(a1) + return res + return _append(uris, base_paths) class RpmPM(PackageManager): def __init__(self, @@ -560,600 +465,390 @@ class RpmPM(PackageManager): self.target_rootfs = target_rootfs self.target_vendor = target_vendor self.task_name = task_name - self.providename = providename - self.fullpkglist = list() - self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM', True) - self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm") - self.install_dir = os.path.join(self.target_rootfs, "install") - self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") - self.smart_cmd = bb.utils.which(os.getenv('PATH'), "smart") - self.smart_opt = "--data-dir=" + os.path.join(target_rootfs, - 'var/lib/smart') - self.scriptlet_wrapper = self.d.expand('${WORKDIR}/scriptlet_wrapper') + if arch_var == None: + self.archs = self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS').replace("-","_") + else: + self.archs = self.d.getVar(arch_var).replace("-","_") + if task_name == "host": + self.primary_arch = self.d.getVar('SDK_ARCH') + else: + self.primary_arch = self.d.getVar('MACHINE_ARCH') + + self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), "oe-rootfs-repo") + bb.utils.mkdirhier(self.rpm_repo_dir) + oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True) + + self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name) + if not os.path.exists(self.d.expand('${T}/saved_packaging_data')): + bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data')) + self.packaging_data_dirs = ['var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf'] self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % self.task_name) - self.saved_rpmlib = self.d.expand('${T}/saved/%s' % self.task_name) - self.image_rpmlib = os.path.join(self.target_rootfs, 'var/lib/rpm') - if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.indexer = RpmIndexer(self.d, self.deploy_dir) - self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, arch_var, os_var) - - self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) + def _configure_dnf(self): + # libsolv handles 'noarch' internally, we don't need to specify it explicitly + archs = [i for i in reversed(self.archs.split()) if i not in ["any", "all", "noarch"]] + # This prevents accidental matching against libsolv's built-in policies + if len(archs) <= 1: + archs = archs + ["bogusarch"] + archconfdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/") + bb.utils.mkdirhier(archconfdir) + open(archconfdir + "arch", 'w').write(":".join(archs)) + + open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w').write("") + + + def _configure_rpm(self): + # We need to configure rpm to use our primary package architecture as the installation architecture, + # and to make it compatible with other package architectures that we use. + # Otherwise it will refuse to proceed with packages installation. + platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/") + rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/") + bb.utils.mkdirhier(platformconfdir) + open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch) + open(rpmrcconfdir + "rpmrc", 'w').write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch)) + + open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n") + if self.d.getVar('RPM_PREFER_ELF_ARCH'): + open(platformconfdir + "macros", 'a').write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH'))) + else: + open(platformconfdir + "macros", 'a').write("%_prefer_color 7") + + if self.d.getVar('RPM_SIGN_PACKAGES') == '1': + signer = get_signer(self.d, self.d.getVar('RPM_GPG_BACKEND')) + pubkey_path = oe.path.join(self.d.getVar('B'), 'rpm-key') + signer.export_pubkey(pubkey_path, self.d.getVar('RPM_GPG_NAME')) + rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmkeys") + cmd = [rpm_bin, '--root=%s' % self.target_rootfs, '--import', pubkey_path] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Importing GPG key failed. Command '%s' " + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - def insert_feeds_uris(self): - if self.feed_uris == "": - return + def create_configs(self): + self._configure_dnf() + self._configure_rpm() - # List must be prefered to least preferred order - default_platform_extra = set() - platform_extra = set() - bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" - for mlib in self.ml_os_list: - for arch in self.ml_prefix_list[mlib]: - plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] - if mlib == bbextendvariant: - default_platform_extra.add(plt) - else: - platform_extra.add(plt) + def write_index(self): + lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock" + lf = bb.utils.lockfile(lockfilename, False) + RpmIndexer(self.d, self.rpm_repo_dir).write_index() + bb.utils.unlockfile(lf) - platform_extra = platform_extra.union(default_platform_extra) + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + from urllib.parse import urlparse - arch_list = [] - for canonical_arch in platform_extra: - arch = canonical_arch.split('-')[0] - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - arch_list.append(arch) - - uri_iterator = 0 - channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list) + if feed_uris == "": + return - for uri in self.feed_uris.split(): - for arch in arch_list: - bb.note('Note: adding Smart channel url%d%s (%s)' % - (uri_iterator, arch, channel_priority)) - self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y' - % (uri_iterator, arch, uri, arch)) - self._invoke_smart('channel --set url%d-%s priority=%d' % - (uri_iterator, arch, channel_priority)) - channel_priority -= 5 - uri_iterator += 1 + bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d")) + remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + for uri in remote_uris: + repo_base = "oe-remote-repo" + "-".join(urlparse(uri).path.split("/")) + if feed_archs is not None: + for arch in feed_archs.split(): + repo_uri = uri + "/" + arch + repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/")) + repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/")) + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write( + "[%s]\nname=%s\nbaseurl=%s\n\n" % (repo_id, repo_name, repo_uri)) + else: + repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/")) + repo_uri = uri + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write( + "[%s]\nname=%s\nbaseurl=%s\n" % (repo_base, repo_name, repo_uri)) - ''' - Create configs for rpm and smart, and multilib is supported - ''' - def create_configs(self): - target_arch = self.d.getVar('TARGET_ARCH', True) - platform = '%s%s-%s' % (target_arch.replace('-', '_'), - self.target_vendor, - self.ml_os_list['default']) - - # List must be prefered to least preferred order - default_platform_extra = list() - platform_extra = list() - bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" - for mlib in self.ml_os_list: - for arch in self.ml_prefix_list[mlib]: - plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] - if mlib == bbextendvariant: - if plt not in default_platform_extra: - default_platform_extra.append(plt) - else: - if plt not in platform_extra: - platform_extra.append(plt) - platform_extra = default_platform_extra + platform_extra + def _prepare_pkg_transaction(self): + os.environ['D'] = self.target_rootfs + os.environ['OFFLINE_ROOT'] = self.target_rootfs + os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['INTERCEPT_DIR'] = oe.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') - self._create_configs(platform, platform_extra) - def _invoke_smart(self, args): - cmd = "%s %s %s" % (self.smart_cmd, self.smart_opt, args) - # bb.note(cmd) - try: - complementary_pkgs = subprocess.check_output(cmd, - stderr=subprocess.STDOUT, - shell=True) - # bb.note(complementary_pkgs) - return complementary_pkgs - except subprocess.CalledProcessError as e: - bb.fatal("Could not invoke smart. Command " - "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output)) - - def _search_pkg_name_in_feeds(self, pkg, feed_archs): - for arch in feed_archs: - arch = arch.replace('-', '_') - for p in self.fullpkglist: - regex_match = r"^%s-[^-]*-[^-]*@%s$" % \ - (re.escape(pkg), re.escape(arch)) - if re.match(regex_match, p) is not None: - # First found is best match - # bb.note('%s -> %s' % (pkg, pkg + '@' + arch)) - return pkg + '@' + arch - - return "" + def install(self, pkgs, attempt_only = False): + if len(pkgs) == 0: + return + self._prepare_pkg_transaction() - ''' - Translate the OE multilib format names to the RPM/Smart format names - It searched the RPM/Smart format names in probable multilib feeds first, - and then searched the default base feed. - ''' - def _pkg_translate_oe_to_smart(self, pkgs, attempt_only=False): - new_pkgs = list() - - for pkg in pkgs: - new_pkg = pkg - # Search new_pkg in probable multilibs first - for mlib in self.ml_prefix_list: - # Jump the default archs - if mlib == 'default': - continue + bad_recommendations = self.d.getVar('BAD_RECOMMENDATIONS') + package_exclude = self.d.getVar('PACKAGE_EXCLUDE') + exclude_pkgs = (bad_recommendations.split() if bad_recommendations else []) + (package_exclude.split() if package_exclude else []) - subst = pkg.replace(mlib + '-', '') - # if the pkg in this multilib feed - if subst != pkg: - feed_archs = self.ml_prefix_list[mlib] - new_pkg = self._search_pkg_name_in_feeds(subst, feed_archs) - if not new_pkg: - # Failed to translate, package not found! - err_msg = '%s not found in the %s feeds (%s).\n' % \ - (pkg, mlib, " ".join(feed_archs)) - if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) - bb.warn(err_msg) - else: - new_pkgs.append(new_pkg) - - break - - # Apparently not a multilib package... - if pkg == new_pkg: - # Search new_pkg in default archs - default_archs = self.ml_prefix_list['default'] - new_pkg = self._search_pkg_name_in_feeds(pkg, default_archs) - if not new_pkg: - err_msg = '%s not found in the base feeds (%s).\n' % \ - (pkg, ' '.join(default_archs)) - if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) - bb.warn(err_msg) - else: - new_pkgs.append(new_pkg) - - return new_pkgs - - def _create_configs(self, platform, platform_extra): - # Setup base system configuration - bb.note("configuring RPM platform settings") - - # Configure internal RPM environment when using Smart - os.environ['RPM_ETCRPM'] = self.etcrpm_dir - bb.utils.mkdirhier(self.etcrpm_dir) - - # Setup temporary directory -- install... - if os.path.exists(self.install_dir): - bb.utils.remove(self.install_dir, True) - bb.utils.mkdirhier(os.path.join(self.install_dir, 'tmp')) - - channel_priority = 5 - platform_dir = os.path.join(self.etcrpm_dir, "platform") - sdkos = self.d.getVar("SDK_OS", True) - with open(platform_dir, "w+") as platform_fd: - platform_fd.write(platform + '\n') - for pt in platform_extra: - channel_priority += 5 - if sdkos: - tmp = re.sub("-%s$" % sdkos, "-%s\n" % sdkos, pt) - tmp = re.sub("-linux.*$", "-linux.*\n", tmp) - platform_fd.write(tmp) - - # Tell RPM that the "/" directory exist and is available - bb.note("configuring RPM system provides") - sysinfo_dir = os.path.join(self.etcrpm_dir, "sysinfo") - bb.utils.mkdirhier(sysinfo_dir) - with open(os.path.join(sysinfo_dir, "Dirnames"), "w+") as dirnames: - dirnames.write("/\n") - - if self.providename: - providename_dir = os.path.join(sysinfo_dir, "Providename") - if not os.path.exists(providename_dir): - providename_content = '\n'.join(self.providename) - providename_content += '\n' - open(providename_dir, "w+").write(providename_content) - - # Configure RPM... we enforce these settings! - bb.note("configuring RPM DB settings") - # After change the __db.* cache size, log file will not be - # generated automatically, that will raise some warnings, - # so touch a bare log for rpm write into it. - rpmlib_log = os.path.join(self.image_rpmlib, 'log', 'log.0000000001') - if not os.path.exists(rpmlib_log): - bb.utils.mkdirhier(os.path.join(self.image_rpmlib, 'log')) - open(rpmlib_log, 'w+').close() - - DB_CONFIG_CONTENT = "# ================ Environment\n" \ - "set_data_dir .\n" \ - "set_create_dir .\n" \ - "set_lg_dir ./log\n" \ - "set_tmp_dir ./tmp\n" \ - "set_flags db_log_autoremove on\n" \ - "\n" \ - "# -- thread_count must be >= 8\n" \ - "set_thread_count 64\n" \ - "\n" \ - "# ================ Logging\n" \ - "\n" \ - "# ================ Memory Pool\n" \ - "set_cachesize 0 1048576 0\n" \ - "set_mp_mmapsize 268435456\n" \ - "\n" \ - "# ================ Locking\n" \ - "set_lk_max_locks 16384\n" \ - "set_lk_max_lockers 16384\n" \ - "set_lk_max_objects 16384\n" \ - "mutex_set_max 163840\n" \ - "\n" \ - "# ================ Replication\n" - - db_config_dir = os.path.join(self.image_rpmlib, 'DB_CONFIG') - if not os.path.exists(db_config_dir): - open(db_config_dir, 'w+').write(DB_CONFIG_CONTENT) - - # Create database so that smart doesn't complain (lazy init) - cmd = "%s --root %s --dbpath /var/lib/rpm -qa > /dev/null" % ( - self.rpm_cmd, - self.target_rootfs) - try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as e: - bb.fatal("Create rpm database failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) - - # Configure smart - bb.note("configuring Smart settings") - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), - True) - self._invoke_smart('config --set rpm-root=%s' % self.target_rootfs) - self._invoke_smart('config --set rpm-dbpath=/var/lib/rpm') - self._invoke_smart('config --set rpm-extra-macros._var=%s' % - self.d.getVar('localstatedir', True)) - cmd = 'config --set rpm-extra-macros._tmppath=/install/tmp' - - prefer_color = self.d.getVar('RPM_PREFER_COLOR', True) - if prefer_color: - if prefer_color not in ['0', '1', '2', '3']: - bb.fatal("Invalid RPM_PREFER_COLOR: %s, it should be one of:\n" - "\t1: ELF32 wins\n" - "\t2: ELF64 wins\n" - "\t3: ELF64 N32 wins (mips64 or mips64el only)" % - prefer_color) - if prefer_color == "3" and self.d.getVar("TUNE_ARCH", True) not in \ - ['mips64', 'mips64el']: - bb.fatal("RPM_PREFER_COLOR = \"3\" is for mips64 or mips64el " - "only.") - self._invoke_smart('config --set rpm-extra-macros._prefer_color=%s' - % prefer_color) - - self._invoke_smart(cmd) - - # Write common configuration for host and target usage - self._invoke_smart('config --set rpm-nolinktos=1') - self._invoke_smart('config --set rpm-noparentdirs=1') - for i in self.d.getVar('BAD_RECOMMENDATIONS', True).split(): - self._invoke_smart('flag --set ignore-recommends %s' % i) - - # Do the following configurations here, to avoid them being - # saved for field upgrade - if self.d.getVar('NO_RECOMMENDATIONS', True).strip() == "1": - self._invoke_smart('config --set ignore-all-recommends=1') - pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE', True) or "" - for i in pkg_exclude.split(): - self._invoke_smart('flag --set exclude-packages %s' % i) - - # Optional debugging - # self._invoke_smart('config --set rpm-log-level=debug') - # cmd = 'config --set rpm-log-file=/tmp/smart-debug-logfile' - # self._invoke_smart(cmd) - ch_already_added = [] - for canonical_arch in platform_extra: - arch = canonical_arch.split('-')[0] - arch_channel = os.path.join(self.deploy_dir, arch) - if os.path.exists(arch_channel) and not arch in ch_already_added: - bb.note('Note: adding Smart channel %s (%s)' % - (arch, channel_priority)) - self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y' - % (arch, arch_channel)) - self._invoke_smart('channel --set %s priority=%d' % - (arch, channel_priority)) - channel_priority -= 5 - - ch_already_added.append(arch) - - bb.note('adding Smart RPM DB channel') - self._invoke_smart('channel --add rpmsys type=rpm-sys -y') - - # Construct install scriptlet wrapper. - # Scripts need to be ordered when executed, this ensures numeric order. - # If we ever run into needing more the 899 scripts, we'll have to. - # change num to start with 1000. - # - SCRIPTLET_FORMAT = "#!/bin/bash\n" \ - "\n" \ - "export PATH=%s\n" \ - "export D=%s\n" \ - 'export OFFLINE_ROOT="$D"\n' \ - 'export IPKG_OFFLINE_ROOT="$D"\n' \ - 'export OPKG_OFFLINE_ROOT="$D"\n' \ - "export INTERCEPT_DIR=%s\n" \ - "export NATIVE_ROOT=%s\n" \ - "\n" \ - "$2 $1/$3 $4\n" \ - "if [ $? -ne 0 ]; then\n" \ - " if [ $4 -eq 1 ]; then\n" \ - " mkdir -p $1/etc/rpm-postinsts\n" \ - " num=100\n" \ - " while [ -e $1/etc/rpm-postinsts/${num}-* ]; do num=$((num + 1)); done\n" \ - " name=`head -1 $1/$3 | cut -d\' \' -f 2`\n" \ - ' echo "#!$2" > $1/etc/rpm-postinsts/${num}-${name}\n' \ - ' echo "# Arg: $4" >> $1/etc/rpm-postinsts/${num}-${name}\n' \ - " cat $1/$3 >> $1/etc/rpm-postinsts/${num}-${name}\n" \ - " chmod +x $1/etc/rpm-postinsts/${num}-${name}\n" \ - " else\n" \ - ' echo "Error: pre/post remove scriptlet failed"\n' \ - " fi\n" \ - "fi\n" - - intercept_dir = self.d.expand('${WORKDIR}/intercept_scripts') - native_root = self.d.getVar('STAGING_DIR_NATIVE', True) - scriptlet_content = SCRIPTLET_FORMAT % (os.environ['PATH'], - self.target_rootfs, - intercept_dir, - native_root) - open(self.scriptlet_wrapper, 'w+').write(scriptlet_content) - - bb.note("Note: configuring RPM cross-install scriptlet_wrapper") - os.chmod(self.scriptlet_wrapper, 0755) - cmd = 'config --set rpm-extra-macros._cross_scriptlet_wrapper=%s' % \ - self.scriptlet_wrapper - self._invoke_smart(cmd) - - # Debug to show smart config info - # bb.note(self._invoke_smart('config --show')) + output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) + + (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) + + (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) + + (["--nogpgcheck"] if self.d.getVar('RPM_SIGN_PACKAGES') != '1' else ["--setopt=gpgcheck=True"]) + + ["install"] + + pkgs) - def update(self): - self._invoke_smart('update rpmsys') + failed_scriptlets_pkgnames = collections.OrderedDict() + for line in output.splitlines(): + if line.startswith("Non-fatal POSTIN scriptlet failure in rpm package"): + failed_scriptlets_pkgnames[line.split()[-1]] = True - ''' - Install pkgs with smart, the pkg name is oe format - ''' - def install(self, pkgs, attempt_only=False): + for pkg in failed_scriptlets_pkgnames.keys(): + self.save_rpmpostinst(pkg) - bb.note("Installing the following packages: %s" % ' '.join(pkgs)) - if attempt_only and len(pkgs) == 0: + def remove(self, pkgs, with_dependencies = True): + if len(pkgs) == 0: return - pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only) - - if not attempt_only: - bb.note('to be installed: %s' % ' '.join(pkgs)) - cmd = "%s %s install -y %s" % \ - (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) - bb.note(cmd) - else: - bb.note('installing attempt only packages...') - bb.note('Attempting %s' % ' '.join(pkgs)) - cmd = "%s %s install --attempt -y %s" % \ - (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) - try: - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) - bb.note(output) - except subprocess.CalledProcessError as e: - bb.fatal("Unable to install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + self._prepare_pkg_transaction() - ''' - Remove pkgs with smart, the pkg name is smart/rpm format - ''' - def remove(self, pkgs, with_dependencies=True): - bb.note('to be removed: ' + ' '.join(pkgs)) - - if not with_dependencies: - cmd = "%s -e --nodeps " % self.rpm_cmd - cmd += "--root=%s " % self.target_rootfs - cmd += "--dbpath=/var/lib/rpm " - cmd += "--define='_cross_scriptlet_wrapper %s' " % \ - self.scriptlet_wrapper - cmd += "--define='_tmppath /install/tmp' %s" % ' '.join(pkgs) + if with_dependencies: + self._invoke_dnf(["remove"] + pkgs) else: - # for pkg in pkgs: - # bb.note('Debug: What required: %s' % pkg) - # bb.note(self._invoke_smart('query %s --show-requiredby' % pkg)) + cmd = bb.utils.which(os.getenv('PATH'), "rpm") + args = ["-e", "--nodeps", "--root=%s" %self.target_rootfs] - cmd = "%s %s remove -y %s" % (self.smart_cmd, - self.smart_opt, - ' '.join(pkgs)) - - try: - bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - bb.note(output) - except subprocess.CalledProcessError as e: - bb.note("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + try: + output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not invoke rpm. Command " + "'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8"))) def upgrade(self): - bb.note('smart upgrade') - self._invoke_smart('upgrade') + self._prepare_pkg_transaction() + self._invoke_dnf(["upgrade"]) - def write_index(self): - result = self.indexer.write_index() - - if result is not None: - bb.fatal(result) + def autoremove(self): + self._prepare_pkg_transaction() + self._invoke_dnf(["autoremove"]) def remove_packaging_data(self): - bb.utils.remove(self.image_rpmlib, True) - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), - True) - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/opkg'), True) - - # remove temp directory - bb.utils.remove(self.d.expand('${IMAGE_ROOTFS}/install'), True) + self._invoke_dnf(["clean", "all"]) + for dir in self.packaging_data_dirs: + bb.utils.remove(oe.path.join(self.target_rootfs, dir), True) def backup_packaging_data(self): - # Save the rpmlib for increment rpm image generation - if os.path.exists(self.saved_rpmlib): - bb.utils.remove(self.saved_rpmlib, True) - shutil.copytree(self.image_rpmlib, - self.saved_rpmlib, - symlinks=True) + # Save the packaging dirs for increment rpm image generation + if os.path.exists(self.saved_packaging_data): + bb.utils.remove(self.saved_packaging_data, True) + for i in self.packaging_data_dirs: + source_dir = oe.path.join(self.target_rootfs, i) + target_dir = oe.path.join(self.saved_packaging_data, i) + shutil.copytree(source_dir, target_dir, symlinks=True) def recovery_packaging_data(self): # Move the rpmlib back - if os.path.exists(self.saved_rpmlib): - if os.path.exists(self.image_rpmlib): - bb.utils.remove(self.image_rpmlib, True) - - bb.note('Recovery packaging data') - shutil.copytree(self.saved_rpmlib, - self.image_rpmlib, + if os.path.exists(self.saved_packaging_data): + for i in self.packaging_data_dirs: + target_dir = oe.path.join(self.target_rootfs, i) + if os.path.exists(target_dir): + bb.utils.remove(target_dir, True) + source_dir = oe.path.join(self.saved_packaging_data, i) + shutil.copytree(source_dir, + target_dir, symlinks=True) - def list_installed(self, format=None): - return self.pkgs_list.list(format) + def list_installed(self): + output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{sourcerpm}\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"], + print_output = False) + packages = {} + current_package = None + current_deps = None + current_state = "initial" + for line in output.splitlines(): + if line.startswith("Package:"): + package_info = line.split(" ")[1:] + current_package = package_info[0] + package_arch = package_info[1] + package_version = package_info[2] + package_srpm = package_info[3] + packages[current_package] = {"arch":package_arch, "ver":package_version, "filename":package_srpm} + current_deps = [] + elif line.startswith("Dependencies:"): + current_state = "dependencies" + elif line.startswith("Recommendations"): + current_state = "recommendations" + elif line.startswith("DependenciesEndHere:"): + current_state = "initial" + packages[current_package]["deps"] = current_deps + elif len(line) > 0: + if current_state == "dependencies": + current_deps.append(line) + elif current_state == "recommendations": + current_deps.append("%s [REC]" % line) + + return packages + + def update(self): + self._invoke_dnf(["makecache"]) + + def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ): + os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs + + dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf") + standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y", + "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), + "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")), + "--repofrompath=oe-repo,%s" % (self.rpm_repo_dir), + "--installroot=%s" % (self.target_rootfs), + "--setopt=logdir=%s" % (self.d.getVar('T')) + ] + cmd = [dnf_cmd] + standard_dnf_args + dnf_args + try: + output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8") + if print_output: + bb.note(output) + return output + except subprocess.CalledProcessError as e: + if print_output: + (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command " + "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + else: + (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command " + "'%s' returned %d:" % (' '.join(cmd), e.returncode)) + return e.output.decode("utf-8") - ''' - If incremental install, we need to determine what we've got, - what we need to add, and what to remove... - The dump_install_solution will dump and save the new install - solution. - ''' def dump_install_solution(self, pkgs): - bb.note('creating new install solution for incremental install') - if len(pkgs) == 0: - return + open(self.solution_manifest, 'w').write(" ".join(pkgs)) + return pkgs - pkgs = self._pkg_translate_oe_to_smart(pkgs, False) - install_pkgs = list() + def load_old_install_solution(self): + if not os.path.exists(self.solution_manifest): + return [] - cmd = "%s %s install -y --dump %s 2>%s" % \ - (self.smart_cmd, - self.smart_opt, - ' '.join(pkgs), - self.solution_manifest) - try: - # Disable rpmsys channel for the fake install - self._invoke_smart('channel --disable rpmsys') + return open(self.solution_manifest, 'r').read().split() - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - with open(self.solution_manifest, 'r') as manifest: - for pkg in manifest.read().split('\n'): - if '@' in pkg: - install_pkgs.append(pkg) + def _script_num_prefix(self, path): + files = os.listdir(path) + numbers = set() + numbers.add(99) + for f in files: + numbers.add(int(f.split("-")[0])) + return max(numbers) + 1 + + def save_rpmpostinst(self, pkg): + bb.note("Saving postinstall script of %s" % (pkg)) + cmd = bb.utils.which(os.getenv('PATH'), "rpm") + args = ["-q", "--root=%s" % self.target_rootfs, "--queryformat", "%{postin}", pkg] + + try: + output = subprocess.check_output([cmd] + args,stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: - bb.note("Unable to dump install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) - # Recovery rpmsys channel - self._invoke_smart('channel --enable rpmsys') - return install_pkgs + bb.fatal("Could not invoke rpm. Command " + "'%s' returned %d:\n%s" % (' '.join([cmd] + args), e.returncode, e.output.decode("utf-8"))) - ''' - If incremental install, we need to determine what we've got, - what we need to add, and what to remove... - The load_old_install_solution will load the previous install - solution - ''' - def load_old_install_solution(self): - bb.note('load old install solution for incremental install') - installed_pkgs = list() - if not os.path.exists(self.solution_manifest): - bb.note('old install solution not exist') - return installed_pkgs + # may need to prepend #!/bin/sh to output - with open(self.solution_manifest, 'r') as manifest: - for pkg in manifest.read().split('\n'): - if '@' in pkg: - installed_pkgs.append(pkg.strip()) + target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/')) + bb.utils.mkdirhier(target_path) + num = self._script_num_prefix(target_path) + saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg)) + open(saved_script_name, 'w').write(output) + os.chmod(saved_script_name, 0o755) - return installed_pkgs + def extract(self, pkg): + output = self._invoke_dnf(["repoquery", "--queryformat", "%{location}", pkg]) + pkg_name = output.splitlines()[-1] + if not pkg_name.endswith(".rpm"): + bb.fatal("dnf could not find package %s in repository: %s" %(pkg, output)) + pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name) + + cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") + rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) - ''' - Dump all available packages in feeds, it should be invoked after the - newest rpm index was created - ''' - def dump_all_available_pkgs(self): - available_manifest = self.d.expand('${T}/saved/available_pkgs.txt') - available_pkgs = list() - cmd = "%s %s query --output %s" % \ - (self.smart_cmd, self.smart_opt, available_manifest) try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - with open(available_manifest, 'r') as manifest: - for pkg in manifest.read().split('\n'): - if '@' in pkg: - available_pkgs.append(pkg.strip()) + cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: - bb.note("Unable to list all available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) - self.fullpkglist = available_pkgs + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + os.chdir(current_dir) - return + return tmp_dir - def save_rpmpostinst(self, pkg): - mlibs = (self.d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() - new_pkg = pkg - # Remove any multilib prefix from the package name - for mlib in mlibs: - if mlib in pkg: - new_pkg = pkg.replace(mlib + '-', '') - break +class OpkgDpkgPM(PackageManager): + """ + This is an abstract class. Do not instantiate this directly. + """ + def __init__(self, d): + super(OpkgDpkgPM, self).__init__(d) - bb.note(' * postponing %s' % new_pkg) - saved_dir = self.target_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') + new_pkg + """ + Returns a dictionary with the package info. - cmd = self.rpm_cmd + ' -q --scripts --root ' + self.target_rootfs - cmd += ' --dbpath=/var/lib/rpm ' + new_pkg - cmd += ' | sed -n -e "/^postinstall scriptlet (using .*):$/,/^.* scriptlet (using .*):$/ {/.*/p}"' - cmd += ' | sed -e "/postinstall scriptlet (using \(.*\)):$/d"' - cmd += ' -e "/^.* scriptlet (using .*):$/d" > %s' % saved_dir + This method extracts the common parts for Opkg and Dpkg + """ + def package_info(self, pkg, cmd): try: - bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() - bb.note(output) - os.chmod(saved_dir, 0755) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: - bb.fatal("Invoke save_rpmpostinst failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + return opkg_query(output) - '''Write common configuration for target usage''' - def rpm_setup_smart_target_config(self): - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), - True) + """ + Returns the path to a tmpdir where resides the contents of a package. - self._invoke_smart('config --set rpm-nolinktos=1') - self._invoke_smart('config --set rpm-noparentdirs=1') - for i in self.d.getVar('BAD_RECOMMENDATIONS', True).split(): - self._invoke_smart('flag --set ignore-recommends %s' % i) - self._invoke_smart('channel --add rpmsys type=rpm-sys -y') + Deleting the tmpdir is responsability of the caller. - ''' - The rpm db lock files were produced after invoking rpm to query on - build system, and they caused the rpm on target didn't work, so we - need to unlock the rpm db by removing the lock files. - ''' - def unlock_rpm_db(self): - # Remove rpm db lock files - rpm_db_locks = glob.glob('%s/var/lib/rpm/__db.*' % self.target_rootfs) - for f in rpm_db_locks: - bb.utils.remove(f, True) + This method extracts the common parts for Opkg and Dpkg + """ + def extract(self, pkg, pkg_info): + + ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") + tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") + pkg_path = pkg_info[pkg]["filepath"] + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + if self.d.getVar('IMAGE_PKGTYPE') == 'deb': + data_tar = 'data.tar.xz' + else: + data_tar = 'data.tar.gz' + + try: + cmd = [ar_cmd, 'x', pkg_path] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + cmd = [tar_cmd, 'xf', data_tar] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename)) + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) + bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) + os.chdir(current_dir) -class OpkgPM(PackageManager): + return tmp_dir + + +class OpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, config_file, archs, task_name='target'): super(OpkgPM, self).__init__(d) @@ -1162,13 +857,13 @@ class OpkgPM(PackageManager): self.pkg_archs = archs self.task_name = task_name - self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True) + self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK") self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") - self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl") - self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs) - self.opkg_args += self.d.getVar("OPKG_ARGS", True) + self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") + self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) + self.opkg_args += self.d.getVar("OPKG_ARGS") - opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') if opkg_lib_dir[0] == "/": opkg_lib_dir = opkg_lib_dir[1:] @@ -1180,10 +875,11 @@ class OpkgPM(PackageManager): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": - self._create_config() - else: + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" + if self.from_feeds: self._create_custom_config() + else: + self._create_config() self.indexer = OpkgIndexer(self.d, self.deploy_dir) @@ -1224,7 +920,7 @@ class OpkgPM(PackageManager): config_file.write("arch %s %d\n" % (arch, priority)) priority += 5 - for line in (self.d.getVar('IPK_FEED_URIS', True) or "").split(): + for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): feed_match = re.match("^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) if feed_match is not None: @@ -1241,18 +937,30 @@ class OpkgPM(PackageManager): specified as compatible for the current machine. NOTE: Development-helper feature, NOT a full-fledged feed. """ - if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True) or "") != "": + if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": for arch in self.pkg_archs.split(): cfg_file_name = os.path.join(self.target_rootfs, - self.d.getVar("sysconfdir", True), + self.d.getVar("sysconfdir"), "opkg", "local-%s-feed.conf" % arch) with open(cfg_file_name, "w+") as cfg_file: cfg_file.write("src/gz local-%s %s/%s" % - arch, - self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True), - arch) + (arch, + self.d.getVar('FEED_DEPLOYDIR_BASE_URI'), + arch)) + + if self.d.getVar('OPKGLIBDIR') != '/var/lib': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" + cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) + cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) + cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) + def _create_config(self): with open(self.config_file, "w+") as config_file: @@ -1269,27 +977,44 @@ class OpkgPM(PackageManager): config_file.write("src oe-%s file:%s\n" % (arch, pkgs_dir)) - def insert_feeds_uris(self): - if self.feed_uris == "": + if self.d.getVar('OPKGLIBDIR') != '/var/lib': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" + config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) + config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) + config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) + + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + if feed_uris == "": return rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' % self.target_rootfs) + feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split() + with open(rootfs_config, "w+") as config_file: uri_iterator = 0 - for uri in self.feed_uris.split(): - config_file.write("src/gz url-%d %s/ipk\n" % - (uri_iterator, uri)) - - for arch in self.pkg_archs.split(): - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - bb.note('Note: adding opkg channel url-%s-%d (%s)' % - (arch, uri_iterator, uri)) + for uri in feed_uris: + if archs: + for arch in archs: + if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))): + continue + bb.note('Adding opkg feed url-%s-%d (%s)' % + (arch, uri_iterator, uri)) + config_file.write("src/gz uri-%s-%d %s/%s\n" % + (arch, uri_iterator, uri, arch)) + else: + bb.note('Adding opkg feed url-%d (%s)' % + (uri_iterator, uri)) + config_file.write("src/gz uri-%d %s\n" % + (uri_iterator, uri)) - config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" % - (arch, uri_iterator, uri, arch)) uri_iterator += 1 def update(self): @@ -1302,12 +1027,12 @@ class OpkgPM(PackageManager): except subprocess.CalledProcessError as e: self.deploy_dir_unlock() bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() def install(self, pkgs, attempt_only=False): - if attempt_only and len(pkgs) == 0: + if not pkgs: return cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) @@ -1316,23 +1041,23 @@ class OpkgPM(PackageManager): os.environ['OFFLINE_ROOT'] = self.target_rootfs os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs - os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR', True), + os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") - os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE', True) + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') try: bb.note("Installing the following packages: %s" % ' '.join(pkgs)) bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) def remove(self, pkgs, with_dependencies=True): if with_dependencies: - cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \ + cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) else: cmd = "%s %s --force-depends remove %s" % \ @@ -1340,11 +1065,11 @@ class OpkgPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1361,11 +1086,15 @@ class OpkgPM(PackageManager): # create the directory back, it's needed by PM lock bb.utils.mkdirhier(self.opkg_dir) - def list_installed(self, format=None): - return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list(format) + def remove_lists(self): + if not self.from_feeds: + bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True) + + def list_installed(self): + return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs() def handle_bad_recommendations(self): - bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True) or "" + bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS") or "" if bad_recommendations.strip() == "": return @@ -1383,10 +1112,10 @@ class OpkgPM(PackageManager): pkg_info = cmd + pkg try: - output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get package info. Command '%s' " - "returned %d:\n%s" % (pkg_info, e.returncode, e.output)) + "returned %d:\n%s" % (pkg_info, e.returncode, e.output.decode("utf-8"))) if output == "": bb.note("Ignored bad recommendation: '%s' is " @@ -1399,6 +1128,10 @@ class OpkgPM(PackageManager): else: status.write(line + "\n") + # Append a blank line after each package entry to ensure that it + # is separated from the following entry + status.write("\n") + ''' The following function dummy installs pkgs and returns the log of output. ''' @@ -1408,18 +1141,21 @@ class OpkgPM(PackageManager): # Create an temp dir as opkg root for dummy installation temp_rootfs = self.d.expand('${T}/opkg') - temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg') + opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + if opkg_lib_dir[0] == "/": + opkg_lib_dir = opkg_lib_dir[1:] + temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') bb.utils.mkdirhier(temp_opkg_dir) opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) - opkg_args += self.d.getVar("OPKG_ARGS", True) + opkg_args += self.d.getVar("OPKG_ARGS") cmd = "%s %s update" % (self.opkg_cmd, opkg_args) try: subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to update. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Dummy installation cmd = "%s %s --noaction install %s " % (self.opkg_cmd, @@ -1429,7 +1165,7 @@ class OpkgPM(PackageManager): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to dummy install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) bb.utils.remove(temp_rootfs, True) @@ -1454,20 +1190,54 @@ class OpkgPM(PackageManager): self.opkg_dir, symlinks=True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) + pkg_info = super(OpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["arch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz")) + + return tmp_dir -class DpkgPM(PackageManager): +class DpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): super(DpkgPM, self).__init__(d) self.target_rootfs = target_rootfs - self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB', True) + self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB') if apt_conf_dir is None: self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt") else: self.apt_conf_dir = apt_conf_dir self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") + self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") + + self.apt_args = d.getVar("APT_ARGS") - self.apt_args = d.getVar("APT_ARGS", True) + self.all_arch_list = archs.split() + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() + self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list) self._create_configs(archs, base_archs) @@ -1507,7 +1277,10 @@ class DpkgPM(PackageManager): """ def run_pre_post_installs(self, package_name=None): info_dir = self.target_rootfs + "/var/lib/dpkg/info" - suffixes = [(".preinst", "Preinstall"), (".postinst", "Postinstall")] + ControlScript = collections.namedtuple("ControlScript", ["suffix", "name", "argument"]) + control_scripts = [ + ControlScript(".preinst", "Preinstall", "install"), + ControlScript(".postinst", "Postinstall", "configure")] status_file = self.target_rootfs + "/var/lib/dpkg/status" installed_pkgs = [] @@ -1524,22 +1297,25 @@ class DpkgPM(PackageManager): os.environ['OFFLINE_ROOT'] = self.target_rootfs os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs - os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR', True), + os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") - os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE', True) + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') failed_pkgs = [] for pkg_name in installed_pkgs: - for suffix in suffixes: - p_full = os.path.join(info_dir, pkg_name + suffix[0]) + for control_script in control_scripts: + p_full = os.path.join(info_dir, pkg_name + control_script.suffix) if os.path.exists(p_full): try: bb.note("Executing %s for package: %s ..." % - (suffix[1].lower(), pkg_name)) - subprocess.check_output(p_full, stderr=subprocess.STDOUT) + (control_script.name.lower(), pkg_name)) + output = subprocess.check_output([p_full, control_script.argument], + stderr=subprocess.STDOUT).decode("utf-8") + bb.note(output) except subprocess.CalledProcessError as e: bb.note("%s for package %s failed with %d:\n%s" % - (suffix[1], pkg_name, e.returncode, e.output)) + (control_script.name, pkg_name, e.returncode, + e.output.decode("utf-8"))) failed_pkgs.append(pkg_name) break @@ -1557,7 +1333,7 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -1576,7 +1352,7 @@ class DpkgPM(PackageManager): except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) # rename *.dpkg-new files/dirs for root, dirs, files in os.walk(self.target_rootfs): @@ -1596,10 +1372,10 @@ class DpkgPM(PackageManager): def remove(self, pkgs, with_dependencies=True): if with_dependencies: os.environ['APT_CONFIG'] = self.apt_conf_file - cmd = "%s remove %s" % (self.apt_get_cmd, ' '.join(pkgs)) + cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs)) else: cmd = "%s --admindir=%s/var/lib/dpkg --instdir=%s" \ - " -r --force-depends %s" % \ + " -P --force-depends %s" % \ (bb.utils.which(os.getenv('PATH'), "dpkg"), self.target_rootfs, self.target_rootfs, ' '.join(pkgs)) @@ -1607,7 +1383,7 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1619,25 +1395,34 @@ class DpkgPM(PackageManager): if result is not None: bb.fatal(result) - def insert_feeds_uris(self): - if self.feed_uris == "": + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + if feed_uris == "": return sources_conf = os.path.join("%s/etc/apt/sources.list" % self.target_rootfs) arch_list = [] - archs = self.d.getVar('PACKAGE_ARCHS', True) - for arch in archs.split(): - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - arch_list.append(arch) + + if feed_archs is None: + for arch in self.all_arch_list: + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + else: + arch_list = feed_archs.split() + + feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) with open(sources_conf, "w+") as sources_file: - for uri in self.feed_uris.split(): - for arch in arch_list: - bb.note('Note: adding dpkg channel at (%s)' % uri) - sources_file.write("deb %s/deb/%s ./\n" % - (uri, arch)) + for uri in feed_uris: + if arch_list: + for arch in arch_list: + bb.note('Adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s/%s ./\n" % + (uri, arch)) + else: + bb.note('Adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s ./\n" % uri) def _create_configs(self, archs, base_archs): base_archs = re.sub("_", "-", base_archs) @@ -1648,9 +1433,10 @@ class DpkgPM(PackageManager): bb.utils.mkdirhier(self.apt_conf_dir) bb.utils.mkdirhier(self.apt_conf_dir + "/lists/partial/") bb.utils.mkdirhier(self.apt_conf_dir + "/apt.conf.d/") + bb.utils.mkdirhier(self.apt_conf_dir + "/preferences.d/") arch_list = [] - for arch in archs.split(): + for arch in self.all_arch_list: if not os.path.exists(os.path.join(self.deploy_dir, arch)): continue arch_list.append(arch) @@ -1665,8 +1451,8 @@ class DpkgPM(PackageManager): priority += 5 - pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE', True) or "" - for pkg in pkg_exclude: + pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or "" + for pkg in pkg_exclude.split(): prefs_file.write( "Package: %s\n" "Pin: release *\n" @@ -1679,15 +1465,31 @@ class DpkgPM(PackageManager): sources_file.write("deb file:%s/ ./\n" % os.path.join(self.deploy_dir, arch)) + base_arch_list = base_archs.split() + multilib_variants = self.d.getVar("MULTILIB_VARIANTS"); + for variant in multilib_variants.split(): + localdata = bb.data.createCopy(self.d) + variant_tune = localdata.getVar("DEFAULTTUNE_virtclass-multilib-" + variant, False) + orig_arch = localdata.getVar("DPKG_ARCH") + localdata.setVar("DEFAULTTUNE", variant_tune) + variant_arch = localdata.getVar("DPKG_ARCH") + if variant_arch not in base_arch_list: + base_arch_list.append(variant_arch) + with open(self.apt_conf_file, "w+") as apt_conf: with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample: for line in apt_conf_sample.read().split("\n"): - line = re.sub("Architecture \".*\";", - "Architecture \"%s\";" % base_archs, line) - line = re.sub("#ROOTFS#", self.target_rootfs, line) - line = re.sub("#APTCONF#", self.apt_conf_dir, line) - - apt_conf.write(line + "\n") + match_arch = re.match(" Architecture \".*\";$", line) + architectures = "" + if match_arch: + for base_arch in base_arch_list: + architectures += "\"%s\";" % base_arch + apt_conf.write(" Architectures {%s};\n" % architectures); + apt_conf.write(" Architecture \"%s\";\n" % base_archs) + else: + line = re.sub("#ROOTFS#", self.target_rootfs, line) + line = re.sub("#APTCONF#", self.apt_conf_dir, line) + apt_conf.write(line + "\n") target_dpkg_dir = "%s/var/lib/dpkg" % self.target_rootfs bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "info")) @@ -1701,7 +1503,7 @@ class DpkgPM(PackageManager): def remove_packaging_data(self): bb.utils.remove(os.path.join(self.target_rootfs, - self.d.getVar('opkglibdir', True)), True) + self.d.getVar('opkglibdir')), True) bb.utils.remove(self.target_rootfs + "/var/lib/dpkg/", True) def fix_broken_dependencies(self): @@ -1713,19 +1515,48 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Cannot fix broken dependencies. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + def list_installed(self): + return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() + + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s show %s" % (self.apt_cache_cmd, pkg) + pkg_info = super(DpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["pkgarch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) - def list_installed(self, format=None): - return DpkgPkgsList(self.d, self.target_rootfs).list() + tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + return tmp_dir def generate_index_files(d): - classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split() + classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split() indexer_map = { - "rpm": (RpmIndexer, d.getVar('DEPLOY_DIR_RPM', True)), - "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK', True)), - "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB', True)) + "rpm": (RpmIndexer, d.getVar('DEPLOY_DIR_RPM')), + "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK')), + "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB')) } result = None diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index cd5f0445f5..32e5c82a94 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -1,4 +1,5 @@ import codecs +import os def packaged(pkg, d): return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) @@ -7,7 +8,7 @@ def read_pkgdatafile(fn): pkgdata = {} def decode(str): - c = codecs.getdecoder("string_escape") + c = codecs.getdecoder("unicode_escape") return c(str)[0] if os.access(fn, os.R_OK): @@ -56,7 +57,7 @@ def read_subpkgdata_dict(pkg, d): def _pkgmap(d): """Return a dictionary mapping package to recipe name.""" - pkgdatadir = d.getVar("PKGDATA_DIR", True) + pkgdatadir = d.getVar("PKGDATA_DIR") pkgmap = {} try: @@ -65,7 +66,7 @@ def _pkgmap(d): bb.warn("No files in %s?" % pkgdatadir) files = [] - for pn in filter(lambda f: not os.path.isdir(os.path.join(pkgdatadir, f)), files): + for pn in [f for f in files if not os.path.isdir(os.path.join(pkgdatadir, f))]: try: pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn)) except OSError: diff --git a/meta/lib/oe/packagegroup.py b/meta/lib/oe/packagegroup.py index 12eb4212ff..4bc5d3e4bb 100644 --- a/meta/lib/oe/packagegroup.py +++ b/meta/lib/oe/packagegroup.py @@ -1,7 +1,7 @@ import itertools def is_optional(feature, d): - packages = d.getVar("FEATURE_PACKAGES_%s" % feature, True) + packages = d.getVar("FEATURE_PACKAGES_%s" % feature) if packages: return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional")) else: @@ -9,18 +9,18 @@ def is_optional(feature, d): def packages(features, d): for feature in features: - packages = d.getVar("FEATURE_PACKAGES_%s" % feature, True) + packages = d.getVar("FEATURE_PACKAGES_%s" % feature) if not packages: - packages = d.getVar("PACKAGE_GROUP_%s" % feature, True) + packages = d.getVar("PACKAGE_GROUP_%s" % feature) for pkg in (packages or "").split(): yield pkg def required_packages(features, d): - req = filter(lambda feature: not is_optional(feature, d), features) + req = [feature for feature in features if not is_optional(feature, d)] return packages(req, d) def optional_packages(features, d): - opt = filter(lambda feature: is_optional(feature, d), features) + opt = [feature for feature in features if is_optional(feature, d)] return packages(opt, d) def active_packages(features, d): diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py index b085c9d6b5..f1ab3dd809 100644 --- a/meta/lib/oe/patch.py +++ b/meta/lib/oe/patch.py @@ -8,12 +8,14 @@ class NotFoundError(bb.BBHandledException): return "Error: %s not found." % self.path class CmdError(bb.BBHandledException): - def __init__(self, exitstatus, output): + def __init__(self, command, exitstatus, output): + self.command = command self.status = exitstatus self.output = output def __str__(self): - return "Command Error: exit status: %d Output:\n%s" % (self.status, self.output) + return "Command Error: '%s' exited with %d Output:\n%s" % \ + (self.command, self.status, self.output) def runcmd(args, dir = None): @@ -32,7 +34,7 @@ def runcmd(args, dir = None): # print("cmd: %s" % cmd) (exitstatus, output) = oe.utils.getstatusoutput(cmd) if exitstatus != 0: - raise CmdError(exitstatus >> 8, output) + raise CmdError(cmd, exitstatus >> 8, output) return output finally: @@ -79,7 +81,7 @@ class PatchSet(object): patch[param] = PatchSet.defaults[param] if patch.get("remote"): - patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d) + patch["file"] = self.d.expand(bb.fetch2.localpath(patch["remote"], self.d)) patch["filemd5"] = bb.utils.md5_file(patch["file"]) @@ -92,6 +94,76 @@ class PatchSet(object): def Refresh(self, remote = None, all = None): raise NotImplementedError() + @staticmethod + def getPatchedFiles(patchfile, striplevel, srcdir=None): + """ + Read a patch file and determine which files it will modify. + Params: + patchfile: the patch file to read + striplevel: the strip level at which the patch is going to be applied + srcdir: optional path to join onto the patched file paths + Returns: + A list of tuples of file path and change mode ('A' for add, + 'D' for delete or 'M' for modify) + """ + + def patchedpath(patchline): + filepth = patchline.split()[1] + if filepth.endswith('/dev/null'): + return '/dev/null' + filesplit = filepth.split(os.sep) + if striplevel > len(filesplit): + bb.error('Patch %s has invalid strip level %d' % (patchfile, striplevel)) + return None + return os.sep.join(filesplit[striplevel:]) + + for encoding in ['utf-8', 'latin-1']: + try: + copiedmode = False + filelist = [] + with open(patchfile) as f: + for line in f: + if line.startswith('--- '): + patchpth = patchedpath(line) + if not patchpth: + break + if copiedmode: + addedfile = patchpth + else: + removedfile = patchpth + elif line.startswith('+++ '): + addedfile = patchedpath(line) + if not addedfile: + break + elif line.startswith('*** '): + copiedmode = True + removedfile = patchedpath(line) + if not removedfile: + break + else: + removedfile = None + addedfile = None + + if addedfile and removedfile: + if removedfile == '/dev/null': + mode = 'A' + elif addedfile == '/dev/null': + mode = 'D' + else: + mode = 'M' + if srcdir: + fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) + else: + fullpath = addedfile + filelist.append((fullpath, mode)) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to decode %s' % patchfile) + + return filelist + class PatchTree(PatchSet): def __init__(self, dir, d): @@ -115,7 +187,8 @@ class PatchTree(PatchSet): def _removePatchFile(self, all = False): if not os.path.exists(self.seriespath): return - patches = open(self.seriespath, 'r+').readlines() + with open(self.seriespath, 'r+') as f: + patches = f.readlines() if all: for p in reversed(patches): self._removePatch(os.path.join(self.patchdir, p.strip())) @@ -148,13 +221,17 @@ class PatchTree(PatchSet): if not force: shellcmd.append('--dry-run') - output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + try: + output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - if force: - return + if force: + return - shellcmd.pop(len(shellcmd) - 1) - output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + shellcmd.pop(len(shellcmd) - 1) + output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + except CmdError as err: + raise bb.BBHandledException("Applying '%s' failed:\n%s" % + (os.path.basename(patch['file']), err.output)) if not reverse: self._appendPatchFile(patch['file'], patch['strippath']) @@ -199,10 +276,200 @@ class PatchTree(PatchSet): self.Pop(all=True) class GitApplyTree(PatchTree): + patch_line_prefix = '%% original patch' + ignore_commit_prefix = '%% ignore' + def __init__(self, dir, d): PatchTree.__init__(self, dir, d) + self.commituser = d.getVar('PATCH_GIT_USER_NAME') + self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL') + + @staticmethod + def extractPatchHeader(patchfile): + """ + Extract just the header lines from the top of a patch file + """ + for encoding in ['utf-8', 'latin-1']: + lines = [] + try: + with open(patchfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): + break + lines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % patchfile) + return lines + + @staticmethod + def decodeAuthor(line): + from email.header import decode_header + authorval = line.split(':', 1)[1].strip().replace('"', '') + result = decode_header(authorval)[0][0] + if hasattr(result, 'decode'): + result = result.decode('utf-8') + return result + + @staticmethod + def interpretPatchHeader(headerlines): + import re + author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') + from_commit_re = re.compile('^From [a-z0-9]{40} .*') + outlines = [] + author = None + date = None + subject = None + for line in headerlines: + if line.startswith('Subject: '): + subject = line.split(':', 1)[1] + # Remove any [PATCH][oe-core] etc. + subject = re.sub(r'\[.+?\]\s*', '', subject) + continue + elif line.startswith('From: ') or line.startswith('Author: '): + authorval = GitApplyTree.decodeAuthor(line) + # git is fussy about author formatting i.e. it must be Name <email@domain> + if author_re.match(authorval): + author = authorval + continue + elif line.startswith('Date: '): + if date is None: + dateval = line.split(':', 1)[1].strip() + # Very crude check for date format, since git will blow up if it's not in the right + # format. Without e.g. a python-dateutils dependency we can't do a whole lot more + if len(dateval) > 12: + date = dateval + continue + elif not author and line.lower().startswith('signed-off-by: '): + authorval = GitApplyTree.decodeAuthor(line) + # git is fussy about author formatting i.e. it must be Name <email@domain> + if author_re.match(authorval): + author = authorval + elif from_commit_re.match(line): + # We don't want the From <commit> line - if it's present it will break rebasing + continue + outlines.append(line) + + if not subject: + firstline = None + for line in headerlines: + line = line.strip() + if firstline: + if line: + # Second line is not blank, the first line probably isn't usable + firstline = None + break + elif line: + firstline = line + if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100: + subject = firstline + + return outlines, author, date, subject + + @staticmethod + def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None): + if d: + commituser = d.getVar('PATCH_GIT_USER_NAME') + commitemail = d.getVar('PATCH_GIT_USER_EMAIL') + if commituser: + cmd += ['-c', 'user.name="%s"' % commituser] + if commitemail: + cmd += ['-c', 'user.email="%s"' % commitemail] + + @staticmethod + def prepareCommit(patchfile, commituser=None, commitemail=None): + """ + Prepare a git commit command line based on the header from a patch file + (typically this is useful for patches that cannot be applied with "git am" due to formatting) + """ + import tempfile + # Process patch header and extract useful information + lines = GitApplyTree.extractPatchHeader(patchfile) + outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) + if not author or not subject or not date: + try: + shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile] + out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) + except CmdError: + out = None + if out: + _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) + if not author: + # If we're setting the author then the date should be set as well + author = newauthor + date = newdate + elif not date: + # If we don't do this we'll get the current date, at least this will be closer + date = newdate + if not subject: + subject = newsubject + if subject and outlines and not outlines[0].strip() == subject: + outlines.insert(0, '%s\n\n' % subject.strip()) + + # Write out commit message to a file + with tempfile.NamedTemporaryFile('w', delete=False) as tf: + tmpfile = tf.name + for line in outlines: + tf.write(line) + # Prepare git command + cmd = ["git"] + GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) + cmd += ["commit", "-F", tmpfile] + # git doesn't like plain email addresses as authors + if author and '<' in author: + cmd.append('--author="%s"' % author) + if date: + cmd.append('--date="%s"' % date) + return (tmpfile, cmd) + + @staticmethod + def extractPatches(tree, startcommit, outdir, paths=None): + import tempfile + import shutil + import re + tempdir = tempfile.mkdtemp(prefix='oepatch') + try: + shellcmd = ["git", "format-patch", startcommit, "-o", tempdir] + if paths: + shellcmd.append('--') + shellcmd.extend(paths) + out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) + if out: + for srcfile in out.split(): + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + outfile = None + try: + with open(srcfile, 'r', encoding=encoding) as f: + for line in f: + checkline = line + if checkline.startswith('Subject: '): + checkline = re.sub(r'\[.+?\]\s*', '', checkline[9:]) + if checkline.startswith(GitApplyTree.patch_line_prefix): + outfile = line.split()[-1].strip() + continue + if checkline.startswith(GitApplyTree.ignore_commit_prefix): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + + if not outfile: + outfile = os.path.basename(srcfile) + with open(os.path.join(outdir, outfile), 'w') as of: + for line in patchlines: + of.write(line) + finally: + shutil.rmtree(tempdir) def _applypatch(self, patch, force = False, reverse = False, run = True): + import shutil + def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True): if reverse: shellcmd.append('-R') @@ -214,17 +481,77 @@ class GitApplyTree(PatchTree): return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Add hooks which add a pointer to the original patch file name in the commit message + reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip() + if not reporoot: + raise Exception("Cannot get repository root for directory %s" % self.dir) + hooks_dir = os.path.join(reporoot, '.git', 'hooks') + hooks_dir_backup = hooks_dir + '.devtool-orig' + if os.path.lexists(hooks_dir_backup): + raise Exception("Git hooks backup directory already exists: %s" % hooks_dir_backup) + if os.path.lexists(hooks_dir): + shutil.move(hooks_dir, hooks_dir_backup) + os.mkdir(hooks_dir) + commithook = os.path.join(hooks_dir, 'commit-msg') + applyhook = os.path.join(hooks_dir, 'applypatch-msg') + with open(commithook, 'w') as f: + # NOTE: the formatting here is significant; if you change it you'll also need to + # change other places which read it back + f.write('echo >> $1\n') + f.write('echo "%s: $PATCHFILE" >> $1\n' % GitApplyTree.patch_line_prefix) + os.chmod(commithook, 0o755) + shutil.copy2(commithook, applyhook) try: - shellcmd = ["git", "--work-tree=.", "am", "-3", "-p%s" % patch['strippath']] - return _applypatchhelper(shellcmd, patch, force, reverse, run) - except CmdError: - shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']] - return _applypatchhelper(shellcmd, patch, force, reverse, run) + patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) + try: + shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] + self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) + shellcmd += ["am", "-3", "--keep-cr", "-p%s" % patch['strippath']] + return _applypatchhelper(shellcmd, patch, force, reverse, run) + except CmdError: + # Need to abort the git am, or we'll still be within it at the end + try: + shellcmd = ["git", "--work-tree=%s" % reporoot, "am", "--abort"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + except CmdError: + pass + # git am won't always clean up after itself, sadly, so... + shellcmd = ["git", "--work-tree=%s" % reporoot, "reset", "--hard", "HEAD"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Also need to take care of any stray untracked files + shellcmd = ["git", "--work-tree=%s" % reporoot, "clean", "-f"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + + # Fall back to git apply + shellcmd = ["git", "--git-dir=%s" % reporoot, "apply", "-p%s" % patch['strippath']] + try: + output = _applypatchhelper(shellcmd, patch, force, reverse, run) + except CmdError: + # Fall back to patch + output = PatchTree._applypatch(self, patch, force, reverse, run) + # Add all files + shellcmd = ["git", "add", "-f", "-A", "."] + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Exclude the patches directory + shellcmd = ["git", "reset", "HEAD", self.patchdir] + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Commit the result + (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) + try: + shellcmd.insert(0, patchfilevar) + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + finally: + os.remove(tmpfile) + return output + finally: + shutil.rmtree(hooks_dir) + if os.path.lexists(hooks_dir_backup): + shutil.move(hooks_dir_backup, hooks_dir) class QuiltTree(PatchSet): def _runcmd(self, args, run = True): - quiltrc = self.d.getVar('QUILTRCFILE', True) + quiltrc = self.d.getVar('QUILTRCFILE') if not run: return ["quilt"] + ["--quiltrc"] + [quiltrc] + args runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir) @@ -254,16 +581,15 @@ class QuiltTree(PatchSet): if not os.path.exists(self.dir): raise NotFoundError(self.dir) if os.path.exists(seriespath): - series = file(seriespath, 'r') - for line in series.readlines(): - patch = {} - parts = line.strip().split() - patch["quiltfile"] = self._quiltpatchpath(parts[0]) - patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) - if len(parts) > 1: - patch["strippath"] = parts[1][2:] - self.patches.append(patch) - series.close() + with open(seriespath, 'r') as f: + for line in f.readlines(): + patch = {} + parts = line.strip().split() + patch["quiltfile"] = self._quiltpatchpath(parts[0]) + patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) + if len(parts) > 1: + patch["strippath"] = parts[1][2:] + self.patches.append(patch) # determine which patches are applied -> self._current try: @@ -285,9 +611,8 @@ class QuiltTree(PatchSet): self.InitFromDir() PatchSet.Import(self, patch, force) oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"]), force=True) - f = open(os.path.join(self.dir, "patches","series"), "a"); - f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"]+"\n") - f.close() + with open(os.path.join(self.dir, "patches", "series"), "a") as f: + f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"] + "\n") patch["quiltfile"] = self._quiltpatchpath(patch["file"]) patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) @@ -402,20 +727,19 @@ class UserResolver(Resolver): # Patch application failed patchcmd = self.patchset.Push(True, False, False) - t = self.patchset.d.getVar('T', True) + t = self.patchset.d.getVar('T') if not t: bb.msg.fatal("Build", "T not set") bb.utils.mkdirhier(t) import random rcfile = "%s/bashrc.%s.%s" % (t, str(os.getpid()), random.random()) - f = open(rcfile, "w") - f.write("echo '*** Manual patch resolution mode ***'\n") - f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n") - f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") - f.write("echo ''\n") - f.write(" ".join(patchcmd) + "\n") - f.close() - os.chmod(rcfile, 0775) + with open(rcfile, "w") as f: + f.write("echo '*** Manual patch resolution mode ***'\n") + f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n") + f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") + f.write("echo ''\n") + f.write(" ".join(patchcmd) + "\n") + os.chmod(rcfile, 0o775) self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d) @@ -445,3 +769,110 @@ class UserResolver(Resolver): os.chdir(olddir) raise os.chdir(olddir) + + +def patch_path(url, fetch, workdir, expand=True): + """Return the local path of a patch, or None if this isn't a patch""" + + local = fetch.localpath(url) + base, ext = os.path.splitext(os.path.basename(local)) + if ext in ('.gz', '.bz2', '.xz', '.Z'): + if expand: + local = os.path.join(workdir, base) + ext = os.path.splitext(base)[1] + + urldata = fetch.ud[url] + if "apply" in urldata.parm: + apply = oe.types.boolean(urldata.parm["apply"]) + if not apply: + return + elif ext not in (".diff", ".patch"): + return + + return local + +def src_patches(d, all=False, expand=True): + workdir = d.getVar('WORKDIR') + fetch = bb.fetch2.Fetch([], d) + patches = [] + sources = [] + for url in fetch.urls: + local = patch_path(url, fetch, workdir, expand) + if not local: + if all: + local = fetch.localpath(url) + sources.append(local) + continue + + urldata = fetch.ud[url] + parm = urldata.parm + patchname = parm.get('pname') or os.path.basename(local) + + apply, reason = should_apply(parm, d) + if not apply: + if reason: + bb.note("Patch %s %s" % (patchname, reason)) + continue + + patchparm = {'patchname': patchname} + if "striplevel" in parm: + striplevel = parm["striplevel"] + elif "pnum" in parm: + #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url) + striplevel = parm["pnum"] + else: + striplevel = '1' + patchparm['striplevel'] = striplevel + + patchdir = parm.get('patchdir') + if patchdir: + patchparm['patchdir'] = patchdir + + localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm)) + patches.append(localurl) + + if all: + return sources + + return patches + + +def should_apply(parm, d): + if "mindate" in parm or "maxdate" in parm: + pn = d.getVar('PN') + srcdate = d.getVar('SRCDATE_%s' % pn) + if not srcdate: + srcdate = d.getVar('SRCDATE') + + if srcdate == "now": + srcdate = d.getVar('DATE') + + if "maxdate" in parm and parm["maxdate"] < srcdate: + return False, 'is outdated' + + if "mindate" in parm and parm["mindate"] > srcdate: + return False, 'is predated' + + + if "minrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and srcrev < parm["minrev"]: + return False, 'applies to later revisions' + + if "maxrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and srcrev > parm["maxrev"]: + return False, 'applies to earlier revisions' + + if "rev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and parm["rev"] not in srcrev: + return False, "doesn't apply to revision" + + if "notrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and parm["notrev"] in srcrev: + return False, "doesn't apply to revision" + + return True, None + diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py index 413ebfb395..448a2b944e 100644 --- a/meta/lib/oe/path.py +++ b/meta/lib/oe/path.py @@ -50,9 +50,30 @@ def make_relative_symlink(path): os.remove(path) os.symlink(base, path) +def replace_absolute_symlinks(basedir, d): + """ + Walk basedir looking for absolute symlinks and replacing them with relative ones. + The absolute links are assumed to be relative to basedir + (compared to make_relative_symlink above which tries to compute common ancestors + using pattern matching instead) + """ + for walkroot, dirs, files in os.walk(basedir): + for file in files + dirs: + path = os.path.join(walkroot, file) + if not os.path.islink(path): + continue + link = os.readlink(path) + if not os.path.isabs(link): + continue + walkdir = os.path.dirname(path.rpartition(basedir)[2]) + base = os.path.relpath(link, walkdir) + bb.debug(2, "Replacing absolute path %s with relative path %s" % (link, base)) + os.remove(path) + os.symlink(base, path) + def format_display(path, metadata): """ Prepare a path for display to the user. """ - rel = relative(metadata.getVar("TOPDIR", True), path) + rel = relative(metadata.getVar("TOPDIR"), path) if len(rel) > len(path): return path else: @@ -65,27 +86,43 @@ def copytree(src, dst): # This way we also preserve hardlinks between files in the tree. bb.utils.mkdirhier(dst) - cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) def copyhardlinktree(src, dst): """ Make the hard link when possible, otherwise copy. """ bb.utils.mkdirhier(dst) if os.path.isdir(src) and not len(os.listdir(src)): - return + return if (os.stat(src).st_dev == os.stat(dst).st_dev): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time - cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --files-from - --no-recursion | tar -xf - -C %s' % (src, src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) - cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + source = '' + if os.path.isdir(src): + if len(glob.glob('%s/.??*' % src)) > 0: + source = './.??* ' + source += './*' + s_dir = src + else: + source = src + s_dir = os.getcwd() + cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst)) + subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) else: copytree(src, dst) def remove(path, recurse=True): - """Equivalent to rm -f or rm -rf""" + """ + Equivalent to rm -f or rm -rf + NOTE: be careful about passing paths that may contain filenames with + wildcards in them (as opposed to passing an actual wildcarded path) - + since we use glob.glob() to expand the path. Filenames containing + square brackets are particularly problematic since the they may not + actually expand to match the original filename. + """ for name in glob.glob(path): try: os.unlink(name) @@ -105,47 +142,6 @@ def symlink(source, destination, force=False): if e.errno != errno.EEXIST or os.readlink(destination) != source: raise -class CalledProcessError(Exception): - def __init__(self, retcode, cmd, output = None): - self.retcode = retcode - self.cmd = cmd - self.output = output - def __str__(self): - return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output) - -# Not needed when we move to python 2.7 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - def find(dir, **walkoptions): """ Given a directory, recurses into that directory, returning all files as absolute paths. """ diff --git a/meta/lib/oe/prservice.py b/meta/lib/oe/prservice.py index b0cbcb1fbc..32dfc15e88 100644 --- a/meta/lib/oe/prservice.py +++ b/meta/lib/oe/prservice.py @@ -1,7 +1,7 @@ def prserv_make_conn(d, check = False): import prserv.serv - host_params = filter(None, (d.getVar("PRSERV_HOST", True) or '').split(':')) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) try: conn = None conn = prserv.serv.PRServerConnection(host_params[0], int(host_params[1])) @@ -9,17 +9,17 @@ def prserv_make_conn(d, check = False): if not conn.ping(): raise Exception('service not available') d.setVar("__PRSERV_CONN",conn) - except Exception, exc: + except Exception as exc: bb.fatal("Connecting to PR service %s:%s failed: %s" % (host_params[0], host_params[1], str(exc))) return conn def prserv_dump_db(d): - if not d.getVar('PRSERV_HOST', True): + if not d.getVar('PRSERV_HOST'): bb.error("Not using network based PR service") return None - conn = d.getVar("__PRSERV_CONN", True) + conn = d.getVar("__PRSERV_CONN") if conn is None: conn = prserv_make_conn(d) if conn is None: @@ -27,18 +27,18 @@ def prserv_dump_db(d): return None #dump db - opt_version = d.getVar('PRSERV_DUMPOPT_VERSION', True) - opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH', True) - opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM', True) - opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL', True)) + opt_version = d.getVar('PRSERV_DUMPOPT_VERSION') + opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH') + opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM') + opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL')) return conn.export(opt_version, opt_pkgarch, opt_checksum, opt_col) def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None): - if not d.getVar('PRSERV_HOST', True): + if not d.getVar('PRSERV_HOST'): bb.error("Not using network based PR service") return None - conn = d.getVar("__PRSERV_CONN", True) + conn = d.getVar("__PRSERV_CONN") if conn is None: conn = prserv_make_conn(d) if conn is None: @@ -58,7 +58,7 @@ def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksu (filter_checksum and filter_checksum != checksum): continue try: - value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum, True)) + value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum)) except BaseException as exc: bb.debug("Not valid value of %s:%s" % (v,str(exc))) continue @@ -72,8 +72,8 @@ def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksu def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): import bb.utils #initilize the output file - bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR', True)) - df = d.getVar('PRSERV_DUMPFILE', True) + bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR')) + df = d.getVar('PRSERV_DUMPFILE') #write data lf = bb.utils.lockfile("%s.lock" % df) f = open(df, "a") @@ -114,7 +114,7 @@ def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): bb.utils.unlockfile(lf) def prserv_check_avail(d): - host_params = filter(None, (d.getVar("PRSERV_HOST", True) or '').split(':')) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) try: if len(host_params) != 2: raise TypeError diff --git a/meta/lib/oe/qa.py b/meta/lib/oe/qa.py index d5cdaa0fcd..3231e60cea 100644 --- a/meta/lib/oe/qa.py +++ b/meta/lib/oe/qa.py @@ -1,3 +1,8 @@ +import os, struct, mmap + +class NotELFFileError(Exception): + pass + class ELFFile: EI_NIDENT = 16 @@ -7,6 +12,8 @@ class ELFFile: EI_OSABI = 7 EI_ABIVERSION = 8 + E_MACHINE = 0x12 + # possible values for EI_CLASS ELFCLASSNONE = 0 ELFCLASS32 = 1 @@ -16,78 +23,104 @@ class ELFFile: EV_CURRENT = 1 # possible values for EI_DATA - ELFDATANONE = 0 - ELFDATA2LSB = 1 - ELFDATA2MSB = 2 + EI_DATA_NONE = 0 + EI_DATA_LSB = 1 + EI_DATA_MSB = 2 + + PT_INTERP = 3 def my_assert(self, expectation, result): if not expectation == result: #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name) - raise Exception("This does not work as expected") + raise NotELFFileError("%s is not an ELF" % self.name) - def __init__(self, name, bits = 0): + def __init__(self, name): self.name = name - self.bits = bits self.objdump_output = {} + # Context Manager functions to close the mmap explicitly + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.data.close() + def open(self): - self.file = file(self.name, "r") - self.data = self.file.read(ELFFile.EI_NIDENT+4) - - self.my_assert(len(self.data), ELFFile.EI_NIDENT+4) - self.my_assert(self.data[0], chr(0x7f) ) - self.my_assert(self.data[1], 'E') - self.my_assert(self.data[2], 'L') - self.my_assert(self.data[3], 'F') - if self.bits == 0: - if self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS32): - self.bits = 32 - elif self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS64): - self.bits = 64 - else: - # Not 32-bit or 64.. lets assert - raise Exception("ELF but not 32 or 64 bit.") - elif self.bits == 32: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS32)) - elif self.bits == 64: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS64)) + with open(self.name, "rb") as f: + try: + self.data = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + except ValueError: + # This means the file is empty + raise NotELFFileError("%s is empty" % self.name) + + # Check the file has the minimum number of ELF table entries + if len(self.data) < ELFFile.EI_NIDENT + 4: + raise NotELFFileError("%s is not an ELF" % self.name) + + # ELF header + self.my_assert(self.data[0], 0x7f) + self.my_assert(self.data[1], ord('E')) + self.my_assert(self.data[2], ord('L')) + self.my_assert(self.data[3], ord('F')) + if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: + self.bits = 32 + elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: + self.bits = 64 else: - raise Exception("Must specify unknown, 32 or 64 bit size.") - self.my_assert(self.data[ELFFile.EI_VERSION], chr(ELFFile.EV_CURRENT) ) - - self.sex = self.data[ELFFile.EI_DATA] - if self.sex == chr(ELFFile.ELFDATANONE): - raise Exception("self.sex == ELFDATANONE") - elif self.sex == chr(ELFFile.ELFDATA2LSB): - self.sex = "<" - elif self.sex == chr(ELFFile.ELFDATA2MSB): - self.sex = ">" - else: - raise Exception("Unknown self.sex") + # Not 32-bit or 64.. lets assert + raise NotELFFileError("ELF but not 32 or 64 bit.") + self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT) + + self.endian = self.data[ELFFile.EI_DATA] + if self.endian not in (ELFFile.EI_DATA_LSB, ELFFile.EI_DATA_MSB): + raise NotELFFileError("Unexpected EI_DATA %x" % self.endian) def osAbi(self): - return ord(self.data[ELFFile.EI_OSABI]) + return self.data[ELFFile.EI_OSABI] def abiVersion(self): - return ord(self.data[ELFFile.EI_ABIVERSION]) + return self.data[ELFFile.EI_ABIVERSION] def abiSize(self): return self.bits def isLittleEndian(self): - return self.sex == "<" + return self.endian == ELFFile.EI_DATA_LSB - def isBigEngian(self): - return self.sex == ">" + def isBigEndian(self): + return self.endian == ELFFile.EI_DATA_MSB + + def getStructEndian(self): + return {ELFFile.EI_DATA_LSB: "<", + ELFFile.EI_DATA_MSB: ">"}[self.endian] + + def getShort(self, offset): + return struct.unpack_from(self.getStructEndian() + "H", self.data, offset)[0] + + def getWord(self, offset): + return struct.unpack_from(self.getStructEndian() + "i", self.data, offset)[0] + + def isDynamic(self): + """ + Return True if there is a .interp segment (therefore dynamically + linked), otherwise False (statically linked). + """ + offset = self.getWord(self.bits == 32 and 0x1C or 0x20) + size = self.getShort(self.bits == 32 and 0x2A or 0x36) + count = self.getShort(self.bits == 32 and 0x2C or 0x38) + + for i in range(0, count): + p_type = self.getWord(offset + i * size) + if p_type == ELFFile.PT_INTERP: + return True + return False def machine(self): """ - We know the sex stored in self.sex and we + We know the endian stored in self.endian and we know the position """ - import struct - (a,) = struct.unpack(self.sex+"H", self.data[18:20]) - return a + return self.getShort(ELFFile.E_MACHINE) def run_objdump(self, cmd, d): import bb.process @@ -96,11 +129,11 @@ class ELFFile: if cmd in self.objdump_output: return self.objdump_output[cmd] - objdump = d.getVar('OBJDUMP', True) + objdump = d.getVar('OBJDUMP') env = os.environ.copy() env["LC_ALL"] = "C" - env["PATH"] = d.getVar('PATH', True) + env["PATH"] = d.getVar('PATH') try: bb.note("%s %s %s" % (objdump, cmd, self.name)) @@ -109,3 +142,30 @@ class ELFFile: except Exception as e: bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e)) return "" + +def elf_machine_to_string(machine): + """ + Return the name of a given ELF e_machine field or the hex value as a string + if it isn't recognised. + """ + try: + return { + 0x02: "SPARC", + 0x03: "x86", + 0x08: "MIPS", + 0x14: "PowerPC", + 0x28: "ARM", + 0x2A: "SuperH", + 0x32: "IA-64", + 0x3E: "x86-64", + 0xB7: "AArch64" + }[machine] + except: + return "Unknown (%s)" % repr(machine) + +if __name__ == "__main__": + import sys + + with ELFFile(sys.argv[1]) as elf: + elf.open() + print(elf.isDynamic()) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py new file mode 100644 index 0000000000..a7fdd36e40 --- /dev/null +++ b/meta/lib/oe/recipeutils.py @@ -0,0 +1,918 @@ +# Utility functions for reading and modifying recipes +# +# Some code borrowed from the OE layer index +# +# Copyright (C) 2013-2016 Intel Corporation +# + +import sys +import os +import os.path +import tempfile +import textwrap +import difflib +from . import utils +import shutil +import re +import fnmatch +import glob +from collections import OrderedDict, defaultdict + + +# Help us to find places to insert values +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()'] +# Variables that sometimes are a bit long but shouldn't be wrapped +nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]'] +list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] +meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] + + +def pn_to_recipe(cooker, pn, mc=''): + """Convert a recipe name (PN) to the path to the recipe file""" + + best = cooker.findBestProvider(pn, mc) + return best[3] + + +def get_unavailable_reasons(cooker, pn): + """If a recipe could not be found, find out why if possible""" + import bb.taskdata + taskdata = bb.taskdata.TaskData(None, skiplist=cooker.skiplist) + return taskdata.get_reasons(pn) + + +def parse_recipe(cooker, fn, appendfiles): + """ + Parse an individual recipe file, optionally with a list of + bbappend files. + """ + import bb.cache + parser = bb.cache.NoCache(cooker.databuilder) + envdata = parser.loadDataFull(fn, appendfiles) + return envdata + + +def get_var_files(fn, varlist, d): + """Find the file in which each of a list of variables is set. + Note: requires variable history to be enabled when parsing. + """ + varfiles = {} + for v in varlist: + history = d.varhistory.variable(v) + files = [] + for event in history: + if 'file' in event and not 'flag' in event: + files.append(event['file']) + if files: + actualfile = files[-1] + else: + actualfile = None + varfiles[v] = actualfile + + return varfiles + + +def split_var_value(value, assignment=True): + """ + Split a space-separated variable's value into a list of items, + taking into account that some of the items might be made up of + expressions containing spaces that should not be split. + Parameters: + value: + The string value to split + assignment: + True to assume that the value represents an assignment + statement, False otherwise. If True, and an assignment + statement is passed in the first item in + the returned list will be the part of the assignment + statement up to and including the opening quote character, + and the last item will be the closing quote. + """ + inexpr = 0 + lastchar = None + out = [] + buf = '' + for char in value: + if char == '{': + if lastchar == '$': + inexpr += 1 + elif char == '}': + inexpr -= 1 + elif assignment and char in '"\'' and inexpr == 0: + if buf: + out.append(buf) + out.append(char) + char = '' + buf = '' + elif char.isspace() and inexpr == 0: + char = '' + if buf: + out.append(buf) + buf = '' + buf += char + lastchar = char + if buf: + out.append(buf) + + # Join together assignment statement and opening quote + outlist = out + if assignment: + assigfound = False + for idx, item in enumerate(out): + if '=' in item: + assigfound = True + if assigfound: + if '"' in item or "'" in item: + outlist = [' '.join(out[:idx+1])] + outlist.extend(out[idx+1:]) + break + return outlist + + +def patch_recipe_lines(fromlines, values, trailing_newline=True): + """Update or insert variable values into lines from a recipe. + Note that some manual inspection/intervention may be required + since this cannot handle all situations. + """ + + import bb.utils + + if trailing_newline: + newline = '\n' + else: + newline = '' + + recipe_progression_res = [] + recipe_progression_restrs = [] + for item in recipe_progression: + if item.endswith('()'): + key = item[:-2] + else: + key = item + restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key + if item.endswith('()'): + recipe_progression_restrs.append(restr + '()') + else: + recipe_progression_restrs.append(restr) + recipe_progression_res.append(re.compile('^%s$' % restr)) + + def get_recipe_pos(variable): + for i, p in enumerate(recipe_progression_res): + if p.match(variable): + return i + return -1 + + remainingnames = {} + for k in values.keys(): + remainingnames[k] = get_recipe_pos(k) + remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1])) + + modifying = False + + def outputvalue(name, lines, rewindcomments=False): + if values[name] is None: + return + rawtext = '%s = "%s"%s' % (name, values[name], newline) + addlines = [] + if name in nowrap_vars: + addlines.append(rawtext) + elif name in list_vars: + splitvalue = split_var_value(values[name], assignment=False) + if len(splitvalue) > 1: + linesplit = ' \\\n' + (' ' * (len(name) + 4)) + addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline)) + else: + addlines.append(rawtext) + else: + wrapped = textwrap.wrap(rawtext) + for wrapline in wrapped[:-1]: + addlines.append('%s \\%s' % (wrapline, newline)) + addlines.append('%s%s' % (wrapped[-1], newline)) + if rewindcomments: + # Ensure we insert the lines before any leading comments + # (that we'd want to ensure remain leading the next value) + for i, ln in reversed(list(enumerate(lines))): + if not ln.startswith('#'): + lines[i+1:i+1] = addlines + break + else: + lines.extend(addlines) + else: + lines.extend(addlines) + + existingnames = [] + def patch_recipe_varfunc(varname, origvalue, op, newlines): + if modifying: + # Insert anything that should come before this variable + pos = get_recipe_pos(varname) + for k in list(remainingnames): + if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: + outputvalue(k, newlines, rewindcomments=True) + del remainingnames[k] + # Now change this variable, if it needs to be changed + if varname in existingnames and op in ['+=', '=', '=+']: + if varname in remainingnames: + outputvalue(varname, newlines) + del remainingnames[varname] + return None, None, 0, True + else: + if varname in values: + existingnames.append(varname) + return origvalue, None, 0, True + + # First run - establish which values we want to set are already in the file + varlist = [re.escape(item) for item in values.keys()] + bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc) + # Second run - actually set everything + modifying = True + varlist.extend(recipe_progression_restrs) + changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) + + if remainingnames: + if tolines and tolines[-1].strip() != '': + tolines.append('\n') + for k in remainingnames.keys(): + outputvalue(k, tolines) + + return changed, tolines + + +def patch_recipe_file(fn, values, patch=False, relpath=''): + """Update or insert variable values into a recipe file (assuming you + have already identified the exact file you want to update.) + Note that some manual inspection/intervention may be required + since this cannot handle all situations. + """ + + with open(fn, 'r') as f: + fromlines = f.readlines() + + _, tolines = patch_recipe_lines(fromlines, values) + + if patch: + relfn = os.path.relpath(fn, relpath) + diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) + return diff + else: + with open(fn, 'w') as f: + f.writelines(tolines) + return None + + +def localise_file_vars(fn, varfiles, varlist): + """Given a list of variables and variable history (fetched with get_var_files()) + find where each variable should be set/changed. This handles for example where a + recipe includes an inc file where variables might be changed - in most cases + we want to update the inc file when changing the variable value rather than adding + it to the recipe itself. + """ + fndir = os.path.dirname(fn) + os.sep + + first_meta_file = None + for v in meta_vars: + f = varfiles.get(v, None) + if f: + actualdir = os.path.dirname(f) + os.sep + if actualdir.startswith(fndir): + first_meta_file = f + break + + filevars = defaultdict(list) + for v in varlist: + f = varfiles[v] + # Only return files that are in the same directory as the recipe or in some directory below there + # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable + # in if we were going to set a value specific to this recipe) + if f: + actualfile = f + else: + # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it + if first_meta_file: + actualfile = first_meta_file + else: + actualfile = fn + + actualdir = os.path.dirname(actualfile) + os.sep + if not actualdir.startswith(fndir): + actualfile = fn + filevars[actualfile].append(v) + + return filevars + +def patch_recipe(d, fn, varvalues, patch=False, relpath=''): + """Modify a list of variable values in the specified recipe. Handles inc files if + used by the recipe. + """ + varlist = varvalues.keys() + varfiles = get_var_files(fn, varlist, d) + locs = localise_file_vars(fn, varfiles, varlist) + patches = [] + for f,v in locs.items(): + vals = {k: varvalues[k] for k in v} + patchdata = patch_recipe_file(f, vals, patch, relpath) + if patch: + patches.append(patchdata) + + if patch: + return patches + else: + return None + + + +def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): + """Copy (local) recipe files, including both files included via include/require, + and files referred to in the SRC_URI variable.""" + import bb.fetch2 + import oe.path + + # FIXME need a warning if the unexpanded SRC_URI value contains variable references + + uris = (d.getVar('SRC_URI') or "").split() + fetch = bb.fetch2.Fetch(uris, d) + if download: + fetch.download() + + # Copy local files to target directory and gather any remote files + bb_dir = os.path.dirname(d.getVar('FILE')) + os.sep + remotes = [] + copied = [] + includes = [path for path in d.getVar('BBINCLUDED').split() if + path.startswith(bb_dir) and os.path.exists(path)] + for path in fetch.localpaths() + includes: + # Only import files that are under the meta directory + if path.startswith(bb_dir): + if not whole_dir: + relpath = os.path.relpath(path, bb_dir) + subdir = os.path.join(tgt_dir, os.path.dirname(relpath)) + if not os.path.exists(subdir): + os.makedirs(subdir) + shutil.copy2(path, os.path.join(tgt_dir, relpath)) + copied.append(relpath) + else: + remotes.append(path) + # Simply copy whole meta dir, if requested + if whole_dir: + shutil.copytree(bb_dir, tgt_dir) + + return copied, remotes + + +def get_recipe_local_files(d, patches=False, archives=False): + """Get a list of local files in SRC_URI within a recipe.""" + import oe.patch + uris = (d.getVar('SRC_URI') or "").split() + fetch = bb.fetch2.Fetch(uris, d) + # FIXME this list should be factored out somewhere else (such as the + # fetcher) though note that this only encompasses actual container formats + # i.e. that can contain multiple files as opposed to those that only + # contain a compressed stream (i.e. .tar.gz as opposed to just .gz) + archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z'] + ret = {} + for uri in uris: + if fetch.ud[uri].type == 'file': + if (not patches and + oe.patch.patch_path(uri, fetch, '', expand=False)): + continue + # Skip files that are referenced by absolute path + fname = fetch.ud[uri].basepath + if os.path.isabs(fname): + continue + # Handle subdir= + subdir = fetch.ud[uri].parm.get('subdir', '') + if subdir: + if os.path.isabs(subdir): + continue + fname = os.path.join(subdir, fname) + localpath = fetch.localpath(uri) + if not archives: + # Ignore archives that will be unpacked + if localpath.endswith(tuple(archive_exts)): + unpack = fetch.ud[uri].parm.get('unpack', True) + if unpack: + continue + ret[fname] = localpath + return ret + + +def get_recipe_patches(d): + """Get a list of the patches included in SRC_URI within a recipe.""" + import oe.patch + patches = oe.patch.src_patches(d, expand=False) + patchfiles = [] + for patch in patches: + _, _, local, _, _, parm = bb.fetch.decodeurl(patch) + patchfiles.append(local) + return patchfiles + + +def get_recipe_patched_files(d): + """ + Get the list of patches for a recipe along with the files each patch modifies. + Params: + d: the datastore for the recipe + Returns: + a dict mapping patch file path to a list of tuples of changed files and + change mode ('A' for add, 'D' for delete or 'M' for modify) + """ + import oe.patch + patches = oe.patch.src_patches(d, expand=False) + patchedfiles = {} + for patch in patches: + _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch) + striplevel = int(parm['striplevel']) + patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', ''))) + return patchedfiles + + +def validate_pn(pn): + """Perform validation on a recipe name (PN) for a new recipe.""" + reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] + if not re.match('^[0-9a-z-.+]+$', pn): + return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn + elif pn in reserved_names: + return 'Recipe name "%s" is invalid: is a reserved keyword' % pn + elif pn.startswith('pn-'): + return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn + elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')): + return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn + return '' + + +def get_bbfile_path(d, destdir, extrapathhint=None): + """ + Determine the correct path for a recipe within a layer + Parameters: + d: Recipe-specific datastore + destdir: destination directory. Can be the path to the base of the layer or a + partial path somewhere within the layer. + extrapathhint: a path relative to the base of the layer to try + """ + import bb.cookerdata + + destdir = os.path.abspath(destdir) + destlayerdir = find_layerdir(destdir) + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + pn = d.getVar('PN') + + bbfilespecs = (confdata.getVar('BBFILES') or '').split() + if destdir == destlayerdir: + for bbfilespec in bbfilespecs: + if not bbfilespec.endswith('.bbappend'): + for match in glob.glob(bbfilespec): + splitext = os.path.splitext(os.path.basename(match)) + if splitext[1] == '.bb': + mpn = splitext[0].split('_')[0] + if mpn == pn: + return os.path.dirname(match) + + # Try to make up a path that matches BBFILES + # this is a little crude, but better than nothing + bpn = d.getVar('BPN') + recipefn = os.path.basename(d.getVar('FILE')) + pathoptions = [destdir] + if extrapathhint: + pathoptions.append(os.path.join(destdir, extrapathhint)) + if destdir == destlayerdir: + pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) + pathoptions.append(os.path.join(destdir, 'recipes', bpn)) + pathoptions.append(os.path.join(destdir, bpn)) + elif not destdir.endswith(('/' + pn, '/' + bpn)): + pathoptions.append(os.path.join(destdir, bpn)) + closepath = '' + for pathoption in pathoptions: + bbfilepath = os.path.join(pathoption, 'test.bb') + for bbfilespec in bbfilespecs: + if fnmatch.fnmatchcase(bbfilepath, bbfilespec): + return pathoption + return None + +def get_bbappend_path(d, destlayerdir, wildcardver=False): + """Determine how a bbappend for a recipe should be named and located within another layer""" + + import bb.cookerdata + + destlayerdir = os.path.abspath(destlayerdir) + recipefile = d.getVar('FILE') + recipefn = os.path.splitext(os.path.basename(recipefile))[0] + if wildcardver and '_' in recipefn: + recipefn = recipefn.split('_', 1)[0] + '_%' + appendfn = recipefn + '.bbappend' + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + + origlayerdir = find_layerdir(recipefile) + if not origlayerdir: + return (None, False) + # Now join this to the path where the bbappend is going and check if it is covered by BBFILES + appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn) + closepath = '' + pathok = True + for bbfilespec in confdata.getVar('BBFILES').split(): + if fnmatch.fnmatchcase(appendpath, bbfilespec): + # Our append path works, we're done + break + elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)): + # Try to find the longest matching path + if len(bbfilespec) > len(closepath): + closepath = bbfilespec + else: + # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure + if closepath: + # bbappend layer's layer.conf at least has a spec that picks up .bbappend files + # Now we just need to substitute out any wildcards + appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir) + if 'recipes-*' in appendsubdir: + # Try to copy this part from the original recipe path + res = re.search('/recipes-[^/]+/', recipefile) + if res: + appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0)) + # This is crude, but we have to do something + appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0]) + appendsubdir = appendsubdir.replace('?', 'a') + appendpath = os.path.join(destlayerdir, appendsubdir, appendfn) + else: + pathok = False + return (appendpath, pathok) + + +def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None): + """ + Writes a bbappend file for a recipe + Parameters: + rd: data dictionary for the recipe + destlayerdir: base directory of the layer to place the bbappend in + (subdirectory path from there will be determined automatically) + srcfiles: dict of source files to add to SRC_URI, where the value + is the full path to the file to be added, and the value is the + original filename as it would appear in SRC_URI or None if it + isn't already present. You may pass None for this parameter if + you simply want to specify your own content via the extralines + parameter. + install: dict mapping entries in srcfiles to a tuple of two elements: + install path (*without* ${D} prefix) and permission value (as a + string, e.g. '0644'). + wildcardver: True to use a % wildcard in the bbappend filename, or + False to make the bbappend specific to the recipe version. + machine: + If specified, make the changes in the bbappend specific to this + machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}" + to be added to the bbappend. + extralines: + Extra lines to add to the bbappend. This may be a dict of name + value pairs, or simply a list of the lines. + removevalues: + Variable values to remove - a dict of names/values. + """ + + if not removevalues: + removevalues = {} + + # Determine how the bbappend should be named + appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) + if not appendpath: + bb.error('Unable to determine layer directory containing %s' % recipefile) + return (None, None) + if not pathok: + bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) + + appenddir = os.path.dirname(appendpath) + bb.utils.mkdirhier(appenddir) + + # FIXME check if the bbappend doesn't get overridden by a higher priority layer? + + layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] + if not os.path.abspath(destlayerdir) in layerdirs: + bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') + + bbappendlines = [] + if extralines: + if isinstance(extralines, dict): + for name, value in extralines.items(): + bbappendlines.append((name, '=', value)) + else: + # Do our best to split it + for line in extralines: + if line[-1] == '\n': + line = line[:-1] + splitline = line.split(None, 2) + if len(splitline) == 3: + bbappendlines.append(tuple(splitline)) + else: + raise Exception('Invalid extralines value passed') + + def popline(varname): + for i in range(0, len(bbappendlines)): + if bbappendlines[i][0] == varname: + line = bbappendlines.pop(i) + return line + return None + + def appendline(varname, op, value): + for i in range(0, len(bbappendlines)): + item = bbappendlines[i] + if item[0] == varname: + bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value) + break + else: + bbappendlines.append((varname, op, value)) + + destsubdir = rd.getVar('PN') + if srcfiles: + bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:')) + + appendoverride = '' + if machine: + bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}')) + appendoverride = '_%s' % machine + copyfiles = {} + if srcfiles: + instfunclines = [] + for newfile, origsrcfile in srcfiles.items(): + srcfile = origsrcfile + srcurientry = None + if not srcfile: + srcfile = os.path.basename(newfile) + srcurientry = 'file://%s' % srcfile + # Double-check it's not there already + # FIXME do we care if the entry is added by another bbappend that might go away? + if not srcurientry in rd.getVar('SRC_URI').split(): + if machine: + appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry) + else: + appendline('SRC_URI', '+=', srcurientry) + copyfiles[newfile] = srcfile + if install: + institem = install.pop(newfile, None) + if institem: + (destpath, perms) = institem + instdestpath = replace_dir_vars(destpath, rd) + instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath) + if not instdirline in instfunclines: + instfunclines.append(instdirline) + instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath)) + if instfunclines: + bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines)) + + bb.note('Writing append file %s' % appendpath) + + if os.path.exists(appendpath): + # Work around lack of nonlocal in python 2 + extvars = {'destsubdir': destsubdir} + + def appendfile_varfunc(varname, origvalue, op, newlines): + if varname == 'FILESEXTRAPATHS_prepend': + if origvalue.startswith('${THISDIR}/'): + popline('FILESEXTRAPATHS_prepend') + extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':')) + elif varname == 'PACKAGE_ARCH': + if machine: + popline('PACKAGE_ARCH') + return (machine, None, 4, False) + elif varname.startswith('do_install_append'): + func = popline(varname) + if func: + instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()] + for line in func[2]: + if not line in instfunclines: + instfunclines.append(line) + return (instfunclines, None, 4, False) + else: + splitval = split_var_value(origvalue, assignment=False) + changed = False + removevar = varname + if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]: + removevar = 'SRC_URI' + line = popline(varname) + if line: + if line[2] not in splitval: + splitval.append(line[2]) + changed = True + else: + line = popline(varname) + if line: + splitval = [line[2]] + changed = True + + if removevar in removevalues: + remove = removevalues[removevar] + if isinstance(remove, str): + if remove in splitval: + splitval.remove(remove) + changed = True + else: + for removeitem in remove: + if removeitem in splitval: + splitval.remove(removeitem) + changed = True + + if changed: + newvalue = splitval + if len(newvalue) == 1: + # Ensure it's written out as one line + if '_append' in varname: + newvalue = ' ' + newvalue[0] + else: + newvalue = newvalue[0] + if not newvalue and (op in ['+=', '.='] or '_append' in varname): + # There's no point appending nothing + newvalue = None + if varname.endswith('()'): + indent = 4 + else: + indent = -1 + return (newvalue, None, indent, True) + return (origvalue, None, 4, False) + + varnames = [item[0] for item in bbappendlines] + if removevalues: + varnames.extend(list(removevalues.keys())) + + with open(appendpath, 'r') as f: + (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) + + destsubdir = extvars['destsubdir'] + else: + updated = False + newlines = [] + + if bbappendlines: + for line in bbappendlines: + if line[0].endswith('()'): + newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2]))) + else: + newlines.append('%s %s "%s"\n\n' % line) + updated = True + + if updated: + with open(appendpath, 'w') as f: + f.writelines(newlines) + + if copyfiles: + if machine: + destsubdir = os.path.join(destsubdir, machine) + for newfile, srcfile in copyfiles.items(): + filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile)) + if os.path.abspath(newfile) != os.path.abspath(filedest): + if newfile.startswith(tempfile.gettempdir()): + newfiledisp = os.path.basename(newfile) + else: + newfiledisp = newfile + bb.note('Copying %s to %s' % (newfiledisp, filedest)) + bb.utils.mkdirhier(os.path.dirname(filedest)) + shutil.copyfile(newfile, filedest) + + return (appendpath, os.path.join(appenddir, destsubdir)) + + +def find_layerdir(fn): + """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" + pth = fn + layerdir = '' + while pth: + if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): + layerdir = pth + break + pth = os.path.dirname(pth) + if pth == '/': + return None + return layerdir + + +def replace_dir_vars(path, d): + """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})""" + dirvars = {} + # Sort by length so we get the variables we're interested in first + for var in sorted(list(d.keys()), key=len): + if var.endswith('dir') and var.lower() == var: + value = d.getVar(var) + if value.startswith('/') and not '\n' in value and value not in dirvars: + dirvars[value] = var + for dirpath in sorted(list(dirvars.keys()), reverse=True): + path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) + return path + +def get_recipe_pv_without_srcpv(pv, uri_type): + """ + Get PV without SRCPV common in SCM's for now only + support git. + + Returns tuple with pv, prefix and suffix. + """ + pfx = '' + sfx = '' + + if uri_type == 'git': + git_regex = re.compile("(?P<pfx>v?)(?P<ver>[^\+]*)((?P<sfx>\+(git)?r?(AUTOINC\+))(?P<rev>.*))?") + m = git_regex.match(pv) + + if m: + pv = m.group('ver') + pfx = m.group('pfx') + sfx = m.group('sfx') + else: + regex = re.compile("(?P<pfx>(v|r)?)(?P<ver>.*)") + m = regex.match(pv) + if m: + pv = m.group('ver') + pfx = m.group('pfx') + + return (pv, pfx, sfx) + +def get_recipe_upstream_version(rd): + """ + Get upstream version of recipe using bb.fetch2 methods with support for + http, https, ftp and git. + + bb.fetch2 exceptions can be raised, + FetchError when don't have network access or upstream site don't response. + NoMethodError when uri latest_versionstring method isn't implemented. + + Returns a dictonary with version, type and datetime. + Type can be A for Automatic, M for Manual and U for Unknown. + """ + from bb.fetch2 import decodeurl + from datetime import datetime + + ru = {} + ru['version'] = '' + ru['type'] = 'U' + ru['datetime'] = '' + + pv = rd.getVar('PV') + + # XXX: If don't have SRC_URI means that don't have upstream sources so + # returns the current recipe version, so that upstream version check + # declares a match. + src_uris = rd.getVar('SRC_URI') + if not src_uris: + ru['version'] = pv + ru['type'] = 'M' + ru['datetime'] = datetime.now() + return ru + + # XXX: we suppose that the first entry points to the upstream sources + src_uri = src_uris.split()[0] + uri_type, _, _, _, _, _ = decodeurl(src_uri) + + manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION") + if manual_upstream_version: + # manual tracking of upstream version. + ru['version'] = manual_upstream_version + ru['type'] = 'M' + + manual_upstream_date = rd.getVar("CHECK_DATE") + if manual_upstream_date: + date = datetime.strptime(manual_upstream_date, "%b %d, %Y") + else: + date = datetime.now() + ru['datetime'] = date + + elif uri_type == "file": + # files are always up-to-date + ru['version'] = pv + ru['type'] = 'A' + ru['datetime'] = datetime.now() + else: + ud = bb.fetch2.FetchData(src_uri, rd) + pupver = ud.method.latest_versionstring(ud, rd) + (upversion, revision) = pupver + + # format git version version+gitAUTOINC+HASH + if uri_type == 'git': + (pv, pfx, sfx) = get_recipe_pv_without_srcpv(pv, uri_type) + + # if contains revision but not upversion use current pv + if upversion == '' and revision: + upversion = pv + + if upversion: + tmp = upversion + upversion = '' + + if pfx: + upversion = pfx + tmp + else: + upversion = tmp + + if sfx: + upversion = upversion + sfx + revision[:10] + + if upversion: + ru['version'] = upversion + ru['type'] = 'A' + + ru['datetime'] = datetime.now() + + return ru diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py index 0424a01aa5..96591f370e 100644 --- a/meta/lib/oe/rootfs.py +++ b/meta/lib/oe/rootfs.py @@ -10,17 +10,18 @@ import subprocess import re -class Rootfs(object): +class Rootfs(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta - def __init__(self, d): + def __init__(self, d, progress_reporter=None, logcatcher=None): self.d = d self.pm = None - self.image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) - self.deploy_dir_image = self.d.getVar('DEPLOY_DIR_IMAGE', True) + self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') + self.deploydir = self.d.getVar('IMGDEPLOYDIR') + self.progress_reporter = progress_reporter + self.logcatcher = logcatcher self.install_order = Manifest.INSTALL_ORDER @@ -40,10 +41,56 @@ class Rootfs(object): def _log_check(self): pass + def _log_check_common(self, type, match): + # Ignore any lines containing log_check to avoid recursion, and ignore + # lines beginning with a + since sh -x may emit code which isn't + # actually executed, but may contain error messages + excludes = [ 'log_check', r'^\+' ] + if hasattr(self, 'log_check_expected_regexes'): + excludes.extend(self.log_check_expected_regexes) + excludes = [re.compile(x) for x in excludes] + r = re.compile(match) + log_path = self.d.expand("${T}/log.do_rootfs") + messages = [] + with open(log_path, 'r') as log: + for line in log: + if self.logcatcher and self.logcatcher.contains(line.rstrip()): + continue + for ee in excludes: + m = ee.search(line) + if m: + break + if m: + continue + + m = r.search(line) + if m: + messages.append('[log_check] %s' % line) + if messages: + if len(messages) == 1: + msg = '1 %s message' % type + else: + msg = '%d %s messages' % (len(messages), type) + msg = '[log_check] %s: found %s in the logfile:\n%s' % \ + (self.d.getVar('PN'), msg, ''.join(messages)) + if type == 'error': + bb.fatal(msg) + else: + bb.warn(msg) + + def _log_check_warn(self): + self._log_check_common('warning', '^(warn|Warn|WARNING:)') + + def _log_check_error(self): + self._log_check_common('error', self.log_check_regex) + def _insert_feed_uris(self): if bb.utils.contains("IMAGE_FEATURES", "package-management", True, False, self.d): - self.pm.insert_feeds_uris() + self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", + self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", + self.d.getVar('PACKAGE_FEED_ARCHS')) + @abstractmethod def _handle_intercept_failure(self, failed_script): @@ -58,8 +105,61 @@ class Rootfs(object): def _cleanup(self): pass + def _setup_dbg_rootfs(self, dirs): + gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' + if gen_debugfs != '1': + return + + bb.note(" Renaming the original rootfs...") + try: + shutil.rmtree(self.image_rootfs + '-orig') + except: + pass + os.rename(self.image_rootfs, self.image_rootfs + '-orig') + + bb.note(" Creating debug rootfs...") + bb.utils.mkdirhier(self.image_rootfs) + + bb.note(" Copying back package database...") + for dir in dirs: + if not os.path.isdir(self.image_rootfs + '-orig' + dir): + continue + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) + shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) + + cpath = oe.cachedpath.CachedPath() + # Copy files located in /usr/lib/debug or /usr/src/debug + for dir in ["/usr/lib/debug", "/usr/src/debug"]: + src = self.image_rootfs + '-orig' + dir + if cpath.exists(src): + dst = self.image_rootfs + dir + bb.utils.mkdirhier(os.path.dirname(dst)) + shutil.copytree(src, dst) + + # Copy files with suffix '.debug' or located in '.debug' dir. + for root, dirs, files in cpath.walk(self.image_rootfs + '-orig'): + relative_dir = root[len(self.image_rootfs + '-orig'):] + for f in files: + if f.endswith('.debug') or '/.debug' in relative_dir: + bb.utils.mkdirhier(self.image_rootfs + relative_dir) + shutil.copy(os.path.join(root, f), + self.image_rootfs + relative_dir) + + bb.note(" Install complementary '*-dbg' packages...") + self.pm.install_complementary('*-dbg') + + bb.note(" Rename debug rootfs...") + try: + shutil.rmtree(self.image_rootfs + '-dbg') + except: + pass + os.rename(self.image_rootfs, self.image_rootfs + '-dbg') + + bb.note(" Restoreing original rootfs...") + os.rename(self.image_rootfs + '-orig', self.image_rootfs) + def _exec_shell_cmd(self, cmd): - fakerootcmd = self.d.getVar('FAKEROOT', True) + fakerootcmd = self.d.getVar('FAKEROOT') if fakerootcmd is not None: exec_cmd = [fakerootcmd, cmd] else: @@ -74,39 +174,46 @@ class Rootfs(object): def create(self): bb.note("###### Generate rootfs #######") - pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND", True) - post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND", True) - - intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True), + pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") + post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") + rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') + + postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR") + if not postinst_intercepts_dir: + postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts") + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") bb.utils.remove(intercepts_dir, True) bb.utils.mkdirhier(self.image_rootfs) - bb.utils.mkdirhier(self.deploy_dir_image) + bb.utils.mkdirhier(self.deploydir) - shutil.copytree(self.d.expand("${COREBASE}/scripts/postinst-intercepts"), - intercepts_dir) - - shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"), - self.deploy_dir_image + - "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt") + shutil.copytree(postinst_intercepts_dir, intercepts_dir) execute_pre_post_process(self.d, pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # call the package manager dependent create method self._create() - sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir', True) + sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') bb.utils.mkdirhier(sysconfdir) with open(sysconfdir + "/version", "w+") as ver: - ver.write(self.d.getVar('BUILDNAME', True) + "\n") + ver.write(self.d.getVar('BUILDNAME') + "\n") + + execute_pre_post_process(self.d, rootfs_post_install_cmds) self._run_intercepts() execute_pre_post_process(self.d, post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d): delayed_postinsts = self._get_delayed_postinsts() @@ -115,63 +222,77 @@ class Rootfs(object): "offline and rootfs is read-only: %s" % delayed_postinsts) - if self.d.getVar('USE_DEVFS', True) != "1": + if self.d.getVar('USE_DEVFS') != "1": self._create_devfs() - self._uninstall_uneeded() + self._uninstall_unneeded() + + if self.progress_reporter: + self.progress_reporter.next_stage() self._insert_feed_uris() self._run_ldconfig() - self._generate_kernel_module_deps() + if self.d.getVar('USE_DEPMOD') != "0": + self._generate_kernel_module_deps() self._cleanup() + self._log_check() + + if self.progress_reporter: + self.progress_reporter.next_stage() - def _uninstall_uneeded(self): + + def _uninstall_unneeded(self): # Remove unneeded init script symlinks delayed_postinsts = self._get_delayed_postinsts() if delayed_postinsts is None: if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")): self._exec_shell_cmd(["update-rc.d", "-f", "-r", - self.d.getVar('IMAGE_ROOTFS', True), + self.d.getVar('IMAGE_ROOTFS'), "run-postinsts", "remove"]) - # Remove unneeded package-management related components - if bb.utils.contains("IMAGE_FEATURES", "package-management", - True, False, self.d): - return - - if delayed_postinsts is None: - installed_pkgs_dir = self.d.expand('${WORKDIR}/installed_pkgs.txt') - pkgs_to_remove = list() - with open(installed_pkgs_dir, "r+") as installed_pkgs: - pkgs_installed = installed_pkgs.read().split('\n') - for pkg_installed in pkgs_installed[:]: - pkg = pkg_installed.split()[0] - if pkg in ["update-rc.d", - "base-passwd", - self.d.getVar("ROOTFS_BOOTSTRAP_INSTALL", True) - ]: - pkgs_to_remove.append(pkg) - pkgs_installed.remove(pkg_installed) + image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", + True, False, self.d) + image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') + + if image_rorfs or image_rorfs_force == "1": + # Remove components that we don't need if it's a read-only rootfs + unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() + pkgs_installed = image_list_installed_packages(self.d) + # Make sure update-alternatives is last on the command line, so + # that it is removed last. This makes sure that its database is + # available while uninstalling packages, allowing alternative + # symlinks of packages to be uninstalled to be managed correctly. + provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") + pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) if len(pkgs_to_remove) > 0: self.pm.remove(pkgs_to_remove, False) - # Update installed_pkgs.txt - open(installed_pkgs_dir, "w+").write('\n'.join(pkgs_installed)) - else: + if delayed_postinsts: self._save_postinsts() + if image_rorfs: + bb.warn("There are post install scripts " + "in a read-only rootfs") - self.pm.remove_packaging_data() + post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") + execute_pre_post_process(self.d, post_uninstall_cmds) + + runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", + True, False, self.d) + if not runtime_pkgmanage: + # Remove the package manager data files + self.pm.remove_packaging_data() def _run_intercepts(self): - intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True), + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") bb.note("Running intercept scripts:") os.environ['D'] = self.image_rootfs + os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE') for script in os.listdir(intercepts_dir): script_full = os.path.join(intercepts_dir, script) @@ -183,8 +304,8 @@ class Rootfs(object): try: subprocess.check_output(script_full) except subprocess.CalledProcessError as e: - bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details!" % - (script, e.returncode)) + bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details! (Output: %s)" % + (script, e.returncode, e.output)) with open(script_full) as intercept: registered_pkgs = None @@ -203,22 +324,37 @@ class Rootfs(object): self._handle_intercept_failure(registered_pkgs) def _run_ldconfig(self): - if self.d.getVar('LDCONFIGDEPEND', True): + if self.d.getVar('LDCONFIGDEPEND'): bb.note("Executing: ldconfig -r" + self.image_rootfs + "-c new -v") self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 'new', '-v']) + def _check_for_kernel_modules(self, modules_dir): + for root, dirs, files in os.walk(modules_dir, topdown=True): + for name in files: + found_ko = name.endswith(".ko") + if found_ko: + return found_ko + return False + def _generate_kernel_module_deps(self): - kernel_abi_ver_file = os.path.join(self.d.getVar('STAGING_KERNEL_DIR', True), + modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') + # if we don't have any modules don't bother to do the depmod + if not self._check_for_kernel_modules(modules_dir): + bb.note("No Kernel Modules found, not running depmod") + return + + kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod", 'kernel-abiversion') - if os.path.exists(kernel_abi_ver_file): - kernel_ver = open(kernel_abi_ver_file).read().strip(' \n') - modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules', kernel_ver) + if not os.path.exists(kernel_abi_ver_file): + bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) + + kernel_ver = open(kernel_abi_ver_file).read().strip(' \n') + versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) - bb.utils.mkdirhier(modules_dir) + bb.utils.mkdirhier(versioned_modules_dir) - self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, - kernel_ver]) + self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver]) """ Create devfs: @@ -230,15 +366,15 @@ class Rootfs(object): """ def _create_devfs(self): devtable_list = [] - devtable = self.d.getVar('IMAGE_DEVICE_TABLE', True) + devtable = self.d.getVar('IMAGE_DEVICE_TABLE') if devtable is not None: devtable_list.append(devtable) else: - devtables = self.d.getVar('IMAGE_DEVICE_TABLES', True) + devtables = self.d.getVar('IMAGE_DEVICE_TABLES') if devtables is None: devtables = 'files/device_table-minimal.txt' for devtable in devtables.split(): - devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH', True), devtable)) + devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) for devtable in devtable_list: self._exec_shell_cmd(["makedevs", "-r", @@ -246,22 +382,24 @@ class Rootfs(object): class RpmRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(RpmRootfs, self).__init__(d) - + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(RpmRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ + '|exit 1|ERROR: |Error: |Error |ERROR '\ + '|Failed |Failed: |Failed$|Failed\(\d+\):)' self.manifest = RpmManifest(d, manifest_dir) self.pm = RpmPM(d, - d.getVar('IMAGE_ROOTFS', True), - self.d.getVar('TARGET_VENDOR', True) + d.getVar('IMAGE_ROOTFS'), + self.d.getVar('TARGET_VENDOR') ) - self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN', True) + self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN') if self.inc_rpm_image_gen != "1": bb.utils.remove(self.image_rootfs, True) else: self.pm.recovery_packaging_data() - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) self.pm.create_configs() @@ -293,17 +431,27 @@ class RpmRootfs(Rootfs): bb.note('incremental removed: %s' % ' '.join(pkg_to_remove)) self.pm.remove(pkg_to_remove) + self.pm.autoremove() + def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() + rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') + rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') # update PM index files self.pm.write_index() - self.pm.dump_all_available_pkgs() + execute_pre_post_process(self.d, rpm_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() if self.inc_rpm_image_gen == "1": self._create_incremental(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.update() pkgs = [] @@ -314,23 +462,39 @@ class RpmRootfs(Rootfs): else: pkgs += pkgs_to_install[pkg_type] + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs_attempt, True) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() - self._log_check() + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) + + execute_pre_post_process(self.d, rpm_post_process_cmds) if self.inc_rpm_image_gen == "1": self.pm.backup_packaging_data() - self.pm.rpm_setup_smart_target_config() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['DEPLOY_DIR_RPM', 'INC_RPM_IMAGE_GEN', 'RPM_PREPROCESS_COMMANDS', - 'RPM_POSTPROCESS_COMMANDS', 'RPM_PREFER_COLOR'] + 'RPM_POSTPROCESS_COMMANDS', 'RPM_PREFER_ELF_ARCH'] def _get_delayed_postinsts(self): postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/rpm-postinsts") @@ -348,27 +512,8 @@ class RpmRootfs(Rootfs): pass def _log_check(self): - r = re.compile('(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)') - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - found_error = 0 - message = "\n" - for line in log.read().split('\n'): - if 'log_check' in line: - continue - - m = r.search(line) - if m: - found_error = 1 - bb.warn('log_check: There were error messages in the logfile') - bb.warn('log_check: Matched keyword: [%s]\n\n' % m.group()) - - if found_error >= 1 and found_error <= 5: - message += line + '\n' - found_error += 1 - - if found_error == 6: - bb.fatal(message) + self._log_check_warn() + self._log_check_error() def _handle_intercept_failure(self, registered_pkgs): rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') @@ -379,28 +524,124 @@ class RpmRootfs(Rootfs): self.pm.save_rpmpostinst(pkg) def _cleanup(self): - # during the execution of postprocess commands, rpm is called several - # times to get the files installed, dependencies, etc. This creates the - # __db.00* (Berkeley DB files that hold locks, rpm specific environment - # settings, etc.), that should not get into the final rootfs - self.pm.unlock_rpm_db() - bb.utils.remove(self.image_rootfs + "/install", True) + pass + +class DpkgOpkgRootfs(Rootfs): + def __init__(self, d, progress_reporter=None, logcatcher=None): + super(DpkgOpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + + def _get_pkgs_postinsts(self, status_file): + def _get_pkg_depends_list(pkg_depends): + pkg_depends_list = [] + # filter version requirements like libc (>= 1.1) + for dep in pkg_depends.split(', '): + m_dep = re.match("^(.*) \(.*\)$", dep) + if m_dep: + dep = m_dep.group(1) + pkg_depends_list.append(dep) + + return pkg_depends_list + + pkgs = {} + pkg_name = "" + pkg_status_match = False + pkg_depends = "" + + with open(status_file) as status: + data = status.read() + status.close() + for line in data.split('\n'): + m_pkg = re.match("^Package: (.*)", line) + m_status = re.match("^Status:.*unpacked", line) + m_depends = re.match("^Depends: (.*)", line) + + if m_pkg is not None: + if pkg_name and pkg_status_match: + pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) + + pkg_name = m_pkg.group(1) + pkg_status_match = False + pkg_depends = "" + elif m_status is not None: + pkg_status_match = True + elif m_depends is not None: + pkg_depends = m_depends.group(1) + + # remove package dependencies not in postinsts + pkg_names = list(pkgs.keys()) + for pkg_name in pkg_names: + deps = pkgs[pkg_name][:] + + for d in deps: + if d not in pkg_names: + pkgs[pkg_name].remove(d) + + return pkgs + def _get_delayed_postinsts_common(self, status_file): + def _dep_resolve(graph, node, resolved, seen): + seen.append(node) -class DpkgRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(DpkgRootfs, self).__init__(d) + for edge in graph[node]: + if edge not in resolved: + if edge in seen: + raise RuntimeError("Packages %s and %s have " \ + "a circular dependency in postinsts scripts." \ + % (node, edge)) + _dep_resolve(graph, edge, resolved, seen) + + resolved.append(node) + + pkg_list = [] + + pkgs = None + if not self.d.getVar('PACKAGE_INSTALL').strip(): + bb.note("Building empty image") + else: + pkgs = self._get_pkgs_postinsts(status_file) + if pkgs: + root = "__packagegroup_postinst__" + pkgs[root] = list(pkgs.keys()) + _dep_resolve(pkgs, root, pkg_list, []) + pkg_list.remove(root) + + if len(pkg_list) == 0: + return None + + return pkg_list + + def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir): + num = 0 + for p in self._get_delayed_postinsts(): + bb.utils.mkdirhier(dst_postinst_dir) + + if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): + shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), + os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) + + num += 1 + +class DpkgRootfs(DpkgOpkgRootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(DpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '^E:' + self.log_check_expected_regexes = \ + [ + "^E: Unmet dependencies." + ] bb.utils.remove(self.image_rootfs, True) - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) self.manifest = DpkgManifest(d, manifest_dir) - self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS', True), - d.getVar('PACKAGE_ARCHS', True), - d.getVar('DPKG_ARCH', True)) + self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS'), + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH')) def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() + deb_pre_process_cmds = self.d.getVar('DEB_PREPROCESS_COMMANDS') + deb_post_process_cmds = self.d.getVar('DEB_POSTPROCESS_COMMANDS') alt_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/alternatives") bb.utils.mkdirhier(alt_dir) @@ -408,74 +649,80 @@ class DpkgRootfs(Rootfs): # update PM index files self.pm.write_index() + execute_pre_post_process(self.d, deb_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + # Don't support incremental, so skip that + self.progress_reporter.next_stage() + self.pm.update() + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + # Don't support attemptonly, so skip that + self.progress_reporter.next_stage() + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/var/lib/dpkg']) + self.pm.fix_broken_dependencies() self.pm.mark_packages("installed") self.pm.run_pre_post_installs() + execute_pre_post_process(self.d, deb_post_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): - return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMAND'] + return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS'] def _get_delayed_postinsts(self): - pkg_list = [] - with open(self.image_rootfs + "/var/lib/dpkg/status") as status: - for line in status: - m_pkg = re.match("^Package: (.*)", line) - m_status = re.match("^Status:.*unpacked", line) - if m_pkg is not None: - pkg_name = m_pkg.group(1) - elif m_status is not None: - pkg_list.append(pkg_name) - - if len(pkg_list) == 0: - return None - - return pkg_list + status_file = self.image_rootfs + "/var/lib/dpkg/status" + return self._get_delayed_postinsts_common(status_file) def _save_postinsts(self): - num = 0 - for p in self._get_delayed_postinsts(): - dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/deb-postinsts") - src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/info") - - bb.utils.mkdirhier(dst_postinst_dir) - - if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): - shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) - - num += 1 + dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/deb-postinsts") + src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/info") + return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) def _handle_intercept_failure(self, registered_pkgs): self.pm.mark_packages("unpacked", registered_pkgs.split()) def _log_check(self): - pass + self._log_check_warn() + self._log_check_error() def _cleanup(self): pass -class OpkgRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(OpkgRootfs, self).__init__(d) +class OpkgRootfs(DpkgOpkgRootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(OpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '(exit 1|Collected errors)' self.manifest = OpkgManifest(d, manifest_dir) - self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True) - self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True) + self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") + self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS") - self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or "" + self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN') or "" if self._remove_old_rootfs(): bb.utils.remove(self.image_rootfs, True) self.pm = OpkgPM(d, @@ -489,7 +736,7 @@ class OpkgRootfs(Rootfs): self.pkg_archs) self.pm.recover_packaging_data() - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) def _prelink_file(self, root_dir, filename): bb.note('prelink %s in %s' % (filename, root_dir)) @@ -544,7 +791,7 @@ class OpkgRootfs(Rootfs): """ def _multilib_sanity_test(self, dirs): - allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP", True) + allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") if allow_replace is None: allow_replace = "" @@ -576,12 +823,12 @@ class OpkgRootfs(Rootfs): files[key] = item def _multilib_test_install(self, pkgs): - ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS", True) + ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") bb.utils.mkdirhier(ml_temp) dirs = [self.image_rootfs] - for variant in self.d.getVar("MULTILIB_VARIANTS", True).split(): + for variant in self.d.getVar("MULTILIB_VARIANTS").split(): ml_target_rootfs = os.path.join(ml_temp, variant) bb.utils.remove(ml_target_rootfs, True) @@ -641,9 +888,9 @@ class OpkgRootfs(Rootfs): old_vars_list = open(vars_list_file, 'r+').read() new_vars_list = '%s:%s:%s\n' % \ - ((self.d.getVar('BAD_RECOMMENDATIONS', True) or '').strip(), - (self.d.getVar('NO_RECOMMENDATIONS', True) or '').strip(), - (self.d.getVar('PACKAGE_EXCLUDE', True) or '').strip()) + ((self.d.getVar('BAD_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('NO_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('PACKAGE_EXCLUDE') or '').strip()) open(vars_list_file, 'w+').write(new_vars_list) if old_vars_list != new_vars_list: @@ -653,23 +900,33 @@ class OpkgRootfs(Rootfs): def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() - opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True) - opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS', True) - rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND', True) + opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') + opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') # update PM index files, unless users provide their own feeds - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": self.pm.write_index() execute_pre_post_process(self.d, opkg_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Steps are a bit different in order, skip next + self.progress_reporter.next_stage() + self.pm.update() self.pm.handle_bad_recommendations() + if self.progress_reporter: + self.progress_reporter.next_stage() + if self.inc_opkg_image_gen == "1": self._remove_extra_packages(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: # For multilib, we perform a sanity test before final install @@ -681,60 +938,50 @@ class OpkgRootfs(Rootfs): self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') + self._setup_dbg_rootfs([opkg_dir]) + execute_pre_post_process(self.d, opkg_post_process_cmds) - execute_pre_post_process(self.d, rootfs_post_install_cmds) if self.inc_opkg_image_gen == "1": self.pm.backup_packaging_data() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] def _get_delayed_postinsts(self): - pkg_list = [] status_file = os.path.join(self.image_rootfs, - self.d.getVar('OPKGLIBDIR', True).strip('/'), + self.d.getVar('OPKGLIBDIR').strip('/'), "opkg", "status") - - with open(status_file) as status: - for line in status: - m_pkg = re.match("^Package: (.*)", line) - m_status = re.match("^Status:.*unpacked", line) - if m_pkg is not None: - pkg_name = m_pkg.group(1) - elif m_status is not None: - pkg_list.append(pkg_name) - - if len(pkg_list) == 0: - return None - - return pkg_list + return self._get_delayed_postinsts_common(status_file) def _save_postinsts(self): - num = 0 - for p in self._get_delayed_postinsts(): - dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") - src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") - - bb.utils.mkdirhier(dst_postinst_dir) - - if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): - shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) - - num += 1 + dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") + src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") + return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) def _handle_intercept_failure(self, registered_pkgs): self.pm.mark_packages("unpacked", registered_pkgs.split()) def _log_check(self): - pass + self._log_check_warn() + self._log_check_error() def _cleanup(self): - pass + self.pm.remove_lists() def get_class_for_type(imgtype): return {"rpm": RpmRootfs, @@ -742,36 +989,36 @@ def get_class_for_type(imgtype): "deb": DpkgRootfs}[imgtype] def variable_depends(d, manifest_dir=None): - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') cls = get_class_for_type(img_type) return cls._depends_list() -def create_rootfs(d, manifest_dir=None): +def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): env_bkp = os.environ.copy() - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": - RpmRootfs(d, manifest_dir).create() + RpmRootfs(d, manifest_dir, progress_reporter, logcatcher).create() elif img_type == "ipk": - OpkgRootfs(d, manifest_dir).create() + OpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() elif img_type == "deb": - DpkgRootfs(d, manifest_dir).create() + DpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() os.environ.clear() os.environ.update(env_bkp) -def image_list_installed_packages(d, format=None, rootfs_dir=None): +def image_list_installed_packages(d, rootfs_dir=None): if not rootfs_dir: - rootfs_dir = d.getVar('IMAGE_ROOTFS', True) + rootfs_dir = d.getVar('IMAGE_ROOTFS') - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": - return RpmPkgsList(d, rootfs_dir).list(format) + return RpmPkgsList(d, rootfs_dir).list_pkgs() elif img_type == "ipk": - return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET", True)).list(format) + return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET")).list_pkgs() elif img_type == "deb": - return DpkgPkgsList(d, rootfs_dir).list(format) + return DpkgPkgsList(d, rootfs_dir).list_pkgs() if __name__ == "__main__": """ diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py index ca349c433c..deb823b6ec 100644 --- a/meta/lib/oe/sdk.py +++ b/meta/lib/oe/sdk.py @@ -5,27 +5,26 @@ from oe.package_manager import * import os import shutil import glob +import traceback -class Sdk(object): - __metaclass__ = ABCMeta - +class Sdk(object, metaclass=ABCMeta): def __init__(self, d, manifest_dir): self.d = d - self.sdk_output = self.d.getVar('SDK_OUTPUT', True) - self.sdk_native_path = self.d.getVar('SDKPATHNATIVE', True).strip('/') - self.target_path = self.d.getVar('SDKTARGETSYSROOT', True).strip('/') - self.sysconfdir = self.d.getVar('sysconfdir', True).strip('/') + self.sdk_output = self.d.getVar('SDK_OUTPUT') + self.sdk_native_path = self.d.getVar('SDKPATHNATIVE').strip('/') + self.target_path = self.d.getVar('SDKTARGETSYSROOT').strip('/') + self.sysconfdir = self.d.getVar('sysconfdir').strip('/') self.sdk_target_sysroot = os.path.join(self.sdk_output, self.target_path) self.sdk_host_sysroot = self.sdk_output if manifest_dir is None: - self.manifest_dir = self.d.getVar("SDK_DIR", True) + self.manifest_dir = self.d.getVar("SDK_DIR") else: self.manifest_dir = manifest_dir - bb.utils.remove(self.sdk_output, True) + self.remove(self.sdk_output, True) self.install_order = Manifest.INSTALL_ORDER @@ -34,29 +33,56 @@ class Sdk(object): pass def populate(self): - bb.utils.mkdirhier(self.sdk_output) + self.mkdirhier(self.sdk_output) # call backend dependent implementation self._populate() # Don't ship any libGL in the SDK - bb.utils.remove(os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('libdir_nativesdk', True).strip('/'), - "libGL*")) + self.remove(os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('libdir_nativesdk').strip('/'), + "libGL*")) # Fix or remove broken .la files - bb.utils.remove(os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('libdir_nativesdk', True).strip('/'), - "*.la")) + self.remove(os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('libdir_nativesdk').strip('/'), + "*.la")) # Link the ld.so.cache file into the hosts filesystem link_name = os.path.join(self.sdk_output, self.sdk_native_path, self.sysconfdir, "ld.so.cache") - bb.utils.mkdirhier(os.path.dirname(link_name)) + self.mkdirhier(os.path.dirname(link_name)) os.symlink("/etc/ld.so.cache", link_name) - execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND', True)) - + execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND')) + + def movefile(self, sourcefile, destdir): + try: + # FIXME: this check of movefile's return code to None should be + # fixed within the function to use only exceptions to signal when + # something goes wrong + if (bb.utils.movefile(sourcefile, destdir) == None): + raise OSError("moving %s to %s failed" + %(sourcefile, destdir)) + #FIXME: using umbrella exc catching because bb.utils method raises it + except Exception as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.error("unable to place %s in final SDK location" % sourcefile) + + def mkdirhier(self, dirpath): + try: + bb.utils.mkdirhier(dirpath) + except OSError as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.fatal("cannot make dir for SDK: %s" % dirpath) + + def remove(self, path, recurse=False): + try: + bb.utils.remove(path, recurse) + #FIXME: using umbrella exc catching because bb.utils method raises it + except Exception as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.warn("cannot remove SDK dir: %s" % path) class RpmSdk(Sdk): def __init__(self, d, manifest_dir=None): @@ -76,7 +102,7 @@ class RpmSdk(Sdk): self.target_pm = RpmPM(d, self.sdk_target_sysroot, - self.d.getVar('TARGET_VENDOR', True), + self.d.getVar('TARGET_VENDOR'), 'target', target_providename ) @@ -92,7 +118,7 @@ class RpmSdk(Sdk): self.host_pm = RpmPM(d, self.sdk_host_sysroot, - self.d.getVar('SDK_VENDOR', True), + self.d.getVar('SDK_VENDOR'), 'host', sdk_providename, "SDK_PACKAGE_ARCHS", @@ -104,47 +130,55 @@ class RpmSdk(Sdk): pm.create_configs() pm.write_index() - pm.dump_all_available_pkgs() pm.update() - for pkg_type in self.install_order: - if pkg_type in pkgs_to_install: - pm.install(pkgs_to_install[pkg_type], - [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + pkgs = [] + pkgs_attempt = [] + for pkg_type in pkgs_to_install: + if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY: + pkgs_attempt += pkgs_to_install[pkg_type] + else: + pkgs += pkgs_to_install[pkg_type] + + pm.install(pkgs) + + pm.install(pkgs_attempt, True) def _populate(self): bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) - self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY', True)) + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) - self.target_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) - self.host_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() # Move host RPM library data native_rpm_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('localstatedir_nativesdk', True).strip('/'), + self.d.getVar('localstatedir_nativesdk').strip('/'), "lib", "rpm" ) - bb.utils.mkdirhier(native_rpm_state_dir) + self.mkdirhier(native_rpm_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "rpm", "*")): - bb.utils.movefile(f, native_rpm_state_dir) + self.movefile(f, native_rpm_state_dir) - bb.utils.remove(os.path.join(self.sdk_output, "var"), True) + self.remove(os.path.join(self.sdk_output, "var"), True) # Move host sysconfig data native_sysconf_dir = os.path.join(self.sdk_output, @@ -152,18 +186,20 @@ class RpmSdk(Sdk): self.d.getVar('sysconfdir', True).strip('/'), ) - bb.utils.mkdirhier(native_sysconf_dir) - for f in glob.glob(os.path.join(self.sdk_output, "etc", "*")): - bb.utils.movefile(f, native_sysconf_dir) - bb.utils.remove(os.path.join(self.sdk_output, "etc"), True) + self.mkdirhier(native_sysconf_dir) + for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")): + self.movefile(f, native_sysconf_dir) + for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", "*")): + self.movefile(f, native_sysconf_dir) + self.remove(os.path.join(self.sdk_output, "etc"), True) class OpkgSdk(Sdk): def __init__(self, d, manifest_dir=None): super(OpkgSdk, self).__init__(d, manifest_dir) - self.target_conf = self.d.getVar("IPKGCONF_TARGET", True) - self.host_conf = self.d.getVar("IPKGCONF_SDK", True) + self.target_conf = self.d.getVar("IPKGCONF_TARGET") + self.host_conf = self.d.getVar("IPKGCONF_SDK") self.target_manifest = OpkgManifest(d, self.manifest_dir, Manifest.MANIFEST_TYPE_SDK_TARGET) @@ -171,15 +207,15 @@ class OpkgSdk(Sdk): Manifest.MANIFEST_TYPE_SDK_HOST) self.target_pm = OpkgPM(d, self.sdk_target_sysroot, self.target_conf, - self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)) + self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) self.host_pm = OpkgPM(d, self.sdk_host_sysroot, self.host_conf, - self.d.getVar("SDK_PACKAGE_ARCHS", True)) + self.d.getVar("SDK_PACKAGE_ARCHS")) def _populate_sysroot(self, pm, manifest): pkgs_to_install = manifest.parse_initial_manifest() - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": pm.write_index() pm.update() @@ -193,44 +229,50 @@ class OpkgSdk(Sdk): bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) - self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY', True)) + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) - bb.utils.mkdirhier(target_sysconfdir) + self.mkdirhier(target_sysconfdir) shutil.copy(self.target_conf, target_sysconfdir) os.chmod(os.path.join(target_sysconfdir, - os.path.basename(self.target_conf)), 0644) + os.path.basename(self.target_conf)), 0o644) - bb.utils.mkdirhier(host_sysconfdir) + self.mkdirhier(host_sysconfdir) shutil.copy(self.host_conf, host_sysconfdir) os.chmod(os.path.join(host_sysconfdir, - os.path.basename(self.host_conf)), 0644) + os.path.basename(self.host_conf)), 0o644) native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('localstatedir_nativesdk', True).strip('/'), + self.d.getVar('localstatedir_nativesdk').strip('/'), "lib", "opkg") - bb.utils.mkdirhier(native_opkg_state_dir) + self.mkdirhier(native_opkg_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "opkg", "*")): - bb.utils.movefile(f, native_opkg_state_dir) + self.movefile(f, native_opkg_state_dir) - bb.utils.remove(os.path.join(self.sdk_output, "var"), True) + self.remove(os.path.join(self.sdk_output, "var"), True) class DpkgSdk(Sdk): def __init__(self, d, manifest_dir=None): super(DpkgSdk, self).__init__(d, manifest_dir) - self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET", True), "apt") - self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET", True), "apt-sdk") + self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt") + self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt-sdk") self.target_manifest = DpkgManifest(d, self.manifest_dir, Manifest.MANIFEST_TYPE_SDK_TARGET) @@ -238,19 +280,19 @@ class DpkgSdk(Sdk): Manifest.MANIFEST_TYPE_SDK_HOST) self.target_pm = DpkgPM(d, self.sdk_target_sysroot, - self.d.getVar("PACKAGE_ARCHS", True), - self.d.getVar("DPKG_ARCH", True), + self.d.getVar("PACKAGE_ARCHS"), + self.d.getVar("DPKG_ARCH"), self.target_conf_dir) self.host_pm = DpkgPM(d, self.sdk_host_sysroot, - self.d.getVar("SDK_PACKAGE_ARCHS", True), - self.d.getVar("DEB_SDK_ARCH", True), + self.d.getVar("SDK_PACKAGE_ARCHS"), + self.d.getVar("DEB_SDK_ARCH"), self.host_conf_dir) def _copy_apt_dir_to(self, dst_dir): - staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE", True) + staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE") - bb.utils.remove(dst_dir, True) + self.remove(dst_dir, True) shutil.copytree(os.path.join(staging_etcdir_native, "apt"), dst_dir) @@ -269,49 +311,57 @@ class DpkgSdk(Sdk): bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) self._copy_apt_dir_to(os.path.join(self.sdk_target_sysroot, "etc", "apt")) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() + bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) self._copy_apt_dir_to(os.path.join(self.sdk_output, self.sdk_native_path, "etc", "apt")) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() + native_dpkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, "var", "lib", "dpkg") - bb.utils.mkdirhier(native_dpkg_state_dir) + self.mkdirhier(native_dpkg_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "dpkg", "*")): - bb.utils.movefile(f, native_dpkg_state_dir) + self.movefile(f, native_dpkg_state_dir) + self.remove(os.path.join(self.sdk_output, "var"), True) - bb.utils.remove(os.path.join(self.sdk_output, "var"), True) -def sdk_list_installed_packages(d, target, format=None, rootfs_dir=None): +def sdk_list_installed_packages(d, target, rootfs_dir=None): if rootfs_dir is None: - sdk_output = d.getVar('SDK_OUTPUT', True) - target_path = d.getVar('SDKTARGETSYSROOT', True).strip('/') + sdk_output = d.getVar('SDK_OUTPUT') + target_path = d.getVar('SDKTARGETSYSROOT').strip('/') rootfs_dir = [sdk_output, os.path.join(sdk_output, target_path)][target is True] - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": arch_var = ["SDK_PACKAGE_ARCHS", None][target is True] os_var = ["SDK_OS", None][target is True] - return RpmPkgsList(d, rootfs_dir, arch_var, os_var).list(format) + return RpmPkgsList(d, rootfs_dir).list_pkgs() elif img_type == "ipk": - conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_Target"][target is True] - return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var, True)).list(format) + conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True] + return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs() elif img_type == "deb": - return DpkgPkgsList(d, rootfs_dir).list(format) + return DpkgPkgsList(d, rootfs_dir).list_pkgs() def populate_sdk(d, manifest_dir=None): env_bkp = os.environ.copy() - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": RpmSdk(d, manifest_dir).populate() elif img_type == "ipk": diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index d58147f78f..f087a019e1 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py @@ -14,6 +14,9 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): def isPackageGroup(fn): inherits = " ".join(dataCache.inherits[fn]) return "/packagegroup.bbclass" in inherits + def isAllArch(fn): + inherits = " ".join(dataCache.inherits[fn]) + return "/allarch.bbclass" in inherits def isImage(fn): return "/image.bbclass" in " ".join(dataCache.inherits[fn]) @@ -36,8 +39,8 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): # Only target packages beyond here - # packagegroups are assumed to have well behaved names which don't change between architecures/tunes - if isPackageGroup(fn): + # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes + if isPackageGroup(fn) and isAllArch(fn) and not isNative(depname): return False # Exclude well defined machine specific configurations which don't change ABI @@ -58,11 +61,24 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): # Default to keep dependencies return True +def sstate_lockedsigs(d): + sigs = {} + types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split() + for t in types: + siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t + lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split() + for ls in lockedsigs: + pn, task, h = ls.split(":", 2) + if pn not in sigs: + sigs[pn] = {} + sigs[pn][task] = [h, siggen_lockedsigs_var] + return sigs + class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): name = "OEBasic" def init_rundepcheck(self, data): - self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split() - self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split() + self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() pass def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) @@ -70,12 +86,173 @@ class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): name = "OEBasicHash" def init_rundepcheck(self, data): - self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split() - self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split() + self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() + self.lockedsigs = sstate_lockedsigs(data) + self.lockedhashes = {} + self.lockedpnmap = {} + self.lockedhashfn = {} + self.machine = data.getVar("MACHINE") + self.mismatch_msgs = [] + self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or + "").split() + self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } pass + + def tasks_resolved(self, virtmap, virtpnmap, dataCache): + # Translate virtual/xxx entries to PN values + newabisafe = [] + for a in self.abisaferecipes: + if a in virtpnmap: + newabisafe.append(virtpnmap[a]) + else: + newabisafe.append(a) + self.abisaferecipes = newabisafe + newsafedeps = [] + for a in self.saferecipedeps: + a1, a2 = a.split("->") + if a1 in virtpnmap: + a1 = virtpnmap[a1] + if a2 in virtpnmap: + a2 = virtpnmap[a2] + newsafedeps.append(a1 + "->" + a2) + self.saferecipedeps = newsafedeps + def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) + def get_taskdata(self): + data = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskdata() + return (data, self.lockedpnmap, self.lockedhashfn) + + def set_taskdata(self, data): + coredata, self.lockedpnmap, self.lockedhashfn = data + super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata) + + def dump_sigs(self, dataCache, options): + sigfile = os.getcwd() + "/locked-sigs.inc" + bb.plain("Writing locked sigs to %s" % sigfile) + self.dump_lockedsigs(sigfile) + return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) + + def get_taskhash(self, fn, task, deps, dataCache): + h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache) + + recipename = dataCache.pkg_fn[fn] + self.lockedpnmap[fn] = recipename + self.lockedhashfn[fn] = dataCache.hashfn[fn] + + unlocked = False + if recipename in self.unlockedrecipes: + unlocked = True + else: + def recipename_from_dep(dep): + # The dep entry will look something like + # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task, + # ... + fn = dep.rsplit('.', 1)[0] + return dataCache.pkg_fn[fn] + + # If any unlocked recipe is in the direct dependencies then the + # current recipe should be unlocked as well. + depnames = [ recipename_from_dep(x) for x in deps ] + if any(x in y for y in depnames for x in self.unlockedrecipes): + self.unlockedrecipes[recipename] = '' + unlocked = True + + if not unlocked and recipename in self.lockedsigs: + if task in self.lockedsigs[recipename]: + k = fn + "." + task + h_locked = self.lockedsigs[recipename][task][0] + var = self.lockedsigs[recipename][task][1] + self.lockedhashes[k] = h_locked + self.taskhash[k] = h_locked + #bb.warn("Using %s %s %s" % (recipename, task, h)) + + if h != h_locked: + self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' + % (recipename, task, h, h_locked, var)) + + return h_locked + #bb.warn("%s %s %s" % (recipename, task, h)) + return h + + def dump_sigtask(self, fn, task, stampbase, runtime): + k = fn + "." + task + if k in self.lockedhashes: + return + super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) + + def dump_lockedsigs(self, sigfile, taskfilter=None): + types = {} + for k in self.runtaskdeps: + if taskfilter: + if not k in taskfilter: + continue + fn = k.rsplit(".",1)[0] + t = self.lockedhashfn[fn].split(" ")[1].split(":")[5] + t = 't-' + t.replace('_', '-') + if t not in types: + types[t] = [] + types[t].append(k) + + with open(sigfile, "w") as f: + l = sorted(types) + for t in l: + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) + types[t].sort() + sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]]) + for k in sortedk: + fn = k.rsplit(".",1)[0] + task = k.rsplit(".",1)[1] + if k not in self.taskhash: + continue + f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n") + f.write(' "\n') + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(l))) + + def dump_siglist(self, sigfile): + with open(sigfile, "w") as f: + tasks = [] + for taskitem in self.taskhash: + (fn, task) = taskitem.rsplit(".", 1) + pn = self.lockedpnmap[fn] + tasks.append((pn, task, fn, self.taskhash[taskitem])) + for (pn, task, fn, taskhash) in sorted(tasks): + f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash)) + + def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): + warn_msgs = [] + error_msgs = [] + sstate_missing_msgs = [] + + for task in range(len(sq_fn)): + if task not in ret: + for pn in self.lockedsigs: + if sq_hash[task] in iter(self.lockedsigs[pn].values()): + if sq_task[task] == 'do_shared_workdir': + continue + sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" + % (pn, sq_task[task], sq_hash[task])) + + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") + if checklevel == 'warn': + warn_msgs += self.mismatch_msgs + elif checklevel == 'error': + error_msgs += self.mismatch_msgs + + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK") + if checklevel == 'warn': + warn_msgs += sstate_missing_msgs + elif checklevel == 'error': + error_msgs += sstate_missing_msgs + + if warn_msgs: + bb.warn("\n".join(warn_msgs)) + if error_msgs: + bb.fatal("\n".join(error_msgs)) + + # Insert these classes into siggen's namespace so it can see and select them bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash @@ -87,9 +264,6 @@ def find_siginfo(pn, taskname, taskhashlist, d): import fnmatch import glob - if taskhashlist: - hashfiles = {} - if not taskname: # We have to derive pn and taskname key = pn @@ -99,11 +273,15 @@ def find_siginfo(pn, taskname, taskhashlist, d): if key.startswith('virtual:native:'): pn = pn + '-native' - if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic']: - pn.replace("-native", "") - + hashfiles = {} filedates = {} + def get_hashval(siginfo): + if siginfo.endswith('.siginfo'): + return siginfo.rpartition(':')[2].partition('_')[0] + else: + return siginfo.rpartition('.')[2] + # First search in stamps dir localdata = d.createCopy() localdata.setVar('MULTIMACH_TARGET_SYS', '*') @@ -111,7 +289,11 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('PV', '*') localdata.setVar('PR', '*') localdata.setVar('EXTENDPE', '') - stamp = localdata.getVar('STAMP', True) + stamp = localdata.getVar('STAMP') + if pn.startswith("gcc-source"): + # gcc-source shared workdir is a special case :( + stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") + filespec = '%s.%s.sigdata.*' % (stamp, taskname) foundall = False import glob @@ -129,6 +311,8 @@ def find_siginfo(pn, taskname, taskhashlist, d): filedates[fullpath] = os.stat(fullpath).st_mtime except OSError: continue + hashval = get_hashval(fullpath) + hashfiles[hashval] = fullpath if not taskhashlist or (len(filedates) < 2 and not foundall): # That didn't work, look in sstate-cache @@ -142,27 +326,25 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('PV', '*') localdata.setVar('PR', '*') localdata.setVar('BB_TASKHASH', hashval) - if pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: + swspec = localdata.getVar('SSTATE_SWSPEC') + if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec: + localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}') + elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") sstatename = taskname[3:] - filespec = '%s_%s.*.siginfo' % (localdata.getVar('SSTATE_PKG', True), sstatename) + filespec = '%s_%s.*.siginfo' % (localdata.getVar('SSTATE_PKG'), sstatename) - if hashval != '*': - sstatedir = "%s/%s" % (d.getVar('SSTATE_DIR', True), hashval[:2]) - else: - sstatedir = d.getVar('SSTATE_DIR', True) - - for root, dirs, files in os.walk(sstatedir): - for fn in files: - fullpath = os.path.join(root, fn) - if fnmatch.fnmatch(fullpath, filespec): - if taskhashlist: - hashfiles[hashval] = fullpath - else: - try: - filedates[fullpath] = os.stat(fullpath).st_mtime - except: - continue + matchedfiles = glob.glob(filespec) + for fullpath in matchedfiles: + actual_hashval = get_hashval(fullpath) + if actual_hashval in hashfiles: + continue + hashfiles[hashval] = fullpath + if not taskhashlist: + try: + filedates[fullpath] = os.stat(fullpath).st_mtime + except: + continue if taskhashlist: return hashfiles @@ -170,3 +352,15 @@ def find_siginfo(pn, taskname, taskhashlist, d): return filedates bb.siggen.find_siginfo = find_siginfo + + +def sstate_get_manifest_filename(task, d): + """ + Return the sstate manifest file path for a particular task. + Also returns the datastore that can be used to query related variables. + """ + d2 = d.createCopy() + extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info') + if extrainf: + d2.setVar("SSTATE_MANMACH", extrainf) + return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2) diff --git a/meta/lib/oe/terminal.py b/meta/lib/oe/terminal.py index 0a623c75b1..0426e15834 100644 --- a/meta/lib/oe/terminal.py +++ b/meta/lib/oe/terminal.py @@ -2,6 +2,7 @@ import logging import oe.classutils import shlex from bb.process import Popen, ExecutionError +from distutils.version import LooseVersion logger = logging.getLogger('BitBake.OE.Terminal') @@ -10,7 +11,8 @@ class UnsupportedTerminal(Exception): pass class NoSupportedTerminals(Exception): - pass + def __init__(self, terms): + self.terms = terms class Registry(oe.classutils.ClassRegistry): @@ -24,9 +26,7 @@ class Registry(oe.classutils.ClassRegistry): return bool(cls.command) -class Terminal(Popen): - __metaclass__ = Registry - +class Terminal(Popen, metaclass=Registry): def __init__(self, sh_cmd, title=None, env=None, d=None): fmt_sh_cmd = self.format_command(sh_cmd, title) try: @@ -40,7 +40,7 @@ class Terminal(Popen): def format_command(self, sh_cmd, title): fmt = {'title': title or 'Terminal', 'command': sh_cmd} - if isinstance(self.command, basestring): + if isinstance(self.command, str): return shlex.split(self.command.format(**fmt)) else: return [element.format(**fmt) for element in self.command] @@ -55,6 +55,36 @@ class Gnome(XTerminal): command = 'gnome-terminal -t "{title}" -x {command}' priority = 2 + def __init__(self, sh_cmd, title=None, env=None, d=None): + # Recent versions of gnome-terminal does not support non-UTF8 charset: + # https://bugzilla.gnome.org/show_bug.cgi?id=732127; as a workaround, + # clearing the LC_ALL environment variable so it uses the locale. + # Once fixed on the gnome-terminal project, this should be removed. + if os.getenv('LC_ALL'): os.putenv('LC_ALL','') + + # We need to know when the command completes but gnome-terminal gives us no way + # to do this. We therefore write the pid to a file using a "phonehome" wrapper + # script, then monitor the pid until it exits. Thanks gnome! + import tempfile + pidfile = tempfile.NamedTemporaryFile(delete = False).name + try: + sh_cmd = "oe-gnome-terminal-phonehome " + pidfile + " " + sh_cmd + XTerminal.__init__(self, sh_cmd, title, env, d) + while os.stat(pidfile).st_size <= 0: + continue + with open(pidfile, "r") as f: + pid = int(f.readline()) + finally: + os.unlink(pidfile) + + import time + while True: + try: + os.kill(pid, 0) + time.sleep(0.1) + except OSError: + return + class Mate(XTerminal): command = 'mate-terminal -t "{title}" -x {command}' priority = 2 @@ -63,17 +93,23 @@ class Xfce(XTerminal): command = 'xfce4-terminal -T "{title}" -e "{command}"' priority = 2 +class Terminology(XTerminal): + command = 'terminology -T="{title}" -e {command}' + priority = 2 + class Konsole(XTerminal): - command = 'konsole -T "{title}" -e {command}' + command = 'konsole --separate --workdir . -p tabtitle="{title}" -e {command}' priority = 2 def __init__(self, sh_cmd, title=None, env=None, d=None): # Check version - vernum = check_konsole_version("konsole") - if vernum: - if vernum.split('.')[0] == "2": - logger.debug(1, 'Konsole from KDE 4.x will not work as devshell, skipping') - raise UnsupportedTerminal(self.name) + vernum = check_terminal_version("konsole") + if vernum and LooseVersion(vernum) < '2.0.0': + # Konsole from KDE 3.x + self.command = 'konsole -T "{title}" -e {command}' + elif vernum and LooseVersion(vernum) < '16.08.1': + # Konsole pre 16.08.01 Has nofork + self.command = 'konsole --nofork --workdir . -p tabtitle="{title}" -e {command}' XTerminal.__init__(self, sh_cmd, title, env, d) class XTerm(XTerminal): @@ -112,6 +148,24 @@ class TmuxRunning(Terminal): if not os.getenv('TMUX'): raise UnsupportedTerminal('tmux is not running') + if not check_tmux_pane_size('tmux'): + raise UnsupportedTerminal('tmux pane too small or tmux < 1.9 version is being used') + + Terminal.__init__(self, sh_cmd, title, env, d) + +class TmuxNewWindow(Terminal): + """Open a new window in the current running tmux session""" + name = 'tmux-new-window' + command = 'tmux new-window -n "{title}" "{command}"' + priority = 2.70 + + def __init__(self, sh_cmd, title=None, env=None, d=None): + if not bb.utils.which(os.getenv('PATH'), 'tmux'): + raise UnsupportedTerminal('tmux is not installed') + + if not os.getenv('TMUX'): + raise UnsupportedTerminal('tmux is not running') + Terminal.__init__(self, sh_cmd, title, env, d) class Tmux(Terminal): @@ -142,7 +196,7 @@ class Custom(Terminal): priority = 3 def __init__(self, sh_cmd, title=None, env=None, d=None): - self.command = d and d.getVar('OE_TERMINAL_CUSTOMCMD', True) + self.command = d and d.getVar('OE_TERMINAL_CUSTOMCMD') if self.command: if not '{command}' in self.command: self.command += ' {command}' @@ -156,6 +210,14 @@ class Custom(Terminal): def prioritized(): return Registry.prioritized() +def get_cmd_list(): + terms = Registry.prioritized() + cmds = [] + for term in terms: + if term.command: + cmds.append(term.command) + return cmds + def spawn_preferred(sh_cmd, title=None, env=None, d=None): """Spawn the first supported terminal, by priority""" for terminal in prioritized(): @@ -165,7 +227,7 @@ def spawn_preferred(sh_cmd, title=None, env=None, d=None): except UnsupportedTerminal: continue else: - raise NoSupportedTerminals() + raise NoSupportedTerminals(get_cmd_list()) def spawn(name, sh_cmd, title=None, env=None, d=None): """Spawn the specified terminal, by name""" @@ -177,15 +239,44 @@ def spawn(name, sh_cmd, title=None, env=None, d=None): pipe = terminal(sh_cmd, title, env, d) output = pipe.communicate()[0] + if output: + output = output.decode("utf-8") if pipe.returncode != 0: raise ExecutionError(sh_cmd, pipe.returncode, output) -def check_konsole_version(konsole): +def check_tmux_pane_size(tmux): + import subprocess as sub + # On older tmux versions (<1.9), return false. The reason + # is that there is no easy way to get the height of the active panel + # on current window without nested formats (available from version 1.9) + vernum = check_terminal_version("tmux") + if vernum and LooseVersion(vernum) < '1.9': + return False + try: + p = sub.Popen('%s list-panes -F "#{?pane_active,#{pane_height},}"' % tmux, + shell=True,stdout=sub.PIPE,stderr=sub.PIPE) + out, err = p.communicate() + size = int(out.strip()) + except OSError as exc: + import errno + if exc.errno == errno.ENOENT: + return None + else: + raise + + return size/2 >= 19 + +def check_terminal_version(terminalName): import subprocess as sub try: - p = sub.Popen(['sh', '-c', '%s --version' % konsole],stdout=sub.PIPE,stderr=sub.PIPE) + cmdversion = '%s --version' % terminalName + if terminalName.startswith('tmux'): + cmdversion = '%s -V' % terminalName + newenv = os.environ.copy() + newenv["LANG"] = "C" + p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv) out, err = p.communicate() - ver_info = out.rstrip().split('\n') + ver_info = out.decode().rstrip().split('\n') except OSError as exc: import errno if exc.errno == errno.ENOENT: @@ -196,6 +287,10 @@ def check_konsole_version(konsole): for ver in ver_info: if ver.startswith('Konsole'): vernum = ver.split(' ')[-1] + if ver.startswith('GNOME Terminal'): + vernum = ver.split(' ')[-1] + if ver.startswith('tmux'): + vernum = ver.split()[-1] return vernum def distro_name(): diff --git a/meta/lib/oe/types.py b/meta/lib/oe/types.py index 7f47c17d0e..4ae58acfac 100644 --- a/meta/lib/oe/types.py +++ b/meta/lib/oe/types.py @@ -33,7 +33,7 @@ def choice(value, choices): Acts as a multiple choice for the user. To use this, set the variable type flag to 'choice', and set the 'choices' flag to a space separated list of valid values.""" - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("choice accepts a string, not '%s'" % type(value)) value = value.lower() @@ -106,7 +106,7 @@ def boolean(value): Valid values for false: 'no', 'n', 'false', 'f', '0' """ - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("boolean accepts a string, not '%s'" % type(value)) value = value.lower() diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 0a1d1080c9..330a5ff94a 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -1,9 +1,4 @@ -try: - # Python 2 - import commands as cmdstatus -except ImportError: - # Python 3 - import subprocess as cmdstatus +import subprocess def read_file(filename): try: @@ -23,30 +18,53 @@ def ifelse(condition, iftrue = True, iffalse = False): return iffalse def conditional(variable, checkvalue, truevalue, falsevalue, d): - if d.getVar(variable,1) == checkvalue: + if d.getVar(variable) == checkvalue: return truevalue else: return falsevalue def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - if float(d.getVar(variable,1)) <= float(checkvalue): + if float(d.getVar(variable)) <= float(checkvalue): return truevalue else: return falsevalue def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - result = bb.utils.vercmp_string(d.getVar(variable,True), checkvalue) + result = bb.utils.vercmp_string(d.getVar(variable), checkvalue) if result <= 0: return truevalue else: return falsevalue def both_contain(variable1, variable2, checkvalue, d): - if d.getVar(variable1,1).find(checkvalue) != -1 and d.getVar(variable2,1).find(checkvalue) != -1: - return checkvalue + val1 = d.getVar(variable1) + val2 = d.getVar(variable2) + val1 = set(val1.split()) + val2 = set(val2.split()) + if isinstance(checkvalue, str): + checkvalue = set(checkvalue.split()) + else: + checkvalue = set(checkvalue) + if checkvalue.issubset(val1) and checkvalue.issubset(val2): + return " ".join(checkvalue) else: return "" +def set_intersect(variable1, variable2, d): + """ + Expand both variables, interpret them as lists of strings, and return the + intersection as a flattened string. + + For example: + s1 = "a b c" + s2 = "b c d" + s3 = set_intersect(s1, s2) + => s3 = "b c" + """ + val1 = set(d.getVar(variable1).split()) + val2 = set(d.getVar(variable2).split()) + return " ".join(val1 & val2) + def prune_suffix(var, suffixes, d): # See if var ends with any of the suffixes listed and # remove it if found @@ -54,7 +72,7 @@ def prune_suffix(var, suffixes, d): if var.endswith(suffix): var = var.replace(suffix, "") - prefix = d.getVar("MLPREFIX", True) + prefix = d.getVar("MLPREFIX") if prefix and var.startswith(prefix): var = var.replace(prefix, "") @@ -62,11 +80,11 @@ def prune_suffix(var, suffixes, d): def str_filter(f, str, d): from re import match - return " ".join(filter(lambda x: match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if match(f, x, 0)]) def str_filter_out(f, str, d): from re import match - return " ".join(filter(lambda x: not match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if not match(f, x, 0)]) def param_bool(cfg, field, dflt = None): """Lookup <field> in <cfg> map and convert it to a boolean; take @@ -79,6 +97,10 @@ def param_bool(cfg, field, dflt = None): return False raise ValueError("invalid value for boolean parameter '%s': '%s'" % (field, value)) +def build_depends_string(depends, task): + """Append a taskname to a string of dependencies as used by the [depends] flag""" + return " ".join(dep + ":" + task for dep in depends.split()) + def inherits(d, *classes): """Return True if the metadata inherits any of the specified classes""" return any(bb.data.inherits_class(cls, d) for cls in classes) @@ -92,9 +114,9 @@ def features_backfill(var,d): # disturbing distributions that have already set DISTRO_FEATURES. # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED - features = (d.getVar(var, True) or "").split() - backfill = (d.getVar(var+"_BACKFILL", True) or "").split() - considered = (d.getVar(var+"_BACKFILL_CONSIDERED", True) or "").split() + features = (d.getVar(var) or "").split() + backfill = (d.getVar(var+"_BACKFILL") or "").split() + considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split() addfeatures = [] for feature in backfill: @@ -110,18 +132,18 @@ def packages_filter_out_system(d): Return a list of packages from PACKAGES with the "system" packages such as PN-dbg PN-doc PN-locale-eb-gb removed. """ - pn = d.getVar('PN', True) - blacklist = map(lambda suffix: pn + suffix, ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')) + pn = d.getVar('PN') + blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] localepkg = pn + "-locale-" pkgs = [] - for pkg in d.getVar('PACKAGES', True).split(): + for pkg in d.getVar('PACKAGES').split(): if pkg not in blacklist and localepkg not in pkg: pkgs.append(pkg) return pkgs def getstatusoutput(cmd): - return cmdstatus.getstatusoutput(cmd) + return subprocess.getstatusoutput(cmd) def trim_version(version, num_parts=2): @@ -151,3 +173,164 @@ def execute_pre_post_process(d, cmds): if cmd != '': bb.note("Executing %s ..." % cmd) bb.build.exec_func(cmd, d) + +def multiprocess_exec(commands, function): + import signal + import multiprocessing + + if not commands: + return [] + + def init_worker(): + signal.signal(signal.SIGINT, signal.SIG_IGN) + + nproc = min(multiprocessing.cpu_count(), len(commands)) + pool = bb.utils.multiprocessingpool(nproc, init_worker) + imap = pool.imap(function, commands) + + try: + res = list(imap) + pool.close() + pool.join() + results = [] + for result in res: + if result is not None: + results.append(result) + return results + + except KeyboardInterrupt: + pool.terminate() + pool.join() + raise + +def squashspaces(string): + import re + return re.sub("\s+", " ", string).strip() + +def format_pkg_list(pkg_dict, ret_format=None): + output = [] + + if ret_format == "arch": + for pkg in sorted(pkg_dict): + output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"])) + elif ret_format == "file": + for pkg in sorted(pkg_dict): + output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"])) + elif ret_format == "ver": + for pkg in sorted(pkg_dict): + output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"])) + elif ret_format == "deps": + for pkg in sorted(pkg_dict): + for dep in pkg_dict[pkg]["deps"]: + output.append("%s|%s" % (pkg, dep)) + else: + for pkg in sorted(pkg_dict): + output.append(pkg) + + return '\n'.join(output) + +def host_gcc_version(d): + import re, subprocess + + compiler = d.getVar("BUILD_CC") + try: + env = os.environ.copy() + env["PATH"] = d.getVar("PATH") + output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) + + match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + if not match: + bb.fatal("Can't get compiler version from %s --version output" % compiler) + + version = match.group(1) + return "-%s" % version if version in ("4.8", "4.9") else "" + +# +# Python 2.7 doesn't have threaded pools (just multiprocessing) +# so implement a version here +# + +from queue import Queue +from threading import Thread + +class ThreadedWorker(Thread): + """Thread executing tasks from a given tasks queue""" + def __init__(self, tasks, worker_init, worker_end): + Thread.__init__(self) + self.tasks = tasks + self.daemon = True + + self.worker_init = worker_init + self.worker_end = worker_end + + def run(self): + from queue import Empty + + if self.worker_init is not None: + self.worker_init(self) + + while True: + try: + func, args, kargs = self.tasks.get(block=False) + except Empty: + if self.worker_end is not None: + self.worker_end(self) + break + + try: + func(self, *args, **kargs) + except Exception as e: + print(e) + finally: + self.tasks.task_done() + +class ThreadedPool: + """Pool of threads consuming tasks from a queue""" + def __init__(self, num_workers, num_tasks, worker_init=None, + worker_end=None): + self.tasks = Queue(num_tasks) + self.workers = [] + + for _ in range(num_workers): + worker = ThreadedWorker(self.tasks, worker_init, worker_end) + self.workers.append(worker) + + def start(self): + for worker in self.workers: + worker.start() + + def add_task(self, func, *args, **kargs): + """Add a task to the queue""" + self.tasks.put((func, args, kargs)) + + def wait_completion(self): + """Wait for completion of all the tasks in the queue""" + self.tasks.join() + for worker in self.workers: + worker.join() + +def write_ld_so_conf(d): + # Some utils like prelink may not have the correct target library paths + # so write an ld.so.conf to help them + ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf") + if os.path.exists(ldsoconf): + bb.utils.remove(ldsoconf) + bb.utils.mkdirhier(os.path.dirname(ldsoconf)) + with open(ldsoconf, "w") as f: + f.write(d.getVar("base_libdir") + '\n') + f.write(d.getVar("libdir") + '\n') + +class ImageQAFailed(bb.build.FuncFailed): + def __init__(self, description, name=None, logfile=None): + self.description = description + self.name = name + self.logfile=logfile + + def __str__(self): + msg = 'Function failed: %s' % self.name + if self.description: + msg = msg + ' (%s)' % self.description + + return msg diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py new file mode 100644 index 0000000000..605f429ecc --- /dev/null +++ b/meta/lib/oeqa/buildperf/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance tests""" +from .base import (BuildPerfTestCase, + BuildPerfTestLoader, + BuildPerfTestResult, + BuildPerfTestRunner, + KernelDropCaches, + runCmd2) +from .test_basic import * diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py new file mode 100644 index 0000000000..6e62b279c1 --- /dev/null +++ b/meta/lib/oeqa/buildperf/base.py @@ -0,0 +1,510 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance test base classes and functionality""" +import json +import logging +import os +import re +import resource +import socket +import shutil +import time +import unittest +import xml.etree.ElementTree as ET +from collections import OrderedDict +from datetime import datetime, timedelta +from functools import partial +from multiprocessing import Process +from multiprocessing import SimpleQueue +from xml.dom import minidom + +import oe.path +from oeqa.utils.commands import CommandError, runCmd, get_bb_vars +from oeqa.utils.git import GitError, GitRepo + +# Get logger for this module +log = logging.getLogger('build-perf') + +# Our own version of runCmd which does not raise AssertErrors which would cause +# errors to interpreted as failures +runCmd2 = partial(runCmd, assert_error=False, limit_exc_output=40) + + +class KernelDropCaches(object): + """Container of the functions for dropping kernel caches""" + sudo_passwd = None + + @classmethod + def check(cls): + """Check permssions for dropping kernel caches""" + from getpass import getpass + from locale import getdefaultlocale + cmd = ['sudo', '-k', '-n', 'tee', '/proc/sys/vm/drop_caches'] + ret = runCmd2(cmd, ignore_status=True, data=b'0') + if ret.output.startswith('sudo:'): + pass_str = getpass( + "\nThe script requires sudo access to drop caches between " + "builds (echo 3 > /proc/sys/vm/drop_caches).\n" + "Please enter your sudo password: ") + cls.sudo_passwd = bytes(pass_str, getdefaultlocale()[1]) + + @classmethod + def drop(cls): + """Drop kernel caches""" + cmd = ['sudo', '-k'] + if cls.sudo_passwd: + cmd.append('-S') + input_data = cls.sudo_passwd + b'\n' + else: + cmd.append('-n') + input_data = b'' + cmd += ['tee', '/proc/sys/vm/drop_caches'] + input_data += b'3' + runCmd2(cmd, data=input_data) + + +def str_to_fn(string): + """Convert string to a sanitized filename""" + return re.sub(r'(\W+)', '-', string, flags=re.LOCALE) + + +class ResultsJsonEncoder(json.JSONEncoder): + """Extended encoder for build perf test results""" + unix_epoch = datetime.utcfromtimestamp(0) + + def default(self, obj): + """Encoder for our types""" + if isinstance(obj, datetime): + # NOTE: we assume that all timestamps are in UTC time + return (obj - self.unix_epoch).total_seconds() + if isinstance(obj, timedelta): + return obj.total_seconds() + return json.JSONEncoder.default(self, obj) + + +class BuildPerfTestResult(unittest.TextTestResult): + """Runner class for executing the individual tests""" + # List of test cases to run + test_run_queue = [] + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestResult, self).__init__(*args, **kwargs) + + self.out_dir = out_dir + self.hostname = socket.gethostname() + self.product = os.getenv('OE_BUILDPERFTEST_PRODUCT', 'oe-core') + self.start_time = self.elapsed_time = None + self.successes = [] + + def addSuccess(self, test): + """Record results from successful tests""" + super(BuildPerfTestResult, self).addSuccess(test) + self.successes.append(test) + + def addError(self, test, err): + """Record results from crashed test""" + test.err = err + super(BuildPerfTestResult, self).addError(test, err) + + def addFailure(self, test, err): + """Record results from failed test""" + test.err = err + super(BuildPerfTestResult, self).addFailure(test, err) + + def addExpectedFailure(self, test, err): + """Record results from expectedly failed test""" + test.err = err + super(BuildPerfTestResult, self).addExpectedFailure(test, err) + + def startTest(self, test): + """Pre-test hook""" + test.base_dir = self.out_dir + log.info("Executing test %s: %s", test.name, test.shortDescription()) + self.stream.write(datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ")) + super(BuildPerfTestResult, self).startTest(test) + + def startTestRun(self): + """Pre-run hook""" + self.start_time = datetime.utcnow() + + def stopTestRun(self): + """Pre-run hook""" + self.elapsed_time = datetime.utcnow() - self.start_time + + def all_results(self): + compound = [('SUCCESS', t, None) for t in self.successes] + \ + [('FAILURE', t, m) for t, m in self.failures] + \ + [('ERROR', t, m) for t, m in self.errors] + \ + [('EXPECTED_FAILURE', t, m) for t, m in self.expectedFailures] + \ + [('UNEXPECTED_SUCCESS', t, None) for t in self.unexpectedSuccesses] + \ + [('SKIPPED', t, m) for t, m in self.skipped] + return sorted(compound, key=lambda info: info[1].start_time) + + + def write_buildstats_json(self): + """Write buildstats file""" + buildstats = OrderedDict() + for _, test, _ in self.all_results(): + for key, val in test.buildstats.items(): + buildstats[test.name + '.' + key] = val + with open(os.path.join(self.out_dir, 'buildstats.json'), 'w') as fobj: + json.dump(buildstats, fobj, cls=ResultsJsonEncoder) + + + def write_results_json(self): + """Write test results into a json-formatted file""" + results = OrderedDict([('tester_host', self.hostname), + ('start_time', self.start_time), + ('elapsed_time', self.elapsed_time), + ('tests', OrderedDict())]) + + for status, test, reason in self.all_results(): + test_result = OrderedDict([('name', test.name), + ('description', test.shortDescription()), + ('status', status), + ('start_time', test.start_time), + ('elapsed_time', test.elapsed_time), + ('measurements', test.measurements)]) + if status in ('ERROR', 'FAILURE', 'EXPECTED_FAILURE'): + test_result['message'] = str(test.err[1]) + test_result['err_type'] = test.err[0].__name__ + test_result['err_output'] = reason + elif reason: + test_result['message'] = reason + + results['tests'][test.name] = test_result + + with open(os.path.join(self.out_dir, 'results.json'), 'w') as fobj: + json.dump(results, fobj, indent=4, + cls=ResultsJsonEncoder) + + def write_results_xml(self): + """Write test results into a JUnit XML file""" + top = ET.Element('testsuites') + suite = ET.SubElement(top, 'testsuite') + suite.set('name', 'oeqa.buildperf') + suite.set('timestamp', self.start_time.isoformat()) + suite.set('time', str(self.elapsed_time.total_seconds())) + suite.set('hostname', self.hostname) + suite.set('failures', str(len(self.failures) + len(self.expectedFailures))) + suite.set('errors', str(len(self.errors))) + suite.set('skipped', str(len(self.skipped))) + + test_cnt = 0 + for status, test, reason in self.all_results(): + test_cnt += 1 + testcase = ET.SubElement(suite, 'testcase') + testcase.set('classname', test.__module__ + '.' + test.__class__.__name__) + testcase.set('name', test.name) + testcase.set('description', test.shortDescription()) + testcase.set('timestamp', test.start_time.isoformat()) + testcase.set('time', str(test.elapsed_time.total_seconds())) + if status in ('ERROR', 'FAILURE', 'EXP_FAILURE'): + if status in ('FAILURE', 'EXP_FAILURE'): + result = ET.SubElement(testcase, 'failure') + else: + result = ET.SubElement(testcase, 'error') + result.set('message', str(test.err[1])) + result.set('type', test.err[0].__name__) + result.text = reason + elif status == 'SKIPPED': + result = ET.SubElement(testcase, 'skipped') + result.text = reason + elif status not in ('SUCCESS', 'UNEXPECTED_SUCCESS'): + raise TypeError("BUG: invalid test status '%s'" % status) + + for data in test.measurements.values(): + measurement = ET.SubElement(testcase, data['type']) + measurement.set('name', data['name']) + measurement.set('legend', data['legend']) + vals = data['values'] + if data['type'] == BuildPerfTestCase.SYSRES: + ET.SubElement(measurement, 'time', + timestamp=vals['start_time'].isoformat()).text = \ + str(vals['elapsed_time'].total_seconds()) + attrib = dict((k, str(v)) for k, v in vals['iostat'].items()) + ET.SubElement(measurement, 'iostat', attrib=attrib) + attrib = dict((k, str(v)) for k, v in vals['rusage'].items()) + ET.SubElement(measurement, 'rusage', attrib=attrib) + elif data['type'] == BuildPerfTestCase.DISKUSAGE: + ET.SubElement(measurement, 'size').text = str(vals['size']) + else: + raise TypeError('BUG: unsupported measurement type') + + suite.set('tests', str(test_cnt)) + + # Use minidom for pretty-printing + dom_doc = minidom.parseString(ET.tostring(top, 'utf-8')) + with open(os.path.join(self.out_dir, 'results.xml'), 'w') as fobj: + dom_doc.writexml(fobj, addindent=' ', newl='\n', encoding='utf-8') + + +class BuildPerfTestCase(unittest.TestCase): + """Base class for build performance tests""" + SYSRES = 'sysres' + DISKUSAGE = 'diskusage' + build_target = None + + def __init__(self, *args, **kwargs): + super(BuildPerfTestCase, self).__init__(*args, **kwargs) + self.name = self._testMethodName + self.base_dir = None + self.start_time = None + self.elapsed_time = None + self.measurements = OrderedDict() + self.buildstats = OrderedDict() + # self.err is supposed to be a tuple from sys.exc_info() + self.err = None + self.bb_vars = get_bb_vars() + # TODO: remove 'times' and 'sizes' arrays when globalres support is + # removed + self.times = [] + self.sizes = [] + + @property + def tmp_dir(self): + return os.path.join(self.base_dir, self.name + '.tmp') + + def shortDescription(self): + return super(BuildPerfTestCase, self).shortDescription() or "" + + def setUp(self): + """Set-up fixture for each test""" + if not os.path.isdir(self.tmp_dir): + os.mkdir(self.tmp_dir) + if self.build_target: + self.run_cmd(['bitbake', self.build_target, '-c', 'fetchall']) + + def tearDown(self): + """Tear-down fixture for each test""" + if os.path.isdir(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + + def run(self, *args, **kwargs): + """Run test""" + self.start_time = datetime.now() + super(BuildPerfTestCase, self).run(*args, **kwargs) + self.elapsed_time = datetime.now() - self.start_time + + def run_cmd(self, cmd): + """Convenience method for running a command""" + cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) + log.info("Logging command: %s", cmd_str) + try: + runCmd2(cmd) + except CommandError as err: + log.error("Command failed: %s", err.retcode) + raise + + def _append_measurement(self, measurement): + """Simple helper for adding measurements results""" + if measurement['name'] in self.measurements: + raise ValueError('BUG: two measurements with the same name in {}'.format( + self.__class__.__name__)) + self.measurements[measurement['name']] = measurement + + def measure_cmd_resources(self, cmd, name, legend, save_bs=False): + """Measure system resource usage of a command""" + def _worker(data_q, cmd, **kwargs): + """Worker process for measuring resources""" + try: + start_time = datetime.now() + ret = runCmd2(cmd, **kwargs) + etime = datetime.now() - start_time + rusage_struct = resource.getrusage(resource.RUSAGE_CHILDREN) + iostat = OrderedDict() + with open('/proc/{}/io'.format(os.getpid())) as fobj: + for line in fobj.readlines(): + key, val = line.split(':') + iostat[key] = int(val) + rusage = OrderedDict() + # Skip unused fields, (i.e. 'ru_ixrss', 'ru_idrss', 'ru_isrss', + # 'ru_nswap', 'ru_msgsnd', 'ru_msgrcv' and 'ru_nsignals') + for key in ['ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', + 'ru_majflt', 'ru_inblock', 'ru_oublock', + 'ru_nvcsw', 'ru_nivcsw']: + rusage[key] = getattr(rusage_struct, key) + data_q.put({'ret': ret, + 'start_time': start_time, + 'elapsed_time': etime, + 'rusage': rusage, + 'iostat': iostat}) + except Exception as err: + data_q.put(err) + + cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) + log.info("Timing command: %s", cmd_str) + data_q = SimpleQueue() + try: + proc = Process(target=_worker, args=(data_q, cmd,)) + proc.start() + data = data_q.get() + proc.join() + if isinstance(data, Exception): + raise data + except CommandError: + log.error("Command '%s' failed", cmd_str) + raise + etime = data['elapsed_time'] + + measurement = OrderedDict([('type', self.SYSRES), + ('name', name), + ('legend', legend)]) + measurement['values'] = OrderedDict([('start_time', data['start_time']), + ('elapsed_time', etime), + ('rusage', data['rusage']), + ('iostat', data['iostat'])]) + if save_bs: + self.save_buildstats(name) + + self._append_measurement(measurement) + + # Append to 'times' array for globalres log + e_sec = etime.total_seconds() + self.times.append('{:d}:{:02d}:{:05.2f}'.format(int(e_sec / 3600), + int((e_sec % 3600) / 60), + e_sec % 60)) + + def measure_disk_usage(self, path, name, legend, apparent_size=False): + """Estimate disk usage of a file or directory""" + cmd = ['du', '-s', '--block-size', '1024'] + if apparent_size: + cmd.append('--apparent-size') + cmd.append(path) + + ret = runCmd2(cmd) + size = int(ret.output.split()[0]) + log.debug("Size of %s path is %s", path, size) + measurement = OrderedDict([('type', self.DISKUSAGE), + ('name', name), + ('legend', legend)]) + measurement['values'] = OrderedDict([('size', size)]) + self._append_measurement(measurement) + # Append to 'sizes' array for globalres log + self.sizes.append(str(size)) + + def save_buildstats(self, measurement_name): + """Save buildstats""" + def split_nevr(nevr): + """Split name and version information from recipe "nevr" string""" + n_e_v, revision = nevr.rsplit('-', 1) + match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$', + n_e_v) + if not match: + # If we're not able to parse a version starting with a number, just + # take the part after last dash + match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$', + n_e_v) + name = match.group('name') + version = match.group('version') + epoch = match.group('epoch') + return name, epoch, version, revision + + def bs_to_json(filename): + """Convert (task) buildstats file into json format""" + bs_json = OrderedDict() + iostat = OrderedDict() + rusage = OrderedDict() + with open(filename) as fobj: + for line in fobj.readlines(): + key, val = line.split(':', 1) + val = val.strip() + if key == 'Started': + start_time = datetime.utcfromtimestamp(float(val)) + bs_json['start_time'] = start_time + elif key == 'Ended': + end_time = datetime.utcfromtimestamp(float(val)) + elif key.startswith('IO '): + split = key.split() + iostat[split[1]] = int(val) + elif key.find('rusage') >= 0: + split = key.split() + ru_key = split[-1] + if ru_key in ('ru_stime', 'ru_utime'): + val = float(val) + else: + val = int(val) + rusage[ru_key] = rusage.get(ru_key, 0) + val + elif key == 'Status': + bs_json['status'] = val + bs_json['elapsed_time'] = end_time - start_time + bs_json['rusage'] = rusage + bs_json['iostat'] = iostat + return bs_json + + log.info('Saving buildstats in JSON format') + bs_dirs = sorted(os.listdir(self.bb_vars['BUILDSTATS_BASE'])) + if len(bs_dirs) > 1: + log.warning("Multiple buildstats found for test %s, only " + "archiving the last one", self.name) + bs_dir = os.path.join(self.bb_vars['BUILDSTATS_BASE'], bs_dirs[-1]) + + buildstats = [] + for fname in os.listdir(bs_dir): + recipe_dir = os.path.join(bs_dir, fname) + if not os.path.isdir(recipe_dir): + continue + name, epoch, version, revision = split_nevr(fname) + recipe_bs = OrderedDict((('name', name), + ('epoch', epoch), + ('version', version), + ('revision', revision), + ('tasks', OrderedDict()))) + for task in os.listdir(recipe_dir): + recipe_bs['tasks'][task] = bs_to_json(os.path.join(recipe_dir, + task)) + buildstats.append(recipe_bs) + + self.buildstats[measurement_name] = buildstats + + def rm_tmp(self): + """Cleanup temporary/intermediate files and directories""" + log.debug("Removing temporary and cache files") + for name in ['bitbake.lock', 'conf/sanity_info', + self.bb_vars['TMPDIR']]: + oe.path.remove(name, recurse=True) + + def rm_sstate(self): + """Remove sstate directory""" + log.debug("Removing sstate-cache") + oe.path.remove(self.bb_vars['SSTATE_DIR'], recurse=True) + + def rm_cache(self): + """Drop bitbake caches""" + oe.path.remove(self.bb_vars['PERSISTENT_DIR'], recurse=True) + + @staticmethod + def sync(): + """Sync and drop kernel caches""" + log.debug("Syncing and dropping kernel caches""") + KernelDropCaches.drop() + os.sync() + # Wait a bit for all the dirty blocks to be written onto disk + time.sleep(3) + + +class BuildPerfTestLoader(unittest.TestLoader): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + +class BuildPerfTestRunner(unittest.TextTestRunner): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestRunner, self).__init__(*args, **kwargs) + self.out_dir = out_dir + + def _makeResult(self): + return BuildPerfTestResult(self.out_dir, self.stream, self.descriptions, + self.verbosity) diff --git a/meta/lib/oeqa/buildperf/test_basic.py b/meta/lib/oeqa/buildperf/test_basic.py new file mode 100644 index 0000000000..a9e4a5b731 --- /dev/null +++ b/meta/lib/oeqa/buildperf/test_basic.py @@ -0,0 +1,125 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Basic set of build performance tests""" +import os +import shutil + +import oe.path +from oeqa.buildperf import BuildPerfTestCase +from oeqa.utils.commands import get_bb_vars + + +class Test1P1(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test1(self): + """Build core-image-sato""" + self.rm_tmp() + self.rm_sstate() + self.rm_cache() + self.sync() + self.measure_cmd_resources(['bitbake', self.build_target], 'build', + 'bitbake ' + self.build_target, save_bs=True) + self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir') + + +class Test1P2(BuildPerfTestCase): + build_target = 'virtual/kernel' + + def test12(self): + """Build virtual/kernel""" + # Build and cleans state in order to get all dependencies pre-built + self.run_cmd(['bitbake', self.build_target]) + self.run_cmd(['bitbake', self.build_target, '-c', 'cleansstate']) + + self.sync() + self.measure_cmd_resources(['bitbake', self.build_target], 'build', + 'bitbake ' + self.build_target) + + +class Test1P3(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test13(self): + """Build core-image-sato with rm_work enabled""" + postfile = os.path.join(self.tmp_dir, 'postfile.conf') + with open(postfile, 'w') as fobj: + fobj.write('INHERIT += "rm_work"\n') + + self.rm_tmp() + self.rm_sstate() + self.rm_cache() + self.sync() + cmd = ['bitbake', '-R', postfile, self.build_target] + self.measure_cmd_resources(cmd, 'build', + 'bitbake' + self.build_target, + save_bs=True) + self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir') + + +class Test2(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test2(self): + """Run core-image-sato do_rootfs with sstate""" + # Build once in order to populate sstate cache + self.run_cmd(['bitbake', self.build_target]) + + self.rm_tmp() + self.rm_cache() + self.sync() + cmd = ['bitbake', self.build_target, '-c', 'rootfs'] + self.measure_cmd_resources(cmd, 'do_rootfs', 'bitbake do_rootfs') + + +class Test3(BuildPerfTestCase): + + def test3(self): + """Bitbake parsing (bitbake -p)""" + # Drop all caches and parse + self.rm_cache() + oe.path.remove(os.path.join(self.bb_vars['TMPDIR'], 'cache'), True) + self.measure_cmd_resources(['bitbake', '-p'], 'parse_1', + 'bitbake -p (no caches)') + # Drop tmp/cache + oe.path.remove(os.path.join(self.bb_vars['TMPDIR'], 'cache'), True) + self.measure_cmd_resources(['bitbake', '-p'], 'parse_2', + 'bitbake -p (no tmp/cache)') + # Parse with fully cached data + self.measure_cmd_resources(['bitbake', '-p'], 'parse_3', + 'bitbake -p (cached)') + + +class Test4(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test4(self): + """eSDK metrics""" + self.run_cmd(['bitbake', '-c', 'do_populate_sdk_ext', + self.build_target]) + self.bb_vars = get_bb_vars(None, self.build_target) + tmp_dir = self.bb_vars['TMPDIR'] + installer = os.path.join( + self.bb_vars['SDK_DEPLOY'], + self.bb_vars['TOOLCHAINEXT_OUTPUTNAME'] + '.sh') + # Measure installer size + self.measure_disk_usage(installer, 'installer_bin', 'eSDK installer', + apparent_size=True) + # Measure deployment time and deployed size + deploy_dir = os.path.join(tmp_dir, 'esdk-deploy') + if os.path.exists(deploy_dir): + shutil.rmtree(deploy_dir) + self.sync() + self.measure_cmd_resources([installer, '-y', '-d', deploy_dir], + 'deploy', 'eSDK deploy') + self.measure_disk_usage(deploy_dir, 'deploy_dir', 'deploy dir', + apparent_size=True) diff --git a/meta/lib/oeqa/controllers/masterimage.py b/meta/lib/oeqa/controllers/masterimage.py index 311f0cf68c..07418fcda1 100644 --- a/meta/lib/oeqa/controllers/masterimage.py +++ b/meta/lib/oeqa/controllers/masterimage.py @@ -24,9 +24,7 @@ from oeqa.utils import CommandError from abc import ABCMeta, abstractmethod -class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): - - __metaclass__ = ABCMeta +class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta): supported_image_fstypes = ['tar.gz', 'tar.bz2'] @@ -34,14 +32,14 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): super(MasterImageHardwareTarget, self).__init__(d) # target ip - addr = d.getVar("TEST_TARGET_IP", True) or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') + addr = d.getVar("TEST_TARGET_IP") or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') self.ip = addr.split(":")[0] try: self.port = addr.split(":")[1] except IndexError: self.port = None bb.note("Target IP: %s" % self.ip) - self.server_ip = d.getVar("TEST_SERVER_IP", True) + self.server_ip = d.getVar("TEST_SERVER_IP") if not self.server_ip: try: self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split("\n")[0].split()[-1] @@ -51,8 +49,8 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): # test rootfs + kernel self.image_fstype = self.get_image_fstype(d) - self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) - self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("KERNEL_IMAGETYPE") + '-' + d.getVar('MACHINE') + '.bin') + self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("IMAGE_LINK_NAME") + '.' + self.image_fstype) + self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') if not os.path.isfile(self.rootfs): # we could've checked that IMAGE_FSTYPES contains tar.gz but the config for running testimage might not be # the same as the config with which the image was build, ie @@ -66,17 +64,17 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): # master ssh connection self.master = None # if the user knows what they are doing, then by all means... - self.user_cmds = d.getVar("TEST_DEPLOY_CMDS", True) + self.user_cmds = d.getVar("TEST_DEPLOY_CMDS") self.deploy_cmds = None # this is the name of the command that controls the power for a board # e.g: TEST_POWERCONTROL_CMD = "/home/user/myscripts/powercontrol.py ${MACHINE} what-ever-other-args-the-script-wants" # the command should take as the last argument "off" and "on" and "cycle" (off, on) - self.powercontrol_cmd = d.getVar("TEST_POWERCONTROL_CMD", True) or None - self.powercontrol_args = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS") or "" + self.powercontrol_cmd = d.getVar("TEST_POWERCONTROL_CMD") or None + self.powercontrol_args = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS", False) or "" - self.serialcontrol_cmd = d.getVar("TEST_SERIALCONTROL_CMD", True) or None - self.serialcontrol_args = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS") or "" + self.serialcontrol_cmd = d.getVar("TEST_SERIALCONTROL_CMD") or None + self.serialcontrol_args = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS", False) or "" self.origenv = os.environ if self.powercontrol_cmd or self.serialcontrol_cmd: @@ -84,7 +82,7 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): # ssh + keys means we need the original user env bborigenv = d.getVar("BB_ORIGENV", False) or {} for key in bborigenv: - val = bborigenv.getVar(key, True) + val = bborigenv.getVar(key) if val is not None: self.origenv[key] = str(val) @@ -161,10 +159,50 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): self.power_cycle(self.connection) -class GummibootTarget(MasterImageHardwareTarget): +class SystemdbootTarget(MasterImageHardwareTarget): + + def __init__(self, d): + super(SystemdbootTarget, self).__init__(d) + # this the value we need to set in the LoaderEntryOneShot EFI variable + # so the system boots the 'test' bootloader label and not the default + # The first four bytes are EFI bits, and the rest is an utf-16le string + # (EFI vars values need to be utf-16) + # $ echo -en "test\0" | iconv -f ascii -t utf-16le | hexdump -C + # 00000000 74 00 65 00 73 00 74 00 00 00 |t.e.s.t...| + self.efivarvalue = r'\x07\x00\x00\x00\x74\x00\x65\x00\x73\x00\x74\x00\x00\x00' + self.deploy_cmds = [ + 'mount -L boot /boot', + 'mkdir -p /mnt/testrootfs', + 'mount -L testrootfs /mnt/testrootfs', + 'modprobe efivarfs', + 'mount -t efivarfs efivarfs /sys/firmware/efi/efivars', + 'cp ~/test-kernel /boot', + 'rm -rf /mnt/testrootfs/*', + 'tar xvf ~/test-rootfs.%s -C /mnt/testrootfs' % self.image_fstype, + 'printf "%s" > /sys/firmware/efi/efivars/LoaderEntryOneShot-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f' % self.efivarvalue + ] + + def _deploy(self): + # make sure these aren't mounted + self.master.run("umount /boot; umount /mnt/testrootfs; umount /sys/firmware/efi/efivars;") + # from now on, every deploy cmd should return 0 + # else an exception will be thrown by sshcontrol + self.master.ignore_status = False + self.master.copy_to(self.rootfs, "~/test-rootfs." + self.image_fstype) + self.master.copy_to(self.kernel, "~/test-kernel") + for cmd in self.deploy_cmds: + self.master.run(cmd) + + def _start(self, params=None): + self.power_cycle(self.master) + # there are better ways than a timeout but this should work for now + time.sleep(120) + + +class SystemdbootTarget(MasterImageHardwareTarget): def __init__(self, d): - super(GummibootTarget, self).__init__(d) + super(SystemdbootTarget, self).__init__(d) # this the value we need to set in the LoaderEntryOneShot EFI variable # so the system boots the 'test' bootloader label and not the default # The first four bytes are EFI bits, and the rest is an utf-16le string diff --git a/meta/lib/oeqa/controllers/testtargetloader.py b/meta/lib/oeqa/controllers/testtargetloader.py index a1b7b1d92b..b51d04b213 100644 --- a/meta/lib/oeqa/controllers/testtargetloader.py +++ b/meta/lib/oeqa/controllers/testtargetloader.py @@ -61,8 +61,6 @@ class TestTargetLoader: obj = getattr(module, target) if obj: from oeqa.targetcontrol import BaseTarget - if (not isinstance(obj, (type, types.ClassType))): - bb.warn("Target {0} found, but not of type Class".format(target)) if( not issubclass(obj, BaseTarget)): bb.warn("Target {0} found, but subclass is not BaseTarget".format(target)) except: diff --git a/meta/lib/oeqa/core/README b/meta/lib/oeqa/core/README new file mode 100644 index 0000000000..0c859fd788 --- /dev/null +++ b/meta/lib/oeqa/core/README @@ -0,0 +1,38 @@ += OEQA Framework = + +== Introduction == + +This is the new OEQA framework the base clases of the framework +are in this module oeqa/core the subsequent components needs to +extend this classes. + +A new/unique runner was created called oe-test and is under scripts/ +oe-test, this new runner scans over oeqa module searching for test +components that supports OETestContextExecutor implemented in context +module (i.e. oeqa/core/context.py). + +For execute an example: + +$ source oe-init-build-env +$ oe-test core + +For list supported components: + +$ oe-test -h + +== Create new Test component == + +Usally for add a new Test component the developer needs to extend +OETestContext/OETestContextExecutor in context.py and OETestCase in +case.py. + +== How to run the testing of the OEQA framework == + +Run all tests: + +$ PATH=$PATH:../../ python3 -m unittest discover -s tests + +Run some test: + +$ cd tests/ +$ ./test_data.py diff --git a/meta/lib/oe/tests/__init__.py b/meta/lib/oeqa/core/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/meta/lib/oe/tests/__init__.py +++ b/meta/lib/oeqa/core/__init__.py diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py new file mode 100644 index 0000000000..d2dbf20f9e --- /dev/null +++ b/meta/lib/oeqa/core/case.py @@ -0,0 +1,46 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import unittest + +from oeqa.core.exception import OEQAMissingVariable + +def _validate_td_vars(td, td_vars, type_msg): + if td_vars: + for v in td_vars: + if not v in td: + raise OEQAMissingVariable("Test %s need %s variable but"\ + " isn't into td" % (type_msg, v)) + +class OETestCase(unittest.TestCase): + # TestContext and Logger instance set by OETestLoader. + tc = None + logger = None + + # td has all the variables needed by the test cases + # is the same across all the test cases. + td = None + + # td_vars has the variables needed by a test class + # or test case instance, if some var isn't into td a + # OEMissingVariable exception is raised + td_vars = None + + @classmethod + def _oeSetUpClass(clss): + _validate_td_vars(clss.td, clss.td_vars, "class") + clss.setUpClassMethod() + + @classmethod + def _oeTearDownClass(clss): + clss.tearDownClassMethod() + + def _oeSetUp(self): + for d in self.decorators: + d.setUpDecorator() + self.setUpMethod() + + def _oeTearDown(self): + for d in self.decorators: + d.tearDownDecorator() + self.tearDownMethod() diff --git a/meta/lib/oeqa/__init__.py b/meta/lib/oeqa/core/cases/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/meta/lib/oeqa/__init__.py +++ b/meta/lib/oeqa/core/cases/__init__.py diff --git a/meta/lib/oeqa/core/cases/example/data.json b/meta/lib/oeqa/core/cases/example/data.json new file mode 100644 index 0000000000..21d6b16d17 --- /dev/null +++ b/meta/lib/oeqa/core/cases/example/data.json @@ -0,0 +1 @@ +{"ARCH": "x86", "IMAGE": "core-image-minimal"}
\ No newline at end of file diff --git a/meta/lib/oeqa/core/cases/example/test_basic.py b/meta/lib/oeqa/core/cases/example/test_basic.py new file mode 100644 index 0000000000..11cf3800cc --- /dev/null +++ b/meta/lib/oeqa/core/cases/example/test_basic.py @@ -0,0 +1,20 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.depends import OETestDepends + +class OETestExample(OETestCase): + def test_example(self): + self.logger.info('IMAGE: %s' % self.td.get('IMAGE')) + self.assertEqual('core-image-minimal', self.td.get('IMAGE')) + self.logger.info('ARCH: %s' % self.td.get('ARCH')) + self.assertEqual('x86', self.td.get('ARCH')) + +class OETestExampleDepend(OETestCase): + @OETestDepends(['OETestExample.test_example']) + def test_example_depends(self): + pass + + def test_example_no_depends(self): + pass diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py new file mode 100644 index 0000000000..4476750a3c --- /dev/null +++ b/meta/lib/oeqa/core/context.py @@ -0,0 +1,243 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import json +import time +import logging +import collections +import re + +from oeqa.core.loader import OETestLoader +from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled + +class OETestContext(object): + loaderClass = OETestLoader + runnerClass = OETestRunner + streamLoggerClass = OEStreamLogger + + files_dir = os.path.abspath(os.path.join(os.path.dirname( + os.path.abspath(__file__)), "../files")) + + def __init__(self, td=None, logger=None): + if not type(td) is dict: + raise TypeError("td isn't dictionary type") + + self.td = td + self.logger = logger + self._registry = {} + self._registry['cases'] = collections.OrderedDict() + self._results = {} + + def _read_modules_from_manifest(self, manifest): + if not os.path.exists(manifest): + raise + + modules = [] + for line in open(manifest).readlines(): + line = line.strip() + if line and not line.startswith("#"): + modules.append(line) + + return modules + + def loadTests(self, module_paths, modules=[], tests=[], + modules_manifest="", modules_required=[], filters={}): + if modules_manifest: + modules = self._read_modules_from_manifest(modules_manifest) + + self.loader = self.loaderClass(self, module_paths, modules, tests, + modules_required, filters) + self.suites = self.loader.discover() + + def runTests(self): + streamLogger = self.streamLoggerClass(self.logger) + self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2) + + self._run_start_time = time.time() + result = self.runner.run(self.suites) + self._run_end_time = time.time() + + return result + + def logSummary(self, result, component, context_msg=''): + self.logger.info("SUMMARY:") + self.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component, + context_msg, result.testsRun, result.testsRun != 1 and "s" or "", + (self._run_end_time - self._run_start_time))) + + if result.wasSuccessful(): + msg = "%s - OK - All required tests passed" % component + else: + msg = "%s - FAIL - Required tests failed" % component + skipped = len(self._results['skipped']) + if skipped: + msg += " (skipped=%d)" % skipped + self.logger.info(msg) + + def _getDetailsNotPassed(self, case, type, desc): + found = False + + for (scase, msg) in self._results[type]: + # XXX: When XML reporting is enabled scase is + # xmlrunner.result._TestInfo instance instead of + # string. + if xmlEnabled: + if case.id() == scase.test_id: + found = True + break + scase_str = scase.test_id + else: + if case == scase: + found = True + break + scase_str = str(scase) + + # When fails at module or class level the class name is passed as string + # so figure out to see if match + m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str) + if m: + if case.__class__.__module__ == m.group('module_name'): + found = True + break + + m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str) + if m: + class_name = "%s.%s" % (case.__class__.__module__, + case.__class__.__name__) + + if class_name == m.group('class_name'): + found = True + break + + if found: + return (found, msg) + + return (found, None) + + def logDetails(self): + self.logger.info("RESULTS:") + for case_name in self._registry['cases']: + case = self._registry['cases'][case_name] + + result_types = ['failures', 'errors', 'skipped', 'expectedFailures'] + result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL'] + + fail = False + desc = None + for idx, name in enumerate(result_types): + (fail, msg) = self._getDetailsNotPassed(case, result_types[idx], + result_desc[idx]) + if fail: + desc = result_desc[idx] + break + + oeid = -1 + for d in case.decorators: + if hasattr(d, 'oeid'): + oeid = d.oeid + + if fail: + self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), + oeid, desc)) + if msg: + self.logger.info(msg) + else: + self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), + oeid, 'PASSED')) + +class OETestContextExecutor(object): + _context_class = OETestContext + + name = 'core' + help = 'core test component example' + description = 'executes core test suite example' + + default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'cases/example')] + default_test_data = os.path.join(default_cases[0], 'data.json') + default_tests = None + + def register_commands(self, logger, subparsers): + self.parser = subparsers.add_parser(self.name, help=self.help, + description=self.description, group='components') + + self.default_output_log = '%s-results-%s.log' % (self.name, + time.strftime("%Y%m%d%H%M%S")) + self.parser.add_argument('--output-log', action='store', + default=self.default_output_log, + help="results output log, default: %s" % self.default_output_log) + self.parser.add_argument('--run-tests', action='store', + default=self.default_tests, + help="tests to run in <module>[.<class>[.<name>]] format. Just works for modules now") + + if self.default_test_data: + self.parser.add_argument('--test-data-file', action='store', + default=self.default_test_data, + help="data file to load, default: %s" % self.default_test_data) + else: + self.parser.add_argument('--test-data-file', action='store', + help="data file to load") + + if self.default_cases: + self.parser.add_argument('CASES_PATHS', action='store', + default=self.default_cases, nargs='*', + help="paths to directories with test cases, default: %s"\ + % self.default_cases) + else: + self.parser.add_argument('CASES_PATHS', action='store', + nargs='+', help="paths to directories with test cases") + + self.parser.set_defaults(func=self.run) + + def _setup_logger(self, logger, args): + formatter = logging.Formatter('%(asctime)s - ' + self.name + \ + ' - %(levelname)s - %(message)s') + sh = logger.handlers[0] + sh.setFormatter(formatter) + fh = logging.FileHandler(args.output_log) + fh.setFormatter(formatter) + logger.addHandler(fh) + + return logger + + def _process_args(self, logger, args): + self.tc_kwargs = {} + self.tc_kwargs['init'] = {} + self.tc_kwargs['load'] = {} + self.tc_kwargs['run'] = {} + + self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args) + if args.test_data_file: + self.tc_kwargs['init']['td'] = json.load( + open(args.test_data_file, "r")) + else: + self.tc_kwargs['init']['td'] = {} + + + if args.run_tests: + self.tc_kwargs['load']['modules'] = args.run_tests.split() + else: + self.tc_kwargs['load']['modules'] = None + + self.module_paths = args.CASES_PATHS + + def run(self, logger, args): + self._process_args(logger, args) + + self.tc = self._context_class(**self.tc_kwargs['init']) + self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) + rc = self.tc.runTests(**self.tc_kwargs['run']) + self.tc.logSummary(rc, self.name) + self.tc.logDetails() + + output_link = os.path.join(os.path.dirname(args.output_log), + "%s-results.log" % self.name) + if os.path.exists(output_link): + os.remove(output_link) + os.symlink(args.output_log, output_link) + + return rc + +_executor_class = OETestContextExecutor diff --git a/meta/lib/oeqa/core/decorator/__init__.py b/meta/lib/oeqa/core/decorator/__init__.py new file mode 100644 index 0000000000..855b6b9d28 --- /dev/null +++ b/meta/lib/oeqa/core/decorator/__init__.py @@ -0,0 +1,71 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from functools import wraps +from abc import abstractmethod + +decoratorClasses = set() + +def registerDecorator(obj): + decoratorClasses.add(obj) + return obj + +class OETestDecorator(object): + case = None # Reference of OETestCase decorated + attrs = None # Attributes to be loaded by decorator implementation + + def __init__(self, *args, **kwargs): + if not self.attrs: + return + + for idx, attr in enumerate(self.attrs): + if attr in kwargs: + value = kwargs[attr] + else: + value = args[idx] + setattr(self, attr, value) + + def __call__(self, func): + @wraps(func) + def wrapped_f(*args, **kwargs): + self.attrs = self.attrs # XXX: Enables OETestLoader discover + return func(*args, **kwargs) + return wrapped_f + + # OETestLoader call it when is loading test cases. + # XXX: Most methods would change the registry for later + # processing; be aware that filtrate method needs to + # run later than bind, so there could be data (in the + # registry) of a cases that were filtered. + def bind(self, registry, case): + self.case = case + self.logger = case.tc.logger + self.case.decorators.append(self) + + # OETestRunner call this method when tries to run + # the test case. + def setUpDecorator(self): + pass + + # OETestRunner call it after a test method has been + # called even if the method raised an exception. + def tearDownDecorator(self): + pass + +class OETestDiscover(OETestDecorator): + + # OETestLoader call it after discover test cases + # needs to return the cases to be run. + @staticmethod + def discover(registry): + return registry['cases'] + +class OETestFilter(OETestDecorator): + + # OETestLoader call it while loading the tests + # in loadTestsFromTestCase method, it needs to + # return a bool, True if needs to be filtered. + # This method must consume the filter used. + @abstractmethod + def filtrate(self, filters): + return False diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py new file mode 100644 index 0000000000..ff7bdd98b7 --- /dev/null +++ b/meta/lib/oeqa/core/decorator/data.py @@ -0,0 +1,98 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.exception import OEQAMissingVariable + +from . import OETestDecorator, registerDecorator + +def has_feature(td, feature): + """ + Checks for feature in DISTRO_FEATURES or IMAGE_FEATURES. + """ + + if (feature in td.get('DISTRO_FEATURES', '') or + feature in td.get('IMAGE_FEATURES', '')): + return True + return False + +@registerDecorator +class skipIfDataVar(OETestDecorator): + """ + Skip test based on value of a data store's variable. + + It will get the info of var from the data store and will + check it against value; if are equal it will skip the test + with msg as the reason. + """ + + attrs = ('var', 'value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %r value is %r to skip test' % + (self.var, self.value)) + self.logger.debug(msg) + if self.case.td.get(self.var) == self.value: + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfNotDataVar(OETestDecorator): + """ + Skip test based on value of a data store's variable. + + It will get the info of var from the data store and will + check it against value; if are not equal it will skip the + test with msg as the reason. + """ + + attrs = ('var', 'value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %r value is not %r to skip test' % + (self.var, self.value)) + self.logger.debug(msg) + if not self.case.td.get(self.var) == self.value: + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfNotInDataVar(OETestDecorator): + """ + Skip test if value is not in data store's variable. + """ + + attrs = ('var', 'value', 'msg') + def setUpDecorator(self): + msg = ('Checking if %r value is in %r to run ' + 'the test' % (self.var, self.value)) + self.logger.debug(msg) + if not self.value in self.case.td.get(self.var): + self.case.skipTest(self.msg) + +@registerDecorator +class OETestDataDepends(OETestDecorator): + attrs = ('td_depends',) + + def setUpDecorator(self): + for v in self.td_depends: + try: + value = self.case.td[v] + except KeyError: + raise OEQAMissingVariable("Test case need %s variable but"\ + " isn't into td" % v) + +@registerDecorator +class skipIfNotFeature(OETestDecorator): + """ + Skip test based on DISTRO_FEATURES. + + value must be in distro features or it will skip the test + with msg as the reason. + """ + + attrs = ('value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %s is in DISTRO_FEATURES ' + 'or IMAGE_FEATURES' % (self.value)) + self.logger.debug(msg) + if not has_feature(self.case.td, self.value): + self.case.skipTest(self.msg) diff --git a/meta/lib/oeqa/core/decorator/depends.py b/meta/lib/oeqa/core/decorator/depends.py new file mode 100644 index 0000000000..195711cf1e --- /dev/null +++ b/meta/lib/oeqa/core/decorator/depends.py @@ -0,0 +1,94 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from unittest import SkipTest + +from oeqa.core.exception import OEQADependency + +from . import OETestDiscover, registerDecorator + +def _add_depends(registry, case, depends): + module_name = case.__module__ + class_name = case.__class__.__name__ + + case_id = case.id() + + for depend in depends: + dparts = depend.split('.') + + if len(dparts) == 1: + depend_id = ".".join((module_name, class_name, dparts[0])) + elif len(dparts) == 2: + depend_id = ".".join((module_name, dparts[0], dparts[1])) + else: + depend_id = depend + + if not case_id in registry: + registry[case_id] = [] + if not depend_id in registry[case_id]: + registry[case_id].append(depend_id) + +def _validate_test_case_depends(cases, depends): + for case in depends: + if not case in cases: + continue + for dep in depends[case]: + if not dep in cases: + raise OEQADependency("TestCase %s depends on %s and isn't available"\ + ", cases available %s." % (case, dep, str(cases.keys()))) + +def _order_test_case_by_depends(cases, depends): + def _dep_resolve(graph, node, resolved, seen): + seen.append(node) + for edge in graph[node]: + if edge not in resolved: + if edge in seen: + raise OEQADependency("Test cases %s and %s have a circular" \ + " dependency." % (node, edge)) + _dep_resolve(graph, edge, resolved, seen) + resolved.append(node) + + dep_graph = {} + dep_graph['__root__'] = cases.keys() + for case in cases: + if case in depends: + dep_graph[case] = depends[case] + else: + dep_graph[case] = [] + + cases_ordered = [] + _dep_resolve(dep_graph, '__root__', cases_ordered, []) + cases_ordered.remove('__root__') + + return [cases[case_id] for case_id in cases_ordered] + +def _skipTestDependency(case, depends): + results = case.tc._results + skipReasons = ['errors', 'failures', 'skipped'] + + for reason in skipReasons: + for test, _ in results[reason]: + if test.id() in depends: + raise SkipTest("Test case %s depends on %s and was in %s." \ + % (case.id(), test.id(), reason)) + +@registerDecorator +class OETestDepends(OETestDiscover): + attrs = ('depends',) + + def bind(self, registry, case): + super(OETestDepends, self).bind(registry, case) + if not registry.get('depends'): + registry['depends'] = {} + _add_depends(registry['depends'], case, self.depends) + + @staticmethod + def discover(registry): + if registry.get('depends'): + _validate_test_case_depends(registry['cases'], registry['depends']) + return _order_test_case_by_depends(registry['cases'], registry['depends']) + else: + return [registry['cases'][case_id] for case_id in registry['cases']] + + def setUpDecorator(self): + _skipTestDependency(self.case, self.depends) diff --git a/meta/lib/oeqa/core/decorator/oeid.py b/meta/lib/oeqa/core/decorator/oeid.py new file mode 100644 index 0000000000..ea8017a55a --- /dev/null +++ b/meta/lib/oeqa/core/decorator/oeid.py @@ -0,0 +1,23 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from . import OETestFilter, registerDecorator +from oeqa.core.utils.misc import intToList + +def _idFilter(oeid, filters): + return False if oeid in filters else True + +@registerDecorator +class OETestID(OETestFilter): + attrs = ('oeid',) + + def bind(self, registry, case): + super(OETestID, self).bind(registry, case) + + def filtrate(self, filters): + if filters.get('oeid'): + filterx = intToList(filters['oeid'], 'oeid') + del filters['oeid'] + if _idFilter(self.oeid, filterx): + return True + return False diff --git a/meta/lib/oeqa/core/decorator/oetag.py b/meta/lib/oeqa/core/decorator/oetag.py new file mode 100644 index 0000000000..ad38ab78a5 --- /dev/null +++ b/meta/lib/oeqa/core/decorator/oetag.py @@ -0,0 +1,24 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from . import OETestFilter, registerDecorator +from oeqa.core.utils.misc import strToList + +def _tagFilter(tags, filters): + return False if set(tags) & set(filters) else True + +@registerDecorator +class OETestTag(OETestFilter): + attrs = ('oetag',) + + def bind(self, registry, case): + super(OETestTag, self).bind(registry, case) + self.oetag = strToList(self.oetag, 'oetag') + + def filtrate(self, filters): + if filters.get('oetag'): + filterx = strToList(filters['oetag'], 'oetag') + del filters['oetag'] + if _tagFilter(self.oetag, filterx): + return True + return False diff --git a/meta/lib/oeqa/core/decorator/oetimeout.py b/meta/lib/oeqa/core/decorator/oetimeout.py new file mode 100644 index 0000000000..a247583f7f --- /dev/null +++ b/meta/lib/oeqa/core/decorator/oetimeout.py @@ -0,0 +1,25 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import signal +from . import OETestDecorator, registerDecorator +from oeqa.core.exception import OEQATimeoutError + +@registerDecorator +class OETimeout(OETestDecorator): + attrs = ('oetimeout',) + + def setUpDecorator(self): + timeout = self.oetimeout + def _timeoutHandler(signum, frame): + raise OEQATimeoutError("Timed out after %s " + "seconds of execution" % timeout) + + self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) + self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) + signal.alarm(self.oetimeout) + + def tearDownDecorator(self): + signal.alarm(0) + signal.signal(signal.SIGALRM, self.alarmSignal) + self.logger.debug("Removed SIGALRM handler") diff --git a/meta/lib/oeqa/core/exception.py b/meta/lib/oeqa/core/exception.py new file mode 100644 index 0000000000..2dfd8402cf --- /dev/null +++ b/meta/lib/oeqa/core/exception.py @@ -0,0 +1,14 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +class OEQAException(Exception): + pass + +class OEQATimeoutError(OEQAException): + pass + +class OEQAMissingVariable(OEQAException): + pass + +class OEQADependency(OEQAException): + pass diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py new file mode 100644 index 0000000000..63a1703536 --- /dev/null +++ b/meta/lib/oeqa/core/loader.py @@ -0,0 +1,272 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import unittest + +from oeqa.core.utils.path import findFile +from oeqa.core.utils.test import getSuiteModules, getCaseID + +from oeqa.core.case import OETestCase +from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ + OETestFilter, OETestDiscover + +def _make_failed_test(classname, methodname, exception, suiteClass): + """ + When loading tests unittest framework stores the exception in a new + class created for be displayed into run(). + + For our purposes will be better to raise the exception in loading + step instead of wait to run the test suite. + """ + raise exception +unittest.loader._make_failed_test = _make_failed_test + +def _find_duplicated_modules(suite, directory): + for module in getSuiteModules(suite): + path = findFile('%s.py' % module, directory) + if path: + raise ImportError("Duplicated %s module found in %s" % (module, path)) + +class OETestLoader(unittest.TestLoader): + caseClass = OETestCase + + kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass', + '_top_level_dir'] + + def __init__(self, tc, module_paths, modules, tests, modules_required, + filters, *args, **kwargs): + self.tc = tc + + self.modules = modules + self.tests = tests + self.modules_required = modules_required + + self.filters = filters + self.decorator_filters = [d for d in decoratorClasses if \ + issubclass(d, OETestFilter)] + self._validateFilters(self.filters, self.decorator_filters) + self.used_filters = [d for d in self.decorator_filters + for f in self.filters + if f in d.attrs] + + if isinstance(module_paths, str): + module_paths = [module_paths] + elif not isinstance(module_paths, list): + raise TypeError('module_paths must be a str or a list of str') + self.module_paths = module_paths + + for kwname in self.kwargs_names: + if kwname in kwargs: + setattr(self, kwname, kwargs[kwname]) + + self._patchCaseClass(self.caseClass) + + def _patchCaseClass(self, testCaseClass): + # Adds custom attributes to the OETestCase class + setattr(testCaseClass, 'tc', self.tc) + setattr(testCaseClass, 'td', self.tc.td) + setattr(testCaseClass, 'logger', self.tc.logger) + + def _validateFilters(self, filters, decorator_filters): + # Validate if filter isn't empty + for key,value in filters.items(): + if not value: + raise TypeError("Filter %s specified is empty" % key) + + # Validate unique attributes + attr_filters = [attr for clss in decorator_filters \ + for attr in clss.attrs] + dup_attr = [attr for attr in attr_filters + if attr_filters.count(attr) > 1] + if dup_attr: + raise TypeError('Detected duplicated attribute(s) %s in filter' + ' decorators' % ' ,'.join(dup_attr)) + + # Validate if filter is supported + for f in filters: + if f not in attr_filters: + classes = ', '.join([d.__name__ for d in decorator_filters]) + raise TypeError('Found "%s" filter but not declared in any of ' + '%s decorators' % (f, classes)) + + def _registerTestCase(self, case): + case_id = case.id() + self.tc._registry['cases'][case_id] = case + + def _handleTestCaseDecorators(self, case): + def _handle(obj): + if isinstance(obj, OETestDecorator): + if not obj.__class__ in decoratorClasses: + raise Exception("Decorator %s isn't registered" \ + " in decoratorClasses." % obj.__name__) + obj.bind(self.tc._registry, case) + + def _walk_closure(obj): + if hasattr(obj, '__closure__') and obj.__closure__: + for f in obj.__closure__: + obj = f.cell_contents + _handle(obj) + _walk_closure(obj) + method = getattr(case, case._testMethodName, None) + _walk_closure(method) + + def _filterTest(self, case): + """ + Returns True if test case must be filtered, False otherwise. + """ + if self.filters: + filters = self.filters.copy() + case_decorators = [cd for cd in case.decorators + if cd.__class__ in self.used_filters] + + # Iterate over case decorators to check if needs to be filtered. + for cd in case_decorators: + if cd.filtrate(filters): + return True + + # Case is missing one or more decorators for all the filters + # being used, so filter test case. + if filters: + return True + + return False + + def _getTestCase(self, testCaseClass, tcName): + if not hasattr(testCaseClass, '__oeqa_loader'): + # In order to support data_vars validation + # monkey patch the default setUp/tearDown{Class} to use + # the ones provided by OETestCase + setattr(testCaseClass, 'setUpClassMethod', + getattr(testCaseClass, 'setUpClass')) + setattr(testCaseClass, 'tearDownClassMethod', + getattr(testCaseClass, 'tearDownClass')) + setattr(testCaseClass, 'setUpClass', + testCaseClass._oeSetUpClass) + setattr(testCaseClass, 'tearDownClass', + testCaseClass._oeTearDownClass) + + # In order to support decorators initialization + # monkey patch the default setUp/tearDown to use + # a setUpDecorators/tearDownDecorators that methods + # will call setUp/tearDown original methods. + setattr(testCaseClass, 'setUpMethod', + getattr(testCaseClass, 'setUp')) + setattr(testCaseClass, 'tearDownMethod', + getattr(testCaseClass, 'tearDown')) + setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp) + setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown) + + setattr(testCaseClass, '__oeqa_loader', True) + + case = testCaseClass(tcName) + setattr(case, 'decorators', []) + + return case + + def loadTestsFromTestCase(self, testCaseClass): + """ + Returns a suite of all tests cases contained in testCaseClass. + """ + if issubclass(testCaseClass, unittest.suite.TestSuite): + raise TypeError("Test cases should not be derived from TestSuite." \ + " Maybe you meant to derive %s from TestCase?" \ + % testCaseClass.__name__) + if not issubclass(testCaseClass, self.caseClass): + raise TypeError("Test %s is not derived from %s" % \ + (testCaseClass.__name__, self.caseClass.__name__)) + + testCaseNames = self.getTestCaseNames(testCaseClass) + if not testCaseNames and hasattr(testCaseClass, 'runTest'): + testCaseNames = ['runTest'] + + suite = [] + for tcName in testCaseNames: + case = self._getTestCase(testCaseClass, tcName) + # Filer by case id + if not (self.tests and not 'all' in self.tests + and not getCaseID(case) in self.tests): + self._handleTestCaseDecorators(case) + + # Filter by decorators + if not self._filterTest(case): + self._registerTestCase(case) + suite.append(case) + + return self.suiteClass(suite) + + def discover(self): + big_suite = self.suiteClass() + for path in self.module_paths: + _find_duplicated_modules(big_suite, path) + suite = super(OETestLoader, self).discover(path, + pattern='*.py', top_level_dir=path) + big_suite.addTests(suite) + + cases = None + discover_classes = [clss for clss in decoratorClasses + if issubclass(clss, OETestDiscover)] + for clss in discover_classes: + cases = clss.discover(self.tc._registry) + + return self.suiteClass(cases) if cases else big_suite + + # XXX After Python 3.5, remove backward compatibility hacks for + # use_load_tests deprecation via *args and **kws. See issue 16662. + if sys.version_info >= (3,5): + def loadTestsFromModule(self, module, *args, pattern=None, **kws): + """ + Returns a suite of all tests cases contained in module. + """ + if module.__name__ in sys.builtin_module_names: + msg = 'Tried to import %s test module but is a built-in' + raise ImportError(msg % module.__name__) + + # Normal test modules are loaded if no modules were specified, + # if module is in the specified module list or if 'all' is in + # module list. + # Underscore modules are loaded only if specified in module list. + load_module = True if not module.__name__.startswith('_') \ + and (not self.modules \ + or module.__name__ in self.modules \ + or 'all' in self.modules) \ + else False + + load_underscore = True if module.__name__.startswith('_') \ + and module.__name__ in self.modules \ + else False + + if load_module or load_underscore: + return super(OETestLoader, self).loadTestsFromModule( + module, *args, pattern=pattern, **kws) + else: + return self.suiteClass() + else: + def loadTestsFromModule(self, module, use_load_tests=True): + """ + Returns a suite of all tests cases contained in module. + """ + if module.__name__ in sys.builtin_module_names: + msg = 'Tried to import %s test module but is a built-in' + raise ImportError(msg % module.__name__) + + # Normal test modules are loaded if no modules were specified, + # if module is in the specified module list or if 'all' is in + # module list. + # Underscore modules are loaded only if specified in module list. + load_module = True if not module.__name__.startswith('_') \ + and (not self.modules \ + or module.__name__ in self.modules \ + or 'all' in self.modules) \ + else False + + load_underscore = True if module.__name__.startswith('_') \ + and module.__name__ in self.modules \ + else False + + if load_module or load_underscore: + return super(OETestLoader, self).loadTestsFromModule( + module, use_load_tests) + else: + return self.suiteClass() diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py new file mode 100644 index 0000000000..44ffecb0cd --- /dev/null +++ b/meta/lib/oeqa/core/runner.py @@ -0,0 +1,76 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import time +import unittest +import logging + +xmlEnabled = False +try: + import xmlrunner + from xmlrunner.result import _XMLTestResult as _TestResult + from xmlrunner.runner import XMLTestRunner as _TestRunner + xmlEnabled = True +except ImportError: + # use the base runner instead + from unittest import TextTestResult as _TestResult + from unittest import TextTestRunner as _TestRunner + +class OEStreamLogger(object): + def __init__(self, logger): + self.logger = logger + self.buffer = "" + + def write(self, msg): + if len(msg) > 1 and msg[0] != '\n': + self.buffer += msg + else: + self.logger.log(logging.INFO, self.buffer.rstrip("\n")) + self.buffer = "" + + def flush(self): + for handler in self.logger.handlers: + handler.flush() + +class OETestResult(_TestResult): + def __init__(self, tc, *args, **kwargs): + super(OETestResult, self).__init__(*args, **kwargs) + + self.tc = tc + + self.tc._results['failures'] = self.failures + self.tc._results['errors'] = self.errors + self.tc._results['skipped'] = self.skipped + self.tc._results['expectedFailures'] = self.expectedFailures + + def startTest(self, test): + super(OETestResult, self).startTest(test) + +class OETestRunner(_TestRunner): + def __init__(self, tc, *args, **kwargs): + if xmlEnabled: + if not kwargs.get('output'): + kwargs['output'] = os.path.join(os.getcwd(), + 'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid())) + + super(OETestRunner, self).__init__(*args, **kwargs) + self.tc = tc + self.resultclass = OETestResult + + # XXX: The unittest-xml-reporting package defines _make_result method instead + # of _makeResult standard on unittest. + if xmlEnabled: + def _make_result(self): + """ + Creates a TestResult object which will be used to store + information about the executed tests. + """ + # override in subclasses if necessary. + return self.resultclass(self.tc, + self.stream, self.descriptions, self.verbosity, self.elapsed_times + ) + else: + def _makeResult(self): + return self.resultclass(self.tc, self.stream, self.descriptions, + self.verbosity) diff --git a/meta/lib/oeqa/core/target/__init__.py b/meta/lib/oeqa/core/target/__init__.py new file mode 100644 index 0000000000..d2468bc257 --- /dev/null +++ b/meta/lib/oeqa/core/target/__init__.py @@ -0,0 +1,33 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from abc import abstractmethod + +class OETarget(object): + + def __init__(self, logger, *args, **kwargs): + self.logger = logger + + @abstractmethod + def start(self): + pass + + @abstractmethod + def stop(self): + pass + + @abstractmethod + def run(self, cmd, timeout=None): + pass + + @abstractmethod + def copyTo(self, localSrc, remoteDst): + pass + + @abstractmethod + def copyFrom(self, remoteSrc, localDst): + pass + + @abstractmethod + def copyDirTo(self, localSrc, remoteDst): + pass diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py new file mode 100644 index 0000000000..2dc521c216 --- /dev/null +++ b/meta/lib/oeqa/core/target/qemu.py @@ -0,0 +1,45 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import signal +import time + +from .ssh import OESSHTarget +from oeqa.utils.qemurunner import QemuRunner + +supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic', 'elf'] + +class OEQemuTarget(OESSHTarget): + def __init__(self, logger, ip, server_ip, timeout=300, user='root', + port=None, machine='', rootfs='', kernel='', kvm=False, + dump_dir='', dump_host_cmds='', display='', bootlog='', + tmpdir='', dir_image='', boottime=60, **kwargs): + + super(OEQemuTarget, self).__init__(logger, ip, server_ip, timeout, + user, port) + + self.ip = ip + self.server_ip = server_ip + self.machine = machine + self.rootfs = rootfs + self.kernel = kernel + self.kvm = kvm + + self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, + deploy_dir_image=dir_image, display=display, + logfile=bootlog, boottime=boottime, + use_kvm=kvm, dump_dir=dump_dir, + dump_host_cmds=dump_host_cmds) + + def start(self, params=None, extra_bootparams=None): + if self.runner.start(params, extra_bootparams=extra_bootparams): + self.ip = self.runner.ip + self.server_ip = self.runner.server_ip + else: + self.stop() + raise RuntimeError("FAILED to start qemu - check the task log and the boot log") + + def stop(self): + self.runner.stop() diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py new file mode 100644 index 0000000000..b80939c0e5 --- /dev/null +++ b/meta/lib/oeqa/core/target/ssh.py @@ -0,0 +1,266 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import time +import select +import logging +import subprocess + +from . import OETarget + +class OESSHTarget(OETarget): + def __init__(self, logger, ip, server_ip, timeout=300, user='root', + port=None, **kwargs): + if not logger: + logger = logging.getLogger('target') + logger.setLevel(logging.INFO) + filePath = os.path.join(os.getcwd(), 'remoteTarget.log') + fileHandler = logging.FileHandler(filePath, 'w', 'utf-8') + formatter = logging.Formatter( + '%(asctime)s.%(msecs)03d %(levelname)s: %(message)s', + '%H:%M:%S') + fileHandler.setFormatter(formatter) + logger.addHandler(fileHandler) + + super(OESSHTarget, self).__init__(logger) + self.ip = ip + self.server_ip = server_ip + self.timeout = timeout + self.user = user + ssh_options = [ + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + '-o', 'LogLevel=ERROR' + ] + self.ssh = ['ssh', '-l', self.user ] + ssh_options + self.scp = ['scp'] + ssh_options + if port: + self.ssh = self.ssh + [ '-p', port ] + self.scp = self.scp + [ '-P', port ] + + def start(self, **kwargs): + pass + + def stop(self, **kwargs): + pass + + def _run(self, command, timeout=None, ignore_status=True): + """ + Runs command in target using SSHProcess. + """ + self.logger.debug("[Running]$ %s" % " ".join(command)) + + starttime = time.time() + status, output = SSHCall(command, self.logger, timeout) + self.logger.debug("[Command returned '%d' after %.2f seconds]" + "" % (status, time.time() - starttime)) + + if status and not ignore_status: + raise AssertionError("Command '%s' returned non-zero exit " + "status %d:\n%s" % (command, status, output)) + + return (status, output) + + def run(self, command, timeout=None): + """ + Runs command in target. + + command: Command to run on target. + timeout: <value>: Kill command after <val> seconds. + None: Kill command default value seconds. + 0: No timeout, runs until return. + """ + targetCmd = 'export PATH=/usr/sbin:/sbin:/usr/bin:/bin; %s' % command + sshCmd = self.ssh + [self.ip, targetCmd] + + if timeout: + processTimeout = timeout + elif timeout==0: + processTimeout = None + else: + processTimeout = self.timeout + + status, output = self._run(sshCmd, processTimeout, True) + self.logger.info('\nCommand: %s\nOutput: %s\n' % (command, output)) + return (status, output) + + def copyTo(self, localSrc, remoteDst): + """ + Copy file to target. + + If local file is symlink, recreate symlink in target. + """ + if os.path.islink(localSrc): + link = os.readlink(localSrc) + dstDir, dstBase = os.path.split(remoteDst) + sshCmd = 'cd %s; ln -s %s %s' % (dstDir, link, dstBase) + return self.run(sshCmd) + + else: + remotePath = '%s@%s:%s' % (self.user, self.ip, remoteDst) + scpCmd = self.scp + [localSrc, remotePath] + return self._run(scpCmd, ignore_status=False) + + def copyFrom(self, remoteSrc, localDst): + """ + Copy file from target. + """ + remotePath = '%s@%s:%s' % (self.user, self.ip, remoteSrc) + scpCmd = self.scp + [remotePath, localDst] + return self._run(scpCmd, ignore_status=False) + + def copyDirTo(self, localSrc, remoteDst): + """ + Copy recursively localSrc directory to remoteDst in target. + """ + + for root, dirs, files in os.walk(localSrc): + # Create directories in the target as needed + for d in dirs: + tmpDir = os.path.join(root, d).replace(localSrc, "") + newDir = os.path.join(remoteDst, tmpDir.lstrip("/")) + cmd = "mkdir -p %s" % newDir + self.run(cmd) + + # Copy files into the target + for f in files: + tmpFile = os.path.join(root, f).replace(localSrc, "") + dstFile = os.path.join(remoteDst, tmpFile.lstrip("/")) + srcFile = os.path.join(root, f) + self.copyTo(srcFile, dstFile) + + def deleteFiles(self, remotePath, files): + """ + Deletes files in target's remotePath. + """ + + cmd = "rm" + if not isinstance(files, list): + files = [files] + + for f in files: + cmd = "%s %s" % (cmd, os.path.join(remotePath, f)) + + self.run(cmd) + + + def deleteDir(self, remotePath): + """ + Deletes target's remotePath directory. + """ + + cmd = "rmdir %s" % remotePath + self.run(cmd) + + + def deleteDirStructure(self, localPath, remotePath): + """ + Delete recursively localPath structure directory in target's remotePath. + + This function is very usefult to delete a package that is installed in + the DUT and the host running the test has such package extracted in tmp + directory. + + Example: + pwd: /home/user/tmp + tree: . + └── work + ├── dir1 + │ └── file1 + └── dir2 + + localpath = "/home/user/tmp" and remotepath = "/home/user" + + With the above variables this function will try to delete the + directory in the DUT in this order: + /home/user/work/dir1/file1 + /home/user/work/dir1 (if dir is empty) + /home/user/work/dir2 (if dir is empty) + /home/user/work (if dir is empty) + """ + + for root, dirs, files in os.walk(localPath, topdown=False): + # Delete files first + tmpDir = os.path.join(root).replace(localPath, "") + remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) + self.deleteFiles(remoteDir, files) + + # Remove dirs if empty + for d in dirs: + tmpDir = os.path.join(root, d).replace(localPath, "") + remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) + self.deleteDir(remoteDir) + +def SSHCall(command, logger, timeout=None, **opts): + + def run(): + nonlocal output + nonlocal process + starttime = time.time() + process = subprocess.Popen(command, **options) + if timeout: + endtime = starttime + timeout + eof = False + while time.time() < endtime and not eof: + logger.debug('time: %s, endtime: %s' % (time.time(), endtime)) + try: + if select.select([process.stdout], [], [], 5)[0] != []: + data = os.read(process.stdout.fileno(), 1024) + if not data: + process.stdout.close() + eof = True + else: + data = data.decode("utf-8") + output += data + logger.debug('Partial data from SSH call: %s' % data) + endtime = time.time() + timeout + except InterruptedError: + continue + + # process hasn't returned yet + if not eof: + process.terminate() + time.sleep(5) + try: + process.kill() + except OSError: + pass + endtime = time.time() - starttime + lastline = ("\nProcess killed - no output for %d seconds. Total" + " running time: %d seconds." % (timeout, endtime)) + logger.debug('Received data from SSH call %s ' % lastline) + output += lastline + + else: + output = process.communicate()[0].decode("utf-8") + logger.debug('Data from SSH call: %s' % output.rstrip()) + + options = { + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "stdin": None, + "shell": False, + "bufsize": -1, + "preexec_fn": os.setsid, + } + options.update(opts) + output = '' + process = None + + # Unset DISPLAY which means we won't trigger SSH_ASKPASS + env = os.environ.copy() + if "DISPLAY" in env: + del env['DISPLAY'] + options['env'] = env + + try: + run() + except: + # Need to guard against a SystemExit or other exception ocurring + # whilst running and ensure we don't leave a process behind. + if process.poll() is None: + process.kill() + logger.debug('Something went wrong, killing SSH process') + raise + return (process.wait(), output.rstrip()) diff --git a/meta/lib/oeqa/core/tests/__init__.py b/meta/lib/oeqa/core/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/core/tests/__init__.py diff --git a/meta/lib/oeqa/core/tests/cases/data.py b/meta/lib/oeqa/core/tests/cases/data.py new file mode 100644 index 0000000000..88003a6adc --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/data.py @@ -0,0 +1,20 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oetag import OETestTag +from oeqa.core.decorator.data import OETestDataDepends + +class DataTest(OETestCase): + data_vars = ['IMAGE', 'ARCH'] + + @OETestDataDepends(['MACHINE',]) + @OETestTag('dataTestOk') + def testDataOk(self): + self.assertEqual(self.td.get('IMAGE'), 'core-image-minimal') + self.assertEqual(self.td.get('ARCH'), 'x86') + self.assertEqual(self.td.get('MACHINE'), 'qemuarm') + + @OETestTag('dataTestFail') + def testDataFail(self): + pass diff --git a/meta/lib/oeqa/core/tests/cases/depends.py b/meta/lib/oeqa/core/tests/cases/depends.py new file mode 100644 index 0000000000..17cdd90b15 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/depends.py @@ -0,0 +1,38 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.depends import OETestDepends + +class DependsTest(OETestCase): + + def testDependsFirst(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsFirst']) + def testDependsSecond(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsSecond']) + def testDependsThird(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsSecond']) + def testDependsFourth(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsThird', 'testDependsFourth']) + def testDependsFifth(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsCircular3']) + def testDependsCircular1(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsCircular1']) + def testDependsCircular2(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsCircular2']) + def testDependsCircular3(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py new file mode 100644 index 0000000000..038d445931 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py @@ -0,0 +1,15 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class AnotherIDTest(OETestCase): + + def testAnotherIdGood(self): + self.assertTrue(True, msg='How is this possible?') + + def testAnotherIdOther(self): + self.assertTrue(True, msg='How is this possible?') + + def testAnotherIdNone(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/valid/another.py b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py new file mode 100644 index 0000000000..c9ffd17773 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py @@ -0,0 +1,9 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class AnotherTest(OETestCase): + + def testAnother(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/oeid.py b/meta/lib/oeqa/core/tests/cases/oeid.py new file mode 100644 index 0000000000..c2d3d32f2d --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/oeid.py @@ -0,0 +1,18 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oeid import OETestID + +class IDTest(OETestCase): + + @OETestID(101) + def testIdGood(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestID(102) + def testIdOther(self): + self.assertTrue(True, msg='How is this possible?') + + def testIdNone(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/oetag.py b/meta/lib/oeqa/core/tests/cases/oetag.py new file mode 100644 index 0000000000..0cae02e75c --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/oetag.py @@ -0,0 +1,18 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oetag import OETestTag + +class TagTest(OETestCase): + + @OETestTag('goodTag') + def testTagGood(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestTag('otherTag') + def testTagOther(self): + self.assertTrue(True, msg='How is this possible?') + + def testTagNone(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/timeout.py b/meta/lib/oeqa/core/tests/cases/timeout.py new file mode 100644 index 0000000000..870c3157f7 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/timeout.py @@ -0,0 +1,18 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from time import sleep + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oetimeout import OETimeout + +class TimeoutTest(OETestCase): + + @OETimeout(1) + def testTimeoutPass(self): + self.assertTrue(True, msg='How is this possible?') + + @OETimeout(1) + def testTimeoutFail(self): + sleep(2) + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/common.py b/meta/lib/oeqa/core/tests/common.py new file mode 100644 index 0000000000..52b18a1c3e --- /dev/null +++ b/meta/lib/oeqa/core/tests/common.py @@ -0,0 +1,35 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import sys +import os + +import unittest +import logging +import os + +logger = logging.getLogger("oeqa") +logger.setLevel(logging.INFO) +consoleHandler = logging.StreamHandler() +formatter = logging.Formatter('OEQATest: %(message)s') +consoleHandler.setFormatter(formatter) +logger.addHandler(consoleHandler) + +def setup_sys_path(): + directory = os.path.dirname(os.path.abspath(__file__)) + oeqa_lib = os.path.realpath(os.path.join(directory, '../../../')) + if not oeqa_lib in sys.path: + sys.path.insert(0, oeqa_lib) + +class TestBase(unittest.TestCase): + def setUp(self): + self.logger = logger + directory = os.path.dirname(os.path.abspath(__file__)) + self.cases_path = os.path.join(directory, 'cases') + + def _testLoader(self, d={}, modules=[], tests=[], filters={}): + from oeqa.core.context import OETestContext + tc = OETestContext(d, self.logger) + tc.loadTests(self.cases_path, modules=modules, tests=tests, + filters=filters) + return tc diff --git a/meta/lib/oeqa/core/tests/test_data.py b/meta/lib/oeqa/core/tests/test_data.py new file mode 100755 index 0000000000..320468cbe4 --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_data.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import unittest +import logging +import os + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.exception import OEQAMissingVariable +from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames + +class TestData(TestBase): + modules = ['data'] + + def test_data_fail_missing_variable(self): + expectedException = "oeqa.core.exception.OEQAMissingVariable" + + tc = self._testLoader(modules=self.modules) + self.assertEqual(False, tc.runTests().wasSuccessful()) + for test, data in tc._results['errors']: + expect = False + if expectedException in data: + expect = True + + self.assertTrue(expect) + + def test_data_fail_wrong_variable(self): + expectedError = 'AssertionError' + d = {'IMAGE' : 'core-image-sato', 'ARCH' : 'arm'} + + tc = self._testLoader(d=d, modules=self.modules) + self.assertEqual(False, tc.runTests().wasSuccessful()) + for test, data in tc._results['failures']: + expect = False + if expectedError in data: + expect = True + + self.assertTrue(expect) + + def test_data_ok(self): + d = {'IMAGE' : 'core-image-minimal', 'ARCH' : 'x86', 'MACHINE' : 'qemuarm'} + + tc = self._testLoader(d=d, modules=self.modules) + self.assertEqual(True, tc.runTests().wasSuccessful()) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py new file mode 100755 index 0000000000..f7d11e885a --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_decorators.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import signal +import unittest + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.exception import OEQADependency +from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs + +class TestFilterDecorator(TestBase): + + def _runFilterTest(self, modules, filters, expect, msg): + tc = self._testLoader(modules=modules, filters=filters) + test_loaded = set(getSuiteCasesNames(tc.suites)) + self.assertEqual(expect, test_loaded, msg=msg) + + def test_oetag(self): + # Get all cases without filtering. + filter_all = {} + test_all = {'testTagGood', 'testTagOther', 'testTagNone'} + msg_all = 'Failed to get all oetag cases without filtering.' + + # Get cases with 'goodTag'. + filter_good = {'oetag':'goodTag'} + test_good = {'testTagGood'} + msg_good = 'Failed to get just one test filtering with "goodTag" oetag.' + + # Get cases with an invalid tag. + filter_invalid = {'oetag':'invalidTag'} + test_invalid = set() + msg_invalid = 'Failed to filter all test using an invalid oetag.' + + tests = ((filter_all, test_all, msg_all), + (filter_good, test_good, msg_good), + (filter_invalid, test_invalid, msg_invalid)) + + for test in tests: + self._runFilterTest(['oetag'], test[0], test[1], test[2]) + + def test_oeid(self): + # Get all cases without filtering. + filter_all = {} + test_all = {'testIdGood', 'testIdOther', 'testIdNone'} + msg_all = 'Failed to get all oeid cases without filtering.' + + # Get cases with '101' oeid. + filter_good = {'oeid': 101} + test_good = {'testIdGood'} + msg_good = 'Failed to get just one tes filtering with "101" oeid.' + + # Get cases with an invalid id. + filter_invalid = {'oeid':999} + test_invalid = set() + msg_invalid = 'Failed to filter all test using an invalid oeid.' + + tests = ((filter_all, test_all, msg_all), + (filter_good, test_good, msg_good), + (filter_invalid, test_invalid, msg_invalid)) + + for test in tests: + self._runFilterTest(['oeid'], test[0], test[1], test[2]) + +class TestDependsDecorator(TestBase): + modules = ['depends'] + + def test_depends_order(self): + tests = ['depends.DependsTest.testDependsFirst', + 'depends.DependsTest.testDependsSecond', + 'depends.DependsTest.testDependsThird', + 'depends.DependsTest.testDependsFourth', + 'depends.DependsTest.testDependsFifth'] + tests2 = list(tests) + tests2[2], tests2[3] = tests[3], tests[2] + tc = self._testLoader(modules=self.modules, tests=tests) + test_loaded = getSuiteCasesIDs(tc.suites) + result = True if test_loaded == tests or test_loaded == tests2 else False + msg = 'Failed to order tests using OETestDepends decorator.\nTest order:'\ + ' %s.\nExpected: %s\nOr: %s' % (test_loaded, tests, tests2) + self.assertTrue(result, msg=msg) + + def test_depends_fail_missing_dependency(self): + expect = "TestCase depends.DependsTest.testDependsSecond depends on "\ + "depends.DependsTest.testDependsFirst and isn't available" + tests = ['depends.DependsTest.testDependsSecond'] + try: + # Must throw OEQADependency because missing 'testDependsFirst' + tc = self._testLoader(modules=self.modules, tests=tests) + self.fail('Expected OEQADependency exception') + except OEQADependency as e: + result = True if expect in str(e) else False + msg = 'Expected OEQADependency exception missing testDependsFirst test' + self.assertTrue(result, msg=msg) + + def test_depends_fail_circular_dependency(self): + expect = 'have a circular dependency' + tests = ['depends.DependsTest.testDependsCircular1', + 'depends.DependsTest.testDependsCircular2', + 'depends.DependsTest.testDependsCircular3'] + try: + # Must throw OEQADependency because circular dependency + tc = self._testLoader(modules=self.modules, tests=tests) + self.fail('Expected OEQADependency exception') + except OEQADependency as e: + result = True if expect in str(e) else False + msg = 'Expected OEQADependency exception having a circular dependency' + self.assertTrue(result, msg=msg) + +class TestTimeoutDecorator(TestBase): + modules = ['timeout'] + + def test_timeout(self): + tests = ['timeout.TimeoutTest.testTimeoutPass'] + msg = 'Failed to run test using OETestTimeout' + alarm_signal = signal.getsignal(signal.SIGALRM) + tc = self._testLoader(modules=self.modules, tests=tests) + self.assertTrue(tc.runTests().wasSuccessful(), msg=msg) + msg = "OETestTimeout didn't restore SIGALRM" + self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) + + def test_timeout_fail(self): + tests = ['timeout.TimeoutTest.testTimeoutFail'] + msg = "OETestTimeout test didn't timeout as expected" + alarm_signal = signal.getsignal(signal.SIGALRM) + tc = self._testLoader(modules=self.modules, tests=tests) + self.assertFalse(tc.runTests().wasSuccessful(), msg=msg) + msg = "OETestTimeout didn't restore SIGALRM" + self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_loader.py b/meta/lib/oeqa/core/tests/test_loader.py new file mode 100755 index 0000000000..b79b8bad4d --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_loader.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import unittest + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.exception import OEQADependency +from oeqa.core.utils.test import getSuiteModules, getSuiteCasesIDs + +class TestLoader(TestBase): + + def test_fail_empty_filter(self): + filters = {'oetag' : ''} + expect = 'Filter oetag specified is empty' + msg = 'Expected TypeError exception for having invalid filter' + try: + # Must throw TypeError because empty filter + tc = self._testLoader(filters=filters) + self.fail(msg) + except TypeError as e: + result = True if expect in str(e) else False + self.assertTrue(result, msg=msg) + + def test_fail_invalid_filter(self): + filters = {'invalid' : 'good'} + expect = 'filter but not declared in any of' + msg = 'Expected TypeError exception for having invalid filter' + try: + # Must throw TypeError because invalid filter + tc = self._testLoader(filters=filters) + self.fail(msg) + except TypeError as e: + result = True if expect in str(e) else False + self.assertTrue(result, msg=msg) + + def test_fail_duplicated_module(self): + cases_path = self.cases_path + invalid_path = os.path.join(cases_path, 'loader', 'invalid') + self.cases_path = [self.cases_path, invalid_path] + expect = 'Duplicated oeid module found in' + msg = 'Expected ImportError exception for having duplicated module' + try: + # Must throw ImportEror because duplicated module + tc = self._testLoader() + self.fail(msg) + except ImportError as e: + result = True if expect in str(e) else False + self.assertTrue(result, msg=msg) + finally: + self.cases_path = cases_path + + def test_filter_modules(self): + expected_modules = {'oeid', 'oetag'} + tc = self._testLoader(modules=expected_modules) + modules = getSuiteModules(tc.suites) + msg = 'Expected just %s modules' % ', '.join(expected_modules) + self.assertEqual(modules, expected_modules, msg=msg) + + def test_filter_cases(self): + modules = ['oeid', 'oetag', 'data'] + expected_cases = {'data.DataTest.testDataOk', + 'oetag.TagTest.testTagGood', + 'oeid.IDTest.testIdGood'} + tc = self._testLoader(modules=modules, tests=expected_cases) + cases = set(getSuiteCasesIDs(tc.suites)) + msg = 'Expected just %s cases' % ', '.join(expected_cases) + self.assertEqual(cases, expected_cases, msg=msg) + + def test_import_from_paths(self): + cases_path = self.cases_path + cases2_path = os.path.join(cases_path, 'loader', 'valid') + expected_modules = {'oeid', 'another'} + self.cases_path = [self.cases_path, cases2_path] + tc = self._testLoader(modules=expected_modules) + modules = getSuiteModules(tc.suites) + self.cases_path = cases_path + msg = 'Expected modules from two different paths' + self.assertEqual(modules, expected_modules, msg=msg) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_runner.py b/meta/lib/oeqa/core/tests/test_runner.py new file mode 100755 index 0000000000..a3f3861fed --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_runner.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import unittest +import logging +import tempfile + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.runner import OEStreamLogger + +class TestRunner(TestBase): + def test_stream_logger(self): + fp = tempfile.TemporaryFile(mode='w+') + + logging.basicConfig(format='%(message)s', stream=fp) + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + oeSL = OEStreamLogger(logger) + + lines = ['init', 'bigline_' * 65535, 'morebigline_' * 65535 * 4, 'end'] + for line in lines: + oeSL.write(line) + + fp.seek(0) + fp_lines = fp.readlines() + for i, fp_line in enumerate(fp_lines): + fp_line = fp_line.strip() + self.assertEqual(lines[i], fp_line) + + fp.close() + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/utils/__init__.py b/meta/lib/oeqa/core/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/core/utils/__init__.py diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py new file mode 100644 index 0000000000..0b223b5d08 --- /dev/null +++ b/meta/lib/oeqa/core/utils/misc.py @@ -0,0 +1,44 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +def toList(obj, obj_type, obj_name="Object"): + if isinstance(obj, obj_type): + return [obj] + elif isinstance(obj, list): + return obj + else: + raise TypeError("%s must be %s or list" % (obj_name, obj_type)) + +def toSet(obj, obj_type, obj_name="Object"): + if isinstance(obj, obj_type): + return {obj} + elif isinstance(obj, list): + return set(obj) + elif isinstance(obj, set): + return obj + else: + raise TypeError("%s must be %s or set" % (obj_name, obj_type)) + +def strToList(obj, obj_name="Object"): + return toList(obj, str, obj_name) + +def strToSet(obj, obj_name="Object"): + return toSet(obj, str, obj_name) + +def intToList(obj, obj_name="Object"): + return toList(obj, int, obj_name) + +def dataStoteToDict(d, variables): + data = {} + + for v in variables: + data[v] = d.getVar(v) + + return data + +def updateTestData(d, td, variables): + """ + Updates variables with values of data store to test data. + """ + for var in variables: + td[var] = d.getVar(var) diff --git a/meta/lib/oeqa/core/utils/path.py b/meta/lib/oeqa/core/utils/path.py new file mode 100644 index 0000000000..a21caad5cb --- /dev/null +++ b/meta/lib/oeqa/core/utils/path.py @@ -0,0 +1,19 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys + +def findFile(file_name, directory): + """ + Search for a file in directory and returns its complete path. + """ + for r, d, f in os.walk(directory): + if file_name in f: + return os.path.join(r, file_name) + return None + +def remove_safe(path): + if os.path.exists(path): + os.remove(path) + diff --git a/meta/lib/oeqa/core/utils/test.py b/meta/lib/oeqa/core/utils/test.py new file mode 100644 index 0000000000..88d5d13981 --- /dev/null +++ b/meta/lib/oeqa/core/utils/test.py @@ -0,0 +1,86 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import inspect +import unittest + +def getSuiteCases(suite): + """ + Returns individual test from a test suite. + """ + tests = [] + + if isinstance(suite, unittest.TestCase): + tests.append(suite) + elif isinstance(suite, unittest.suite.TestSuite): + for item in suite: + tests.extend(getSuiteCases(item)) + + return tests + +def getSuiteModules(suite): + """ + Returns modules in a test suite. + """ + modules = set() + for test in getSuiteCases(suite): + modules.add(getCaseModule(test)) + return modules + +def getSuiteCasesInfo(suite, func): + """ + Returns test case info from suite. Info is fetched from func. + """ + tests = [] + for test in getSuiteCases(suite): + tests.append(func(test)) + return tests + +def getSuiteCasesNames(suite): + """ + Returns test case names from suite. + """ + return getSuiteCasesInfo(suite, getCaseMethod) + +def getSuiteCasesIDs(suite): + """ + Returns test case ids from suite. + """ + return getSuiteCasesInfo(suite, getCaseID) + +def getSuiteCasesFiles(suite): + """ + Returns test case files paths from suite. + """ + return getSuiteCasesInfo(suite, getCaseFile) + +def getCaseModule(test_case): + """ + Returns test case module name. + """ + return test_case.__module__ + +def getCaseClass(test_case): + """ + Returns test case class name. + """ + return test_case.__class__.__name__ + +def getCaseID(test_case): + """ + Returns test case complete id. + """ + return test_case.id() + +def getCaseFile(test_case): + """ + Returns test case file path. + """ + return inspect.getsourcefile(test_case.__class__) + +def getCaseMethod(test_case): + """ + Returns test case method name. + """ + return getCaseID(test_case).split('.')[-1] diff --git a/meta/lib/oeqa/runtime/files/test.c b/meta/lib/oeqa/files/test.c index 2d8389c92e..2d8389c92e 100644 --- a/meta/lib/oeqa/runtime/files/test.c +++ b/meta/lib/oeqa/files/test.c diff --git a/meta/lib/oeqa/files/test.cpp b/meta/lib/oeqa/files/test.cpp new file mode 100644 index 0000000000..9e1a76473d --- /dev/null +++ b/meta/lib/oeqa/files/test.cpp @@ -0,0 +1,3 @@ +#include <limits> + +int main() {}
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/files/test.pl b/meta/lib/oeqa/files/test.pl index 689c8f1635..689c8f1635 100644 --- a/meta/lib/oeqa/runtime/files/test.pl +++ b/meta/lib/oeqa/files/test.pl diff --git a/meta/lib/oeqa/runtime/files/test.py b/meta/lib/oeqa/files/test.py index f3a2273c52..f389225d72 100644 --- a/meta/lib/oeqa/runtime/files/test.py +++ b/meta/lib/oeqa/files/test.py @@ -3,4 +3,4 @@ import os os.system('touch /tmp/testfile.python') a = 9.01e+21 - 9.01e+21 + 0.01 -print "the value of a is %s" % a +print("the value of a is %s" % a) diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index dcbd498fca..f7171260e7 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -7,50 +7,77 @@ # It also has some helper functions and it's responsible for actually starting the tests -import os, re, mmap +import os, re, mmap, sys import unittest import inspect -from oeqa.utils.decorators import LogResults +import subprocess +import signal +import shutil +import functools +try: + import bb +except ImportError: + pass +import logging -def loadTests(tc): +import oeqa.runtime +# Exported test doesn't require sdkext +try: + import oeqa.sdkext +except ImportError: + pass +from oeqa.utils.decorators import LogResults, gettag, getResults - # set the context object passed from the test class - setattr(oeTest, "tc", tc) - # set ps command to use - setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") - # prepare test suite, loader and runner - suite = unittest.TestSuite() - testloader = unittest.TestLoader() - testloader.sortTestMethodsUsing = None - suite = testloader.loadTestsFromNames(tc.testslist) +logger = logging.getLogger("BitBake") - return suite +def getVar(obj): + #extend form dict, if a variable didn't exists, need find it in testcase + class VarDict(dict): + def __getitem__(self, key): + return gettag(obj, key) + return VarDict() -def runTests(tc): +def checkTags(tc, tagexp): + return eval(tagexp, None, getVar(tc)) - suite = loadTests(tc) - print("Test modules %s" % tc.testslist) - print("Found %s tests" % suite.countTestCases()) - runner = unittest.TextTestRunner(verbosity=2) - result = runner.run(suite) - - return result +def filterByTagExp(testsuite, tagexp): + if not tagexp: + return testsuite + caseList = [] + for each in testsuite: + if not isinstance(each, unittest.BaseTestSuite): + if checkTags(each, tagexp): + caseList.append(each) + else: + caseList.append(filterByTagExp(each, tagexp)) + return testsuite.__class__(caseList) @LogResults class oeTest(unittest.TestCase): + pscmd = "ps" longMessage = True @classmethod def hasPackage(self, pkg): + """ + True if the full package name exists in the manifest, False otherwise. + """ + return pkg in oeTest.tc.pkgmanifest - if re.search(pkg, oeTest.tc.pkgmanifest): - return True + @classmethod + def hasPackageMatch(self, match): + """ + True if match exists in the manifest as a regular expression substring, + False otherwise. + """ + for s in oeTest.tc.pkgmanifest: + if re.match(match, s): + return True return False @classmethod def hasFeature(self,feature): - if feature in oeTest.tc.imagefeatures or \ feature in oeTest.tc.distrofeatures: return True @@ -58,11 +85,48 @@ class oeTest(unittest.TestCase): return False class oeRuntimeTest(oeTest): - def __init__(self, methodName='runTest'): self.target = oeRuntimeTest.tc.target super(oeRuntimeTest, self).__init__(methodName) + def setUp(self): + # Install packages in the DUT + self.tc.install_uninstall_packages(self.id()) + + # Check if test needs to run + if self.tc.sigterm: + self.fail("Got SIGTERM") + elif (type(self.target).__name__ == "QemuTarget"): + self.assertTrue(self.target.check(), msg = "Qemu not running?") + + self.setUpLocal() + + # a setup method before tests but after the class instantiation + def setUpLocal(self): + pass + + def tearDown(self): + # Uninstall packages in the DUT + self.tc.install_uninstall_packages(self.id(), False) + + res = getResults() + # If a test fails or there is an exception dump + # for QemuTarget only + if (type(self.target).__name__ == "QemuTarget" and + (self.id() in res.getErrorList() or + self.id() in res.getFailList())): + self.tc.host_dumper.create_dir(self._testMethodName) + self.tc.host_dumper.dump_host() + self.target.target_dumper.dump_target( + self.tc.host_dumper.dump_dir) + print ("%s dump data stored in %s" % (self._testMethodName, + self.tc.host_dumper.dump_dir)) + + self.tearDownLocal() + + # Method to be run after tearDown and implemented by child classes + def tearDownLocal(self): + pass def getmodule(pos=2): # stack returns a list of tuples containg frame information @@ -90,3 +154,463 @@ def skipModuleUnless(cond, reason): if not cond: skipModule(reason, 3) + +_buffer_logger = "" +def custom_verbose(msg, *args, **kwargs): + global _buffer_logger + if msg[-1] != "\n": + _buffer_logger += msg + else: + _buffer_logger += msg + try: + bb.plain(_buffer_logger.rstrip("\n"), *args, **kwargs) + except NameError: + logger.info(_buffer_logger.rstrip("\n"), *args, **kwargs) + _buffer_logger = "" + +class TestContext(object): + def __init__(self, d, exported=False): + self.d = d + + self.testsuites = self._get_test_suites() + + if exported: + path = [os.path.dirname(os.path.abspath(__file__))] + extrapath = "" + else: + path = d.getVar("BBPATH").split(':') + extrapath = "lib/oeqa" + + self.testslist = self._get_tests_list(path, extrapath) + self.testsrequired = self._get_test_suites_required() + + self.filesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime/files") + self.corefilesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + self.imagefeatures = d.getVar("IMAGE_FEATURES").split() + self.distrofeatures = d.getVar("DISTRO_FEATURES").split() + + # get testcase list from specified file + # if path is a relative path, then relative to build/conf/ + def _read_testlist(self, fpath, builddir): + if not os.path.isabs(fpath): + fpath = os.path.join(builddir, "conf", fpath) + if not os.path.exists(fpath): + bb.fatal("No such manifest file: ", fpath) + tcs = [] + for line in open(fpath).readlines(): + line = line.strip() + if line and not line.startswith("#"): + tcs.append(line) + return " ".join(tcs) + + # return test list by type also filter if TEST_SUITES is specified + def _get_tests_list(self, bbpath, extrapath): + testslist = [] + + type = self._get_test_namespace() + + # This relies on lib/ under each directory in BBPATH being added to sys.path + # (as done by default in base.bbclass) + for testname in self.testsuites: + if testname != "auto": + if testname.startswith("oeqa."): + testslist.append(testname) + continue + found = False + for p in bbpath: + if os.path.exists(os.path.join(p, extrapath, type, testname + ".py")): + testslist.append("oeqa." + type + "." + testname) + found = True + break + elif os.path.exists(os.path.join(p, extrapath, type, testname.split(".")[0] + ".py")): + testslist.append("oeqa." + type + "." + testname) + found = True + break + if not found: + bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname) + + if "auto" in self.testsuites: + def add_auto_list(path): + files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) + for f in files: + module = 'oeqa.' + type + '.' + f[:-3] + if module not in testslist: + testslist.append(module) + + for p in bbpath: + testpath = os.path.join(p, 'lib', 'oeqa', type) + bb.debug(2, 'Searching for tests in %s' % testpath) + if os.path.exists(testpath): + add_auto_list(testpath) + + return testslist + + def getTestModules(self): + """ + Returns all the test modules in the testlist. + """ + + import pkgutil + + modules = [] + for test in self.testslist: + if re.search("\w+\.\w+\.test_\S+", test): + test = '.'.join(t.split('.')[:3]) + module = pkgutil.get_loader(test) + modules.append(module) + + return modules + + def getModulefromID(self, test_id): + """ + Returns the test module based on a test id. + """ + + module_name = ".".join(test_id.split(".")[:3]) + modules = self.getTestModules() + for module in modules: + if module.name == module_name: + return module + + return None + + def getTests(self, test): + '''Return all individual tests executed when running the suite.''' + # Unfortunately unittest does not have an API for this, so we have + # to rely on implementation details. This only needs to work + # for TestSuite containing TestCase. + method = getattr(test, '_testMethodName', None) + if method: + # leaf case: a TestCase + yield test + else: + # Look into TestSuite. + tests = getattr(test, '_tests', []) + for t1 in tests: + for t2 in self.getTests(t1): + yield t2 + + def loadTests(self): + setattr(oeTest, "tc", self) + + testloader = unittest.TestLoader() + testloader.sortTestMethodsUsing = None + suites = [testloader.loadTestsFromName(name) for name in self.testslist] + suites = filterByTagExp(suites, getattr(self, "tagexp", None)) + + # Determine dependencies between suites by looking for @skipUnlessPassed + # method annotations. Suite A depends on suite B if any method in A + # depends on a method on B. + for suite in suites: + suite.dependencies = [] + suite.depth = 0 + for test in self.getTests(suite): + methodname = getattr(test, '_testMethodName', None) + if methodname: + method = getattr(test, methodname) + depends_on = getattr(method, '_depends_on', None) + if depends_on: + for dep_suite in suites: + if depends_on in [getattr(t, '_testMethodName', None) for t in self.getTests(dep_suite)]: + if dep_suite not in suite.dependencies and \ + dep_suite is not suite: + suite.dependencies.append(dep_suite) + break + else: + logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % + (test, depends_on)) + + # Use brute-force topological sort to determine ordering. Sort by + # depth (higher depth = must run later), with original ordering to + # break ties. + def set_suite_depth(suite): + for dep in suite.dependencies: + new_depth = set_suite_depth(dep) + 1 + if new_depth > suite.depth: + suite.depth = new_depth + return suite.depth + + for index, suite in enumerate(suites): + set_suite_depth(suite) + suite.index = index + + def cmp(a, b): + return (a > b) - (a < b) + + def cmpfunc(a, b): + return cmp((a.depth, a.index), (b.depth, b.index)) + + suites.sort(key=functools.cmp_to_key(cmpfunc)) + + self.suite = testloader.suiteClass(suites) + + return self.suite + + def runTests(self): + logger.info("Test modules %s" % self.testslist) + if hasattr(self, "tagexp") and self.tagexp: + logger.info("Filter test cases by tags: %s" % self.tagexp) + logger.info("Found %s tests" % self.suite.countTestCases()) + runner = unittest.TextTestRunner(verbosity=2) + if 'bb' in sys.modules: + runner.stream.write = custom_verbose + + return runner.run(self.suite) + +class RuntimeTestContext(TestContext): + def __init__(self, d, target, exported=False): + super(RuntimeTestContext, self).__init__(d, exported) + + self.target = target + + self.pkgmanifest = {} + manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), + d.getVar("IMAGE_LINK_NAME") + ".manifest") + nomanifest = d.getVar("IMAGE_NO_MANIFEST") + if nomanifest is None or nomanifest != "1": + try: + with open(manifest) as f: + for line in f: + (pkg, arch, version) = line.strip().split() + self.pkgmanifest[pkg] = (version, arch) + except IOError as e: + bb.fatal("No package manifest file found. Did you build the image?\n%s" % e) + + def _get_test_namespace(self): + return "runtime" + + def _get_test_suites(self): + testsuites = [] + + manifests = (self.d.getVar("TEST_SUITES_MANIFEST") or '').split() + if manifests: + for manifest in manifests: + testsuites.extend(self._read_testlist(manifest, + self.d.getVar("TOPDIR")).split()) + + else: + testsuites = self.d.getVar("TEST_SUITES").split() + + return testsuites + + def _get_test_suites_required(self): + return [t for t in self.d.getVar("TEST_SUITES").split() if t != "auto"] + + def loadTests(self): + super(RuntimeTestContext, self).loadTests() + if oeTest.hasPackage("procps"): + oeRuntimeTest.pscmd = "ps -ef" + + def extract_packages(self): + """ + Find packages that will be needed during runtime. + """ + + modules = self.getTestModules() + bbpaths = self.d.getVar("BBPATH").split(":") + + shutil.rmtree(self.d.getVar("TEST_EXTRACTED_DIR")) + shutil.rmtree(self.d.getVar("TEST_PACKAGED_DIR")) + for module in modules: + json_file = self._getJsonFile(module) + if json_file: + needed_packages = self._getNeededPackages(json_file) + self._perform_package_extraction(needed_packages) + + def _perform_package_extraction(self, needed_packages): + """ + Extract packages that will be needed during runtime. + """ + + import oe.path + + extracted_path = self.d.getVar("TEST_EXTRACTED_DIR") + packaged_path = self.d.getVar("TEST_PACKAGED_DIR") + + for key,value in needed_packages.items(): + packages = () + if isinstance(value, dict): + packages = (value, ) + elif isinstance(value, list): + packages = value + else: + bb.fatal("Failed to process needed packages for %s; " + "Value must be a dict or list" % key) + + for package in packages: + pkg = package["pkg"] + rm = package.get("rm", False) + extract = package.get("extract", True) + if extract: + dst_dir = os.path.join(extracted_path, pkg) + else: + dst_dir = os.path.join(packaged_path) + + # Extract package and copy it to TEST_EXTRACTED_DIR + pkg_dir = self._extract_in_tmpdir(pkg) + if extract: + + # Same package used for more than one test, + # don't need to extract again. + if os.path.exists(dst_dir): + continue + oe.path.copytree(pkg_dir, dst_dir) + shutil.rmtree(pkg_dir) + + # Copy package to TEST_PACKAGED_DIR + else: + self._copy_package(pkg) + + def _getJsonFile(self, module): + """ + Returns the path of the JSON file for a module, empty if doesn't exitst. + """ + + module_file = module.path + json_file = "%s.json" % module_file.rsplit(".", 1)[0] + if os.path.isfile(module_file) and os.path.isfile(json_file): + return json_file + else: + return "" + + def _getNeededPackages(self, json_file, test=None): + """ + Returns a dict with needed packages based on a JSON file. + + + If a test is specified it will return the dict just for that test. + """ + + import json + + needed_packages = {} + + with open(json_file) as f: + test_packages = json.load(f) + for key,value in test_packages.items(): + needed_packages[key] = value + + if test: + if test in needed_packages: + needed_packages = needed_packages[test] + else: + needed_packages = {} + + return needed_packages + + def _extract_in_tmpdir(self, pkg): + """" + Returns path to a temp directory where the package was + extracted without dependencies. + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR"), pkg) + pm = get_package_manager(self.d, pkg_path) + extract_dir = pm.extract(pkg) + shutil.rmtree(pkg_path) + + return extract_dir + + def _copy_package(self, pkg): + """ + Copy the RPM, DEB or IPK package to dst_dir + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR"), pkg) + dst_dir = self.d.getVar("TEST_PACKAGED_DIR") + pm = get_package_manager(self.d, pkg_path) + pkg_info = pm.package_info(pkg) + file_path = pkg_info[pkg]["filepath"] + shutil.copy2(file_path, dst_dir) + shutil.rmtree(pkg_path) + + def install_uninstall_packages(self, test_id, pkg_dir, install): + """ + Check if the test requires a package and Install/Uninstall it in the DUT + """ + + test = test_id.split(".")[4] + module = self.getModulefromID(test_id) + json = self._getJsonFile(module) + if json: + needed_packages = self._getNeededPackages(json, test) + if needed_packages: + self._install_uninstall_packages(needed_packages, pkg_dir, install) + + def _install_uninstall_packages(self, needed_packages, pkg_dir, install=True): + """ + Install/Uninstall packages in the DUT without using a package manager + """ + + if isinstance(needed_packages, dict): + packages = [needed_packages] + elif isinstance(needed_packages, list): + packages = needed_packages + + for package in packages: + pkg = package["pkg"] + rm = package.get("rm", False) + extract = package.get("extract", True) + src_dir = os.path.join(pkg_dir, pkg) + + # Install package + if install and extract: + self.target.connection.copy_dir_to(src_dir, "/") + + # Uninstall package + elif not install and rm: + self.target.connection.delete_dir_structure(src_dir, "/") + +class ImageTestContext(RuntimeTestContext): + def __init__(self, d, target, host_dumper): + super(ImageTestContext, self).__init__(d, target) + + self.tagexp = d.getVar("TEST_SUITES_TAGS") + + self.host_dumper = host_dumper + + self.sigterm = False + self.origsigtermhandler = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, self._sigterm_exception) + + def _sigterm_exception(self, signum, stackframe): + bb.warn("TestImage received SIGTERM, shutting down...") + self.sigterm = True + self.target.stop() + + def install_uninstall_packages(self, test_id, install=True): + """ + Check if the test requires a package and Install/Uninstall it in the DUT + """ + + pkg_dir = self.d.getVar("TEST_EXTRACTED_DIR") + super(ImageTestContext, self).install_uninstall_packages(test_id, pkg_dir, install) + +class ExportTestContext(RuntimeTestContext): + def __init__(self, d, target, exported=False, parsedArgs={}): + """ + This class is used when exporting tests and when are executed outside OE environment. + + parsedArgs can contain the following: + - tag: Filter test by tag. + """ + super(ExportTestContext, self).__init__(d, target, exported) + + tag = parsedArgs.get("tag", None) + self.tagexp = tag if tag != None else d.getVar("TEST_SUITES_TAGS") + + self.sigterm = None + + def install_uninstall_packages(self, test_id, install=True): + """ + Check if the test requires a package and Install/Uninstall it in the DUT + """ + + export_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + extracted_dir = self.d.getVar("TEST_EXPORT_EXTRACTED_DIR") + pkg_dir = os.path.join(export_dir, extracted_dir) + super(ExportTestContext, self).install_uninstall_packages(test_id, pkg_dir, install) diff --git a/meta/lib/oeqa/runexported.py b/meta/lib/oeqa/runexported.py index e1b6642ec2..9cfea0f7ab 100755 --- a/meta/lib/oeqa/runexported.py +++ b/meta/lib/oeqa/runexported.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) 2013 Intel Corporation @@ -21,7 +21,7 @@ import sys import os import time -from optparse import OptionParser +import argparse try: import simplejson as json @@ -30,7 +30,8 @@ except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa"))) -from oeqa.oetest import runTests +from oeqa.oetest import ExportTestContext +from oeqa.utils.commands import runCmd, updateEnv from oeqa.utils.sshcontrol import SSHControl # this isn't pretty but we need a fake target object @@ -42,14 +43,14 @@ class FakeTarget(object): self.ip = None self.server_ip = None self.datetime = time.strftime('%Y%m%d%H%M%S',time.gmtime()) - self.testdir = d.getVar("TEST_LOG_DIR", True) - self.pn = d.getVar("PN", True) + self.testdir = d.getVar("TEST_LOG_DIR") + self.pn = d.getVar("PN") def exportStart(self): self.sshlog = os.path.join(self.testdir, "ssh_target_log.%s" % self.datetime) sshloglink = os.path.join(self.testdir, "ssh_target_log") - if os.path.islink(sshloglink): - os.unlink(sshloglink) + if os.path.lexists(sshloglink): + os.remove(sshloglink) os.symlink(self.sshlog, sshloglink) print("SSH log file: %s" % self.sshlog) self.connection = SSHControl(self.ip, logfile=self.sshlog) @@ -68,73 +69,85 @@ class MyDataDict(dict): def getVar(self, key, unused = None): return self.get(key, "") -class TestContext(object): - def __init__(self): - self.d = None - self.target = None - def main(): - usage = "usage: %prog [options] <json file>" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--target-ip", dest="ip", help="The IP address of the target machine. Use this to \ + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--target-ip", dest="ip", help="The IP address of the target machine. Use this to \ overwrite the value determined from TEST_TARGET_IP at build time") - parser.add_option("-s", "--server-ip", dest="server_ip", help="The IP address of this machine. Use this to \ + parser.add_argument("-s", "--server-ip", dest="server_ip", help="The IP address of this machine. Use this to \ overwrite the value determined from TEST_SERVER_IP at build time.") - parser.add_option("-d", "--deploy-dir", dest="deploy_dir", help="Full path to the package feeds, that this \ + parser.add_argument("-d", "--deploy-dir", dest="deploy_dir", help="Full path to the package feeds, that this \ the contents of what used to be DEPLOY_DIR on the build machine. If not specified it will use the value \ specified in the json if that directory actually exists or it will error out.") - parser.add_option("-l", "--log-dir", dest="log_dir", help="This sets the path for TEST_LOG_DIR. If not specified \ + parser.add_argument("-l", "--log-dir", dest="log_dir", help="This sets the path for TEST_LOG_DIR. If not specified \ the current dir is used. This is used for usually creating a ssh log file and a scp test file.") + parser.add_argument("-a", "--tag", dest="tag", help="Only run test with specified tag.") + parser.add_argument("json", help="The json file exported by the build system", default="testdata.json", nargs='?') - (options, args) = parser.parse_args() - if len(args) != 1: - parser.error("Incorrect number of arguments. The one and only argument should be a json file exported by the build system") + args = parser.parse_args() - with open(args[0], "r") as f: + with open(args.json, "r") as f: loaded = json.load(f) - if options.ip: - loaded["target"]["ip"] = options.ip - if options.server_ip: - loaded["target"]["server_ip"] = options.server_ip + if args.ip: + loaded["target"]["ip"] = args.ip + if args.server_ip: + loaded["target"]["server_ip"] = args.server_ip d = MyDataDict() for key in loaded["d"].keys(): d[key] = loaded["d"][key] - if options.log_dir: - d["TEST_LOG_DIR"] = options.log_dir + if args.log_dir: + d["TEST_LOG_DIR"] = args.log_dir else: d["TEST_LOG_DIR"] = os.path.abspath(os.path.dirname(__file__)) - if options.deploy_dir: - d["DEPLOY_DIR"] = options.deploy_dir + if args.deploy_dir: + d["DEPLOY_DIR"] = args.deploy_dir else: if not os.path.isdir(d["DEPLOY_DIR"]): - raise Exception("The path to DEPLOY_DIR does not exists: %s" % d["DEPLOY_DIR"]) + print("WARNING: The path to DEPLOY_DIR does not exist: %s" % d["DEPLOY_DIR"]) + + parsedArgs = {} + parsedArgs["tag"] = args.tag + extract_sdk(d) target = FakeTarget(d) for key in loaded["target"].keys(): setattr(target, key, loaded["target"][key]) - tc = TestContext() - setattr(tc, "d", d) - setattr(tc, "target", target) - for key in loaded.keys(): - if key != "d" and key != "target": - setattr(tc, key, loaded[key]) - target.exportStart() - runTests(tc) + tc = ExportTestContext(d, target, True, parsedArgs) + tc.loadTests() + tc.runTests() return 0 +def extract_sdk(d): + """ + Extract SDK if needed + """ + + export_dir = os.path.dirname(os.path.realpath(__file__)) + tools_dir = d.getVar("TEST_EXPORT_SDK_DIR") + tarball_name = "%s.sh" % d.getVar("TEST_EXPORT_SDK_NAME") + tarball_path = os.path.join(export_dir, tools_dir, tarball_name) + extract_path = os.path.join(export_dir, "sysroot") + if os.path.isfile(tarball_path): + print ("Found SDK tarball %s. Extracting..." % tarball_path) + result = runCmd("%s -y -d %s" % (tarball_path, extract_path)) + for f in os.listdir(extract_path): + if f.startswith("environment-setup"): + print("Setting up SDK environment...") + env_file = os.path.join(extract_path, f) + updateEnv(env_file) + if __name__ == "__main__": try: ret = main() except Exception: ret = 1 import traceback - traceback.print_exc(5) + traceback.print_exc() sys.exit(ret) diff --git a/meta/lib/oeqa/runtime/__init__.py b/meta/lib/oeqa/runtime/__init__.py deleted file mode 100644 index 4cf3fa76b6..0000000000 --- a/meta/lib/oeqa/runtime/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Enable other layers to have tests in the same named directory -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/runtime/buildcvs.py b/meta/lib/oeqa/runtime/buildcvs.py deleted file mode 100644 index fe6cbfbcd5..0000000000 --- a/meta/lib/oeqa/runtime/buildcvs.py +++ /dev/null @@ -1,31 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class BuildCvsTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2") - self.project.download_archive() - - @testcase(205) - @skipUnlessPassed("test_ssh") - def test_cvs(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - self.assertEqual(self.project.run_install(), 0, - msg="Running make install failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/runtime/buildiptables.py b/meta/lib/oeqa/runtime/buildiptables.py deleted file mode 100644 index 09e252df8c..0000000000 --- a/meta/lib/oeqa/runtime/buildiptables.py +++ /dev/null @@ -1,31 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class BuildIptablesTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") - self.project.download_archive() - - @testcase(206) - @skipUnlessPassed("test_ssh") - def test_iptables(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - self.assertEqual(self.project.run_install(), 0, - msg="Running make install failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/runtime/buildsudoku.py b/meta/lib/oeqa/runtime/buildsudoku.py deleted file mode 100644 index 802b060010..0000000000 --- a/meta/lib/oeqa/runtime/buildsudoku.py +++ /dev/null @@ -1,28 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class SudokuTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") - self.project.download_archive() - - @testcase(207) - @skipUnlessPassed("test_ssh") - def test_sudoku(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/runtime/case.py b/meta/lib/oeqa/runtime/case.py new file mode 100644 index 0000000000..c1485c9860 --- /dev/null +++ b/meta/lib/oeqa/runtime/case.py @@ -0,0 +1,17 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.utils.package_manager import install_package, uninstall_package + +class OERuntimeTestCase(OETestCase): + # target instance set by OERuntimeTestLoader. + target = None + + def _oeSetUp(self): + super(OERuntimeTestCase, self)._oeSetUp() + install_package(self) + + def _oeTearDown(self): + super(OERuntimeTestCase, self)._oeTearDown() + uninstall_package(self) diff --git a/meta/lib/oeqa/runtime/cases/_ptest.py b/meta/lib/oeqa/runtime/cases/_ptest.py new file mode 100644 index 0000000000..aaed9a5352 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/_ptest.py @@ -0,0 +1,103 @@ +import os +import shutil +import subprocess + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature +from oeqa.runtime.decorator.package import OEHasPackage + +from oeqa.runtime.cases.dnf import DnfTest +from oeqa.utils.logparser import * +from oeqa.utils.httpserver import HTTPService + +class PtestRunnerTest(DnfTest): + + @classmethod + def setUpClass(cls): + rpm_deploy = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm') + cls.repo_server = HTTPService(rpm_deploy, cls.tc.target.server_ip) + cls.repo_server.start() + + @classmethod + def tearDownClass(cls): + cls.repo_server.stop() + + # a ptest log parser + def parse_ptest(self, logfile): + parser = Lparser(test_0_pass_regex="^PASS:(.+)", + test_0_fail_regex="^FAIL:(.+)", + section_0_begin_regex="^BEGIN: .*/(.+)/ptest", + section_0_end_regex="^END: .*/(.+)/ptest") + parser.init() + result = Result() + + with open(logfile, errors='replace') as f: + for line in f: + result_tuple = parser.parse_line(line) + if not result_tuple: + continue + result_tuple = line_type, category, status, name = parser.parse_line(line) + + if line_type == 'section' and status == 'begin': + current_section = name + continue + + if line_type == 'section' and status == 'end': + current_section = None + continue + + if line_type == 'test' and status == 'pass': + result.store(current_section, name, status) + continue + + if line_type == 'test' and status == 'fail': + result.store(current_section, name, status) + continue + + result.sort_tests() + return result + + def _install_ptest_packages(self): + # Get ptest packages that can be installed in the image. + packages_dir = os.path.join(self.tc.td['DEPLOY_DIR'], 'rpm') + ptest_pkgs = [pkg[:pkg.find('-ptest')+6] + for _, _, filenames in os.walk(packages_dir) + for pkg in filenames + if 'ptest' in pkg + and pkg[:pkg.find('-ptest')] in self.tc.image_packages] + + repo_url = 'http://%s:%s' % (self.target.server_ip, + self.repo_server.port) + dnf_options = ('--repofrompath=oe-ptest-repo,%s ' + '--nogpgcheck ' + 'install -y' % repo_url) + self.dnf('%s %s ptest-runner' % (dnf_options, ' '.join(ptest_pkgs))) + + @skipIfNotFeature('package-management', + 'Test requires package-management to be in DISTRO_FEATURES') + @skipIfNotFeature('ptest', + 'Test requires package-management to be in DISTRO_FEATURES') + @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm', + 'RPM is not the primary package manager') + @OEHasPackage(['dnf']) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_ptestrunner(self): + self.ptest_log = os.path.join(self.tc.td['TEST_LOG_DIR'], + 'ptest-%s.log' % self.tc.td['DATETIME']) + self._install_ptest_packages() + + (runnerstatus, result) = self.target.run('/usr/bin/ptest-runner > /tmp/ptest.log 2>&1', 0) + #exit code is !=0 even if ptest-runner executes because some ptest tests fail. + self.assertTrue(runnerstatus != 127, msg="Cannot execute ptest-runner!") + self.target.copyFrom('/tmp/ptest.log', self.ptest_log) + shutil.copyfile(self.ptest_log, "ptest.log") + + result = self.parse_ptest("ptest.log") + log_results_to_location = "./results" + if os.path.exists(log_results_to_location): + shutil.rmtree(log_results_to_location) + os.makedirs(log_results_to_location) + + result.log_as_files(log_results_to_location, test_status = ['pass','fail']) diff --git a/meta/lib/oeqa/runtime/cases/_qemutiny.py b/meta/lib/oeqa/runtime/cases/_qemutiny.py new file mode 100644 index 0000000000..7b5b48141f --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/_qemutiny.py @@ -0,0 +1,8 @@ +from oeqa.runtime.case import OERuntimeTestCase + +class QemuTinyTest(OERuntimeTestCase): + + def test_boot_tiny(self): + status, output = self.target.run_serial('uname -a') + msg = "Cannot detect poky tiny boot!" + self.assertTrue("yocto-tiny" in output, msg) diff --git a/meta/lib/oeqa/runtime/cases/buildcpio.py b/meta/lib/oeqa/runtime/cases/buildcpio.py new file mode 100644 index 0000000000..59edc9c2c1 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/buildcpio.py @@ -0,0 +1,30 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +from oeqa.runtime.utils.targetbuildproject import TargetBuildProject + +class BuildCpioTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + uri = 'https://ftp.gnu.org/gnu/cpio' + uri = '%s/cpio-2.12.tar.bz2' % uri + cls.project = TargetBuildProject(cls.tc.target, + uri, + dl_dir = cls.tc.td['DL_DIR']) + cls.project.download_archive() + + @classmethod + def tearDownClass(cls): + cls.project.clean() + + @OETestID(205) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_cpio(self): + self.project.run_configure() + self.project.run_make() + self.project.run_install() diff --git a/meta/lib/oeqa/runtime/cases/buildgalculator.py b/meta/lib/oeqa/runtime/cases/buildgalculator.py new file mode 100644 index 0000000000..7c9d4a392b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/buildgalculator.py @@ -0,0 +1,28 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +from oeqa.runtime.utils.targetbuildproject import TargetBuildProject + +class GalculatorTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + uri = 'http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2' + cls.project = TargetBuildProject(cls.tc.target, + uri, + dl_dir = cls.tc.td['DL_DIR']) + cls.project.download_archive() + + @classmethod + def tearDownClass(cls): + cls.project.clean() + + @OETestID(1526) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_galculator(self): + self.project.run_configure() + self.project.run_make() diff --git a/meta/lib/oeqa/runtime/cases/buildiptables.py b/meta/lib/oeqa/runtime/cases/buildiptables.py new file mode 100644 index 0000000000..002b16c483 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/buildiptables.py @@ -0,0 +1,34 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +from oeqa.runtime.utils.targetbuildproject import TargetBuildProject + +class BuildIptablesTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + uri = 'http://downloads.yoctoproject.org/mirror/sources' + uri = '%s/iptables-1.4.13.tar.bz2' % uri + cls.project = TargetBuildProject(cls.tc.target, + uri, + dl_dir = cls.tc.td['DL_DIR']) + cls.project.download_archive() + + @classmethod + def tearDownClass(cls): + cls.project.clean() + + @OETestID(206) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_iptables(self): + self.project.run_configure() + self.project.run_make() + self.project.run_install() + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/runtime/cases/connman.py b/meta/lib/oeqa/runtime/cases/connman.py new file mode 100644 index 0000000000..12456b4172 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/connman.py @@ -0,0 +1,30 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class ConnmanTest(OERuntimeTestCase): + + def service_status(self, service): + if 'systemd' in self.tc.td['DISTRO_FEATURES']: + (_, output) = self.target.run('systemctl status -l %s' % service) + return output + else: + return "Unable to get status or logs for %s" % service + + @OETestID(961) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(["connman"]) + def test_connmand_help(self): + (status, output) = self.target.run('/usr/sbin/connmand --help') + msg = 'Failed to get connman help. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(221) + @OETestDepends(['connman.ConnmanTest.test_connmand_help']) + def test_connmand_running(self): + cmd = '%s | grep [c]onnmand' % self.tc.target_cmds['ps'] + (status, output) = self.target.run(cmd) + if status != 0: + self.logger.info(self.service_status("connman")) + self.fail("No connmand process running") diff --git a/meta/lib/oeqa/runtime/cases/date.py b/meta/lib/oeqa/runtime/cases/date.py new file mode 100644 index 0000000000..ece7338de7 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/date.py @@ -0,0 +1,38 @@ +import re + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class DateTest(OERuntimeTestCase): + + def setUp(self): + if self.tc.td.get('VIRTUAL-RUNTIME_init_manager') == 'systemd': + self.logger.debug('Stopping systemd-timesyncd daemon') + self.target.run('systemctl stop systemd-timesyncd') + + def tearDown(self): + if self.tc.td.get('VIRTUAL-RUNTIME_init_manager') == 'systemd': + self.logger.debug('Starting systemd-timesyncd daemon') + self.target.run('systemctl start systemd-timesyncd') + + @OETestID(211) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_date(self): + (status, output) = self.target.run('date +"%Y-%m-%d %T"') + msg = 'Failed to get initial date, output: %s' % output + self.assertEqual(status, 0, msg=msg) + oldDate = output + + sampleDate = '"2016-08-09 10:00:00"' + (status, output) = self.target.run("date -s %s" % sampleDate) + self.assertEqual(status, 0, msg='Date set failed, output: %s' % output) + + (status, output) = self.target.run("date -R") + p = re.match('Tue, 09 Aug 2016 10:00:.. \+0000', output) + msg = 'The date was not set correctly, output: %s' % output + self.assertTrue(p, msg=msg) + + (status, output) = self.target.run('date -s "%s"' % oldDate) + msg = 'Failed to reset date, output: %s' % output + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/df.py b/meta/lib/oeqa/runtime/cases/df.py new file mode 100644 index 0000000000..aecc32d7ce --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/df.py @@ -0,0 +1,13 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class DfTest(OERuntimeTestCase): + + @OETestID(234) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_df(self): + cmd = "df / | sed -n '2p' | awk '{print $4}'" + (status,output) = self.target.run(cmd) + msg = 'Not enough space on image. Current size is %s' % output + self.assertTrue(int(output)>5120, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/dnf.py b/meta/lib/oeqa/runtime/cases/dnf.py new file mode 100644 index 0000000000..2f87296b4e --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/dnf.py @@ -0,0 +1,123 @@ +import os +import re +import subprocess +from oeqa.utils.httpserver import HTTPService + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature +from oeqa.runtime.decorator.package import OEHasPackage + +class DnfTest(OERuntimeTestCase): + + def dnf(self, command, expected = 0): + command = 'dnf %s' % command + status, output = self.target.run(command, 1500) + message = os.linesep.join([command, output]) + self.assertEqual(status, expected, message) + return output + +class DnfBasicTest(DnfTest): + + @skipIfNotFeature('package-management', + 'Test requires package-management to be in IMAGE_FEATURES') + @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm', + 'RPM is not the primary package manager') + @OEHasPackage(['dnf']) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OETestID(1735) + def test_dnf_help(self): + self.dnf('--help') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1739) + def test_dnf_version(self): + self.dnf('--version') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1737) + def test_dnf_info(self): + self.dnf('info dnf') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1738) + def test_dnf_search(self): + self.dnf('search dnf') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1736) + def test_dnf_history(self): + self.dnf('history') + +class DnfRepoTest(DnfTest): + + @classmethod + def setUpClass(cls): + cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-testimage-repo'), + cls.tc.target.server_ip) + cls.repo_server.start() + + @classmethod + def tearDownClass(cls): + cls.repo_server.stop() + + def dnf_with_repo(self, command): + pkgarchs = os.listdir(os.path.join(self.tc.td['WORKDIR'], 'oe-testimage-repo')) + deploy_url = 'http://%s:%s/' %(self.target.server_ip, self.repo_server.port) + cmdlinerepoopts = ["--repofrompath=oe-testimage-repo-%s,%s%s" %(arch, deploy_url, arch) for arch in pkgarchs] + + self.dnf(" ".join(cmdlinerepoopts) + " --nogpgcheck " + command) + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1744) + def test_dnf_makecache(self): + self.dnf_with_repo('makecache') + + +# Does not work when repo is specified on the command line +# @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) +# def test_dnf_repolist(self): +# self.dnf_with_repo('repolist') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) + @OETestID(1746) + def test_dnf_repoinfo(self): + self.dnf_with_repo('repoinfo') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) + @OETestID(1740) + def test_dnf_install(self): + self.dnf_with_repo('install -y run-postinsts-dev') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install']) + @OETestID(1741) + def test_dnf_install_dependency(self): + self.dnf_with_repo('remove -y run-postinsts') + self.dnf_with_repo('install -y run-postinsts-dev') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install_dependency']) + @OETestID(1742) + def test_dnf_install_from_disk(self): + self.dnf_with_repo('remove -y run-postinsts-dev') + self.dnf_with_repo('install -y --downloadonly run-postinsts-dev') + status, output = self.target.run('find /var/cache/dnf -name run-postinsts-dev*rpm', 1500) + self.assertEqual(status, 0, output) + self.dnf_with_repo('install -y %s' % output) + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install_from_disk']) + @OETestID(1743) + def test_dnf_install_from_http(self): + output = subprocess.check_output('%s %s -name run-postinsts-dev*' % (bb.utils.which(os.getenv('PATH'), "find"), + os.path.join(self.tc.td['WORKDIR'], 'oe-testimage-repo')), shell=True).decode("utf-8") + rpm_path = output.split("/")[-2] + "/" + output.split("/")[-1] + url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, rpm_path) + self.dnf_with_repo('remove -y run-postinsts-dev') + self.dnf_with_repo('install -y %s' % url) + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install']) + @OETestID(1745) + def test_dnf_reinstall(self): + self.dnf_with_repo('reinstall -y run-postinsts-dev') + + diff --git a/meta/lib/oeqa/runtime/cases/gcc.py b/meta/lib/oeqa/runtime/cases/gcc.py new file mode 100644 index 0000000000..911083156f --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/gcc.py @@ -0,0 +1,73 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class GccCompileTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + dst = '/tmp/' + src = os.path.join(cls.tc.files_dir, 'test.c') + cls.tc.target.copyTo(src, dst) + + src = os.path.join(cls.tc.runtime_files_dir, 'testmakefile') + cls.tc.target.copyTo(src, dst) + + src = os.path.join(cls.tc.files_dir, 'test.cpp') + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + files = '/tmp/test.c /tmp/test.o /tmp/test /tmp/testmakefile' + cls.tc.target.run('rm %s' % files) + + @OETestID(203) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_gcc_compile(self): + status, output = self.target.run('gcc /tmp/test.c -o /tmp/test -lm') + msg = 'gcc compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(200) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_gpp_compile(self): + status, output = self.target.run('g++ /tmp/test.c -o /tmp/test -lm') + msg = 'g++ compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(1142) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_gpp2_compile(self): + status, output = self.target.run('g++ /tmp/test.cpp -o /tmp/test -lm') + msg = 'g++ compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(204) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_make(self): + status, output = self.target.run('cd /tmp; make -f testmakefile') + msg = 'running make failed, output %s' % output + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/kernelmodule.py b/meta/lib/oeqa/runtime/cases/kernelmodule.py new file mode 100644 index 0000000000..11ad7b7f01 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/kernelmodule.py @@ -0,0 +1,40 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class KernelModuleTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + src = os.path.join(cls.tc.runtime_files_dir, 'hellomod.c') + dst = '/tmp/hellomod.c' + cls.tc.target.copyTo(src, dst) + + src = os.path.join(cls.tc.runtime_files_dir, 'hellomod_makefile') + dst = '/tmp/Makefile' + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + files = '/tmp/Makefile /tmp/hellomod.c' + cls.tc.target.run('rm %s' % files) + + @OETestID(1541) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['gcc.GccCompileTest.test_gcc_compile']) + def test_kernel_module(self): + cmds = [ + 'cd /usr/src/kernel && make scripts', + 'cd /tmp && make', + 'cd /tmp && insmod hellomod.ko', + 'lsmod | grep hellomod', + 'dmesg | grep Hello', + 'rmmod hellomod', 'dmesg | grep "Cleaning up hellomod"' + ] + for cmd in cmds: + status, output = self.target.run(cmd, 900) + self.assertEqual(status, 0, msg='\n'.join([cmd, output])) diff --git a/meta/lib/oeqa/runtime/cases/ldd.py b/meta/lib/oeqa/runtime/cases/ldd.py new file mode 100644 index 0000000000..c6d92fd5af --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/ldd.py @@ -0,0 +1,25 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class LddTest(OERuntimeTestCase): + + @OETestID(962) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_ldd_exists(self): + status, output = self.target.run('which ldd') + msg = 'ldd does not exist in PATH: which ldd: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(239) + @OETestDepends(['ldd.LddTest.test_ldd_exists']) + def test_ldd_rtldlist_check(self): + cmd = ('for i in $(which ldd | xargs cat | grep "^RTLDLIST"| ' + 'cut -d\'=\' -f2|tr -d \'"\'); ' + 'do test -f $i && echo $i && break; done') + status, output = self.target.run(cmd) + msg = "ldd path not correct or RTLDLIST files don't exist." + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/logrotate.py b/meta/lib/oeqa/runtime/cases/logrotate.py new file mode 100644 index 0000000000..992fef2989 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/logrotate.py @@ -0,0 +1,42 @@ +# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=289 testcase +# Note that the image under test must have logrotate installed + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class LogrotateTest(OERuntimeTestCase): + + @classmethod + def tearDownClass(cls): + cls.tc.target.run('rm -rf $HOME/logrotate_dir') + + @OETestID(1544) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['logrotate']) + def test_1_logrotate_setup(self): + status, output = self.target.run('mkdir $HOME/logrotate_dir') + msg = 'Could not create logrotate_dir. Output: %s' % output + self.assertEqual(status, 0, msg = msg) + + cmd = ('sed -i "s#wtmp {#wtmp {\\n olddir $HOME/logrotate_dir#"' + ' /etc/logrotate.conf') + status, output = self.target.run(cmd) + msg = ('Could not write to logrotate.conf file. Status and output: ' + ' %s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + @OETestID(1542) + @OETestDepends(['logrotate.LogrotateTest.test_1_logrotate_setup']) + def test_2_logrotate(self): + status, output = self.target.run('logrotate -f /etc/logrotate.conf') + msg = ('logrotate service could not be reloaded. Status and output: ' + '%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + _, output = self.target.run('ls -la $HOME/logrotate_dir/ | wc -l') + msg = ('new logfile could not be created. List of files within log ' + 'directory: %s' % ( + self.target.run('ls -la $HOME/logrotate_dir')[1])) + self.assertTrue(int(output)>=3, msg = msg) diff --git a/meta/lib/oeqa/runtime/cases/multilib.py b/meta/lib/oeqa/runtime/cases/multilib.py new file mode 100644 index 0000000000..8c167f1008 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/multilib.py @@ -0,0 +1,41 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotInDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +class MultilibTest(OERuntimeTestCase): + + def archtest(self, binary, arch): + """ + Check that ``binary`` has the ELF class ``arch`` (e.g. ELF32/ELF64). + """ + + status, output = self.target.run('readelf -h %s' % binary) + self.assertEqual(status, 0, 'Failed to readelf %s' % binary) + + l = [l.split()[1] for l in output.split('\n') if "Class:" in l] + if l: + theclass = l[0] + else: + self.fail('Cannot parse readelf. Output:\n%s' % output) + + msg = "%s isn't %s (is %s)" % (binary, arch, theclass) + self.assertEqual(theclass, arch, msg=msg) + + @OETestID(1593) + @skipIfNotInDataVar('MULTILIBS', 'multilib:lib32', + "This isn't a multilib:lib32 image") + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_check_multilib_libc(self): + """ + Check that a multilib image has both 32-bit and 64-bit libc in. + """ + self.archtest("/lib/libc.so.6", "ELF32") + self.archtest("/lib64/libc.so.6", "ELF64") + + @OETestID(279) + @OETestDepends(['multilib.MultilibTest.test_check_multilib_libc']) + @OEHasPackage(['lib32-connman']) + def test_file_connman(self): + self.archtest("/usr/sbin/connmand", "ELF32") diff --git a/meta/lib/oeqa/runtime/cases/oe_syslog.py b/meta/lib/oeqa/runtime/cases/oe_syslog.py new file mode 100644 index 0000000000..005b6978d9 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/oe_syslog.py @@ -0,0 +1,66 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +class SyslogTest(OERuntimeTestCase): + + @OETestID(201) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(["busybox-syslog", "sysklogd"]) + def test_syslog_running(self): + cmd = '%s | grep -i [s]yslogd' % self.tc.target_cmds['ps'] + status, output = self.target.run(cmd) + msg = "No syslogd process; ps output: %s" % output + self.assertEqual(status, 0, msg=msg) + +class SyslogTestConfig(OERuntimeTestCase): + + @OETestID(1149) + @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running']) + def test_syslog_logger(self): + status, output = self.target.run('logger foobar') + msg = "Can't log into syslog. Output: %s " % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('grep foobar /var/log/messages') + if status != 0: + if self.tc.td.get("VIRTUAL-RUNTIME_init_manager") == "systemd": + status, output = self.target.run('journalctl -o cat | grep foobar') + else: + status, output = self.target.run('logread | grep foobar') + msg = ('Test log string not found in /var/log/messages or logread.' + ' Output: %s ' % output) + self.assertEqual(status, 0, msg=msg) + + @OETestID(1150) + @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running']) + def test_syslog_restart(self): + if "systemd" != self.tc.td.get("VIRTUAL-RUNTIME_init_manager", ""): + (_, _) = self.target.run('/etc/init.d/syslog restart') + else: + (_, _) = self.target.run('systemctl restart syslog.service') + + + @OETestID(202) + @OETestDepends(['oe_syslog.SyslogTestConfig.test_syslog_logger']) + @OEHasPackage(["!sysklogd", "busybox"]) + @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', + 'Not appropiate for systemd image') + def test_syslog_startup_config(self): + cmd = 'echo "LOGFILE=/var/log/test" >> /etc/syslog-startup.conf' + self.target.run(cmd) + status, output = self.target.run('/etc/init.d/syslog restart') + msg = ('Could not restart syslog service. Status and output:' + ' %s and %s' % (status,output)) + self.assertEqual(status, 0, msg) + + cmd = 'logger foobar && grep foobar /var/log/test' + status,output = self.target.run(cmd) + msg = 'Test log string not found. Output: %s ' % output + self.assertEqual(status, 0, msg=msg) + + cmd = "sed -i 's#LOGFILE=/var/log/test##' /etc/syslog-startup.conf" + self.target.run(cmd) + self.target.run('/etc/init.d/syslog restart') diff --git a/meta/lib/oeqa/runtime/cases/pam.py b/meta/lib/oeqa/runtime/cases/pam.py new file mode 100644 index 0000000000..3654cdc946 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/pam.py @@ -0,0 +1,33 @@ +# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=287 testcase +# Note that the image under test must have "pam" in DISTRO_FEATURES + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class PamBasicTest(OERuntimeTestCase): + + @OETestID(1543) + @skipIfNotFeature('pam', 'Test requires pam to be in DISTRO_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_pam(self): + status, output = self.target.run('login --help') + msg = ('login command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 1, msg = msg) + + status, output = self.target.run('passwd --help') + msg = ('passwd command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + status, output = self.target.run('su --help') + msg = ('su command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + status, output = self.target.run('useradd --help') + msg = ('useradd command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py new file mode 100644 index 0000000000..6e929469c4 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -0,0 +1,359 @@ +import os + +from subprocess import check_output +from shutil import rmtree +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +#in the future these lists could be moved outside of module +errors = ["error", "cannot", "can\'t", "failed"] + +common_errors = [ + "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", + "dma timeout", + "can\'t add hid device:", + "usbhid: probe of ", + "_OSC failed (AE_ERROR)", + "_OSC failed (AE_SUPPORT)", + "AE_ALREADY_EXISTS", + "ACPI _OSC request failed (AE_SUPPORT)", + "can\'t disable ASPM", + "Failed to load module \"vesa\"", + "Failed to load module vesa", + "Failed to load module \"modesetting\"", + "Failed to load module modesetting", + "Failed to load module \"glx\"", + "Failed to load module \"fbdev\"", + "Failed to load module fbdev", + "Failed to load module glx", + "[drm] Cannot find any crtc or sizes - going 1024x768", + "_OSC failed (AE_NOT_FOUND); disabling ASPM", + "Open ACPI failed (/var/run/acpid.socket) (No such file or directory)", + "NX (Execute Disable) protection cannot be enabled: non-PAE kernel!", + "hd.: possibly failed opcode", + 'NETLINK INITIALIZATION FAILED', + 'kernel: Cannot find map file', + 'omap_hwmod: debugss: _wait_target_disable failed', + 'VGA arbiter: cannot open kernel arbiter, no multi-card support', + 'Failed to find URL:http://ipv4.connman.net/online/status.html', + 'Online check failed for', + 'netlink init failed', + 'Fast TSC calibration', + "BAR 0-9", + "Failed to load module \"ati\"", + "controller can't do DEVSLP, turning off", + "stmmac_dvr_probe: warning: cannot get CSR clock", + "error: couldn\'t mount because of unsupported optional features", + "GPT: Use GNU Parted to correct GPT errors", + "Cannot set xattr user.Librepo.DownloadInProgress", + ] + +video_related = [ + "uvesafb", +] + +x86_common = [ + '[drm:psb_do_init] *ERROR* Debug is', + 'wrong ELF class', + 'Could not enable PowerButton event', + 'probe of LNXPWRBN:00 failed with error -22', + 'pmd_set_huge: Cannot satisfy', + 'failed to setup card detect gpio', + 'amd_nb: Cannot enumerate AMD northbridges', + 'failed to retrieve link info, disabling eDP', + 'Direct firmware load for iwlwifi', +] + common_errors + +qemux86_common = [ + 'wrong ELF class', + "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", + "can't claim BAR ", + 'amd_nb: Cannot enumerate AMD northbridges', + 'uvesafb: 5000 ms task timeout, infinitely waiting', + 'tsc: HPET/PMTIMER calibration failed', +] + common_errors + +ignore_errors = { + 'default' : common_errors, + 'qemux86' : [ + 'Failed to access perfctr msr (MSR', + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', + ] + qemux86_common, + 'qemux86-64' : qemux86_common, + 'qemumips' : [ + 'Failed to load module "glx"', + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', + ] + common_errors, + 'qemumips64' : [ + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', + ] + common_errors, + 'qemuppc' : [ + 'PCI 0000:00 Cannot reserve Legacy IO [io 0x0000-0x0fff]', + 'host side 80-wire cable detection failed, limiting max speed', + 'mode "640x480" test failed', + 'Failed to load module "glx"', + 'can\'t handle BAR above 4GB', + 'Cannot reserve Legacy IO', + ] + common_errors, + 'qemuarm' : [ + 'mmci-pl18x: probe of fpga:05 failed with error -22', + 'mmci-pl18x: probe of fpga:0b failed with error -22', + 'Failed to load module "glx"', + 'OF: amba_device_add() failed (-19) for /amba/smc@10100000', + 'OF: amba_device_add() failed (-19) for /amba/mpmc@10110000', + 'OF: amba_device_add() failed (-19) for /amba/sctl@101e0000', + 'OF: amba_device_add() failed (-19) for /amba/watchdog@101e1000', + 'OF: amba_device_add() failed (-19) for /amba/sci@101f0000', + 'OF: amba_device_add() failed (-19) for /amba/ssp@101f4000', + 'OF: amba_device_add() failed (-19) for /amba/fpga/sci@a000', + 'Failed to initialize \'/amba/timer@101e3000\': -22', + 'jitterentropy: Initialization failed with host not compliant with requirements: 2', + ] + common_errors, + 'qemuarm64' : [ + 'Fatal server error:', + '(EE) Server terminated with error (1). Closing log file.', + 'dmi: Firmware registration failed.', + 'irq: type mismatch, failed to map hwirq-27 for /intc', + ] + common_errors, + 'emenlow' : [ + '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', + '(EE) Failed to load module "psb"', + '(EE) Failed to load module psb', + '(EE) Failed to load module "psbdrv"', + '(EE) Failed to load module psbdrv', + '(EE) open /dev/fb0: No such file or directory', + '(EE) AIGLX: reverting to software rendering', + ] + x86_common, + 'intel-core2-32' : [ + 'ACPI: No _BQC method, cannot determine initial brightness', + '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', + '(EE) Failed to load module "psb"', + '(EE) Failed to load module psb', + '(EE) Failed to load module "psbdrv"', + '(EE) Failed to load module psbdrv', + '(EE) open /dev/fb0: No such file or directory', + '(EE) AIGLX: reverting to software rendering', + 'dmi: Firmware registration failed.', + 'ioremap error for 0x78', + ] + x86_common, + 'intel-corei7-64' : [ + 'can\'t set Max Payload Size to 256', + 'intel_punit_ipc: can\'t request region for resource', + '[drm] parse error at position 4 in video mode \'efifb\'', + 'ACPI Error: Could not enable RealTimeClock event', + 'ACPI Warning: Could not enable fixed event - RealTimeClock', + 'hci_intel INT33E1:00: Unable to retrieve gpio', + 'hci_intel: probe of INT33E1:00 failed', + 'can\'t derive routing for PCI INT A', + 'failed to read out thermal zone', + 'Bluetooth: hci0: Setting Intel event mask failed', + 'ttyS2 - failed to request DMA', + ] + x86_common, + 'crownbay' : x86_common, + 'genericx86' : x86_common, + 'genericx86-64' : [ + 'Direct firmware load for i915', + 'Failed to load firmware i915', + 'Failed to fetch GuC', + 'Failed to initialize GuC', + 'Failed to load DMC firmware', + 'The driver is built-in, so to load the firmware you need to', + ] + x86_common, + 'edgerouter' : [ + 'Fatal server error:', + ] + common_errors, + 'jasperforest' : [ + 'Activated service \'org.bluez\' failed:', + 'Unable to find NFC netlink family', + ] + common_errors, +} + +log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] + +class ParseLogsTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + cls.errors = errors + + # When systemd is enabled we need to notice errors on + # circular dependencies in units. + if 'systemd' in cls.td.get('DISTRO_FEATURES', ''): + cls.errors.extend([ + 'Found ordering cycle on', + 'Breaking ordering cycle by deleting job', + 'deleted to break ordering cycle', + 'Ordering cycle found, skipping', + ]) + + cls.ignore_errors = ignore_errors + cls.log_locations = log_locations + cls.msg = '' + is_lsb, _ = cls.tc.target.run("which LSB_Test.sh") + if is_lsb == 0: + for machine in cls.ignore_errors: + cls.ignore_errors[machine] = cls.ignore_errors[machine] \ + + video_related + + def getMachine(self): + return self.td.get('MACHINE', '') + + def getWorkdir(self): + return self.td.get('WORKDIR', '') + + # Get some information on the CPU of the machine to display at the + # beginning of the output. This info might be useful in some cases. + def getHardwareInfo(self): + hwi = "" + cmd = ('cat /proc/cpuinfo | grep "model name" | head -n1 | ' + " awk 'BEGIN{FS=\":\"}{print $2}'") + _, cpu_name = self.target.run(cmd) + + cmd = ('cat /proc/cpuinfo | grep "cpu cores" | head -n1 | ' + "awk {'print $4'}") + _, cpu_physical_cores = self.target.run(cmd) + + cmd = 'cat /proc/cpuinfo | grep "processor" | wc -l' + _, cpu_logical_cores = self.target.run(cmd) + + _, cpu_arch = self.target.run('uname -m') + + hwi += 'Machine information: \n' + hwi += '*******************************\n' + hwi += 'Machine name: ' + self.getMachine() + '\n' + hwi += 'CPU: ' + str(cpu_name) + '\n' + hwi += 'Arch: ' + str(cpu_arch)+ '\n' + hwi += 'Physical cores: ' + str(cpu_physical_cores) + '\n' + hwi += 'Logical cores: ' + str(cpu_logical_cores) + '\n' + hwi += '*******************************\n' + + return hwi + + # Go through the log locations provided and if it's a folder + # create a list with all the .log files in it, if it's a file + # just add it to that list. + def getLogList(self, log_locations): + logs = [] + for location in log_locations: + status, _ = self.target.run('test -f ' + str(location)) + if status == 0: + logs.append(str(location)) + else: + status, _ = self.target.run('test -d ' + str(location)) + if status == 0: + cmd = 'find ' + str(location) + '/*.log -maxdepth 1 -type f' + status, output = self.target.run(cmd) + if status == 0: + output = output.splitlines() + for logfile in output: + logs.append(os.path.join(location, str(logfile))) + return logs + + # Copy the log files to be parsed locally + def transfer_logs(self, log_list): + workdir = self.getWorkdir() + self.target_logs = workdir + '/' + 'target_logs' + target_logs = self.target_logs + if os.path.exists(target_logs): + rmtree(self.target_logs) + os.makedirs(target_logs) + for f in log_list: + self.target.copyFrom(str(f), target_logs) + + # Get the local list of logs + def get_local_log_list(self, log_locations): + self.transfer_logs(self.getLogList(log_locations)) + list_dir = os.listdir(self.target_logs) + dir_files = [os.path.join(self.target_logs, f) for f in list_dir] + logs = [f for f in dir_files if os.path.isfile(f)] + return logs + + # Build the grep command to be used with filters and exclusions + def build_grepcmd(self, errors, ignore_errors, log): + grepcmd = 'grep ' + grepcmd += '-Ei "' + for error in errors: + grepcmd += error + '|' + grepcmd = grepcmd[:-1] + grepcmd += '" ' + str(log) + " | grep -Eiv \'" + + try: + errorlist = ignore_errors[self.getMachine()] + except KeyError: + self.msg += 'No ignore list found for this machine, using default\n' + errorlist = ignore_errors['default'] + + for ignore_error in errorlist: + ignore_error = ignore_error.replace('(', '\(') + ignore_error = ignore_error.replace(')', '\)') + ignore_error = ignore_error.replace("'", '.') + ignore_error = ignore_error.replace('?', '\?') + ignore_error = ignore_error.replace('[', '\[') + ignore_error = ignore_error.replace(']', '\]') + ignore_error = ignore_error.replace('*', '\*') + ignore_error = ignore_error.replace('0-9', '[0-9]') + grepcmd += ignore_error + '|' + grepcmd = grepcmd[:-1] + grepcmd += "\'" + + return grepcmd + + # Grep only the errors so that their context could be collected. + # Default context is 10 lines before and after the error itself + def parse_logs(self, errors, ignore_errors, logs, + lines_before = 10, lines_after = 10): + results = {} + rez = [] + grep_output = '' + + for log in logs: + result = None + thegrep = self.build_grepcmd(errors, ignore_errors, log) + + try: + result = check_output(thegrep, shell=True).decode('utf-8') + except: + pass + + if result is not None: + results[log.replace('target_logs/','')] = {} + rez = result.splitlines() + + for xrez in rez: + try: + cmd = ['grep', '-F', xrez, '-B', str(lines_before)] + cmd += ['-A', str(lines_after), log] + grep_output = check_output(cmd).decode('utf-8') + except: + pass + results[log.replace('target_logs/','')][xrez]=grep_output + + return results + + # Get the output of dmesg and write it in a file. + # This file is added to log_locations. + def write_dmesg(self): + (status, dmesg) = self.target.run('dmesg > /tmp/dmesg_output.log') + + @OETestID(1059) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_parselogs(self): + self.write_dmesg() + log_list = self.get_local_log_list(self.log_locations) + result = self.parse_logs(self.errors, self.ignore_errors, log_list) + print(self.getHardwareInfo()) + errcount = 0 + for log in result: + self.msg += 'Log: ' + log + '\n' + self.msg += '-----------------------\n' + for error in result[log]: + errcount += 1 + self.msg += 'Central error: ' + str(error) + '\n' + self.msg += '***********************\n' + self.msg += result[str(log)][str(error)] + '\n' + self.msg += '***********************\n' + self.msg += '%s errors found in logs.' % errcount + self.assertEqual(errcount, 0, msg=self.msg) diff --git a/meta/lib/oeqa/runtime/cases/perl.py b/meta/lib/oeqa/runtime/cases/perl.py new file mode 100644 index 0000000000..d0b7e8ed92 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/perl.py @@ -0,0 +1,37 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class PerlTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + src = os.path.join(cls.tc.files_dir, 'test.pl') + dst = '/tmp/test.pl' + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + dst = '/tmp/test.pl' + cls.tc.target.run('rm %s' % dst) + + @OETestID(1141) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['perl']) + def test_perl_exists(self): + status, output = self.target.run('which perl') + msg = 'Perl binary not in PATH or not on target.' + self.assertEqual(status, 0, msg=msg) + + @OETestID(208) + @OETestDepends(['perl.PerlTest.test_perl_exists']) + def test_perl_works(self): + status, output = self.target.run('perl /tmp/test.pl') + msg = 'Exit status was not 0. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + msg = 'Incorrect output: %s' % output + self.assertEqual(output, "the value of a is 0.01", msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ping.py b/meta/lib/oeqa/runtime/cases/ping.py new file mode 100644 index 0000000000..02f580abee --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/ping.py @@ -0,0 +1,24 @@ +from subprocess import Popen, PIPE + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.oetimeout import OETimeout + +class PingTest(OERuntimeTestCase): + + @OETimeout(30) + @OETestID(964) + def test_ping(self): + output = '' + count = 0 + while count < 5: + cmd = 'ping -c 1 %s' % self.target.ip + proc = Popen(cmd, shell=True, stdout=PIPE) + output += proc.communicate()[0].decode('utf-8') + if proc.poll() == 0: + count += 1 + else: + count = 0 + msg = ('Expected 5 consecutive, got %d.\n' + 'ping output is:\n%s' % (count,output)) + self.assertEqual(count, 5, msg = msg) diff --git a/meta/lib/oeqa/runtime/cases/python.py b/meta/lib/oeqa/runtime/cases/python.py new file mode 100644 index 0000000000..bf3e179163 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/python.py @@ -0,0 +1,43 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class PythonTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + src = os.path.join(cls.tc.files_dir, 'test.py') + dst = '/tmp/test.py' + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + dst = '/tmp/test.py' + cls.tc.target.run('rm %s' % dst) + + @OETestID(1145) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['python-core']) + def test_python_exists(self): + status, output = self.target.run('which python') + msg = 'Python binary not in PATH or not on target.' + self.assertEqual(status, 0, msg=msg) + + @OETestID(965) + @OETestDepends(['python.PythonTest.test_python_exists']) + def test_python_stdout(self): + status, output = self.target.run('python /tmp/test.py') + msg = 'Exit status was not 0. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + msg = 'Incorrect output: %s' % output + self.assertEqual(output, "the value of a is 0.01", msg=msg) + + @OETestID(1146) + @OETestDepends(['python.PythonTest.test_python_stdout']) + def test_python_testfile(self): + status, output = self.target.run('ls /tmp/testfile.python') + self.assertEqual(status, 0, msg='Python test file generate failed.') diff --git a/meta/lib/oeqa/runtime/cases/rpm.py b/meta/lib/oeqa/runtime/cases/rpm.py new file mode 100644 index 0000000000..05b94c7b40 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/rpm.py @@ -0,0 +1,142 @@ +import os +import fnmatch + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage +from oeqa.core.utils.path import findFile + +class RpmBasicTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + if cls.tc.td['PACKAGE_CLASSES'].split()[0] != 'package_rpm': + cls.skipTest('Tests require image to be build from rpm') + + @OETestID(960) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_rpm_help(self): + status, output = self.target.run('rpm --help') + msg = 'status and output: %s and %s' % (status, output) + self.assertEqual(status, 0, msg=msg) + + @OETestID(191) + @OETestDepends(['rpm.RpmBasicTest.test_rpm_help']) + def test_rpm_query(self): + status, output = self.target.run('rpm -q rpm') + msg = 'status and output: %s and %s' % (status, output) + self.assertEqual(status, 0, msg=msg) + +class RpmInstallRemoveTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + if cls.tc.td['PACKAGE_CLASSES'].split()[0] != 'package_rpm': + cls.skipTest('Tests require image to be build from rpm') + + pkgarch = cls.td['TUNE_PKGARCH'].replace('-', '_') + rpmdir = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm', pkgarch) + # Pick rpm-doc as a test file to get installed, because it's small + # and it will always be built for standard targets + rpm_doc = 'rpm-doc-*.%s.rpm' % pkgarch + for f in fnmatch.filter(os.listdir(rpmdir), rpm_doc): + test_file = os.path.join(rpmdir, f) + dst = '/tmp/rpm-doc.rpm' + cls.tc.target.copyTo(test_file, dst) + + @classmethod + def tearDownClass(cls): + dst = '/tmp/rpm-doc.rpm' + cls.tc.target.run('rm -f %s' % dst) + + @OETestID(192) + @OETestDepends(['rpm.RpmBasicTest.test_rpm_help']) + def test_rpm_install(self): + status, output = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') + msg = 'Failed to install rpm-doc package: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(194) + @OETestDepends(['rpm.RpmInstallRemoveTest.test_rpm_install']) + def test_rpm_remove(self): + status,output = self.target.run('rpm -e rpm-doc') + msg = 'Failed to remove rpm-doc package: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(1096) + @OETestDepends(['rpm.RpmBasicTest.test_rpm_query']) + def test_rpm_query_nonroot(self): + + def set_up_test_user(u): + status, output = self.target.run('id -u %s' % u) + if status: + status, output = self.target.run('useradd %s' % u) + msg = 'Failed to create new user: %s' % output + self.assertTrue(status == 0, msg=msg) + + def exec_as_test_user(u): + status, output = self.target.run('su -c id %s' % u) + msg = 'Failed to execute as new user' + self.assertTrue("({0})".format(u) in output, msg=msg) + + status, output = self.target.run('su -c "rpm -qa" %s ' % u) + msg = 'status: %s. Cannot run rpm -qa: %s' % (status, output) + self.assertEqual(status, 0, msg=msg) + + def unset_up_test_user(u): + status, output = self.target.run('userdel -r %s' % u) + msg = 'Failed to erase user: %s' % output + self.assertTrue(status == 0, msg=msg) + + tuser = 'test1' + + try: + set_up_test_user(tuser) + exec_as_test_user(tuser) + finally: + unset_up_test_user(tuser) + + @OETestID(195) + @OETestDepends(['rpm.RpmInstallRemoveTest.test_rpm_remove']) + def test_check_rpm_install_removal_log_file_size(self): + """ + Summary: Check that rpm writes into /var/log/messages + Expected: There should be some RPM prefixed entries in the above file. + Product: BSPs + Author: Alexandru Georgescu <alexandru.c.georgescu@intel.com> + Author: Alexander Kanavin <alexander.kanavin@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + db_files_cmd = 'ls /var/lib/rpm/__db.*' + check_log_cmd = "grep RPM /var/log/messages | wc -l" + + # Make sure that some database files are under /var/lib/rpm as '__db.xxx' + status, output = self.target.run(db_files_cmd) + msg = 'Failed to find database files under /var/lib/rpm/ as __db.xxx' + self.assertEqual(0, status, msg=msg) + + # Remove the package just in case + self.target.run('rpm -e rpm-doc') + + # Install/Remove a package 10 times + for i in range(10): + status, output = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') + msg = 'Failed to install rpm-doc package. Reason: {}'.format(output) + self.assertEqual(0, status, msg=msg) + + status, output = self.target.run('rpm -e rpm-doc') + msg = 'Failed to remove rpm-doc package. Reason: {}'.format(output) + self.assertEqual(0, status, msg=msg) + + # if using systemd this should ensure all entries are flushed to /var + status, output = self.target.run("journalctl --sync") + # Get the amount of entries in the log file + status, output = self.target.run(check_log_cmd) + msg = 'Failed to get the final size of the log file.' + self.assertEqual(0, status, msg=msg) + + # Check that there's enough of them + self.assertGreaterEqual(int(output), 80, + 'Cound not find sufficient amount of rpm entries in /var/log/messages, found {} entries'.format(output)) diff --git a/meta/lib/oeqa/runtime/cases/scanelf.py b/meta/lib/oeqa/runtime/cases/scanelf.py new file mode 100644 index 0000000000..3ba1f78af9 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/scanelf.py @@ -0,0 +1,26 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class ScanelfTest(OERuntimeTestCase): + scancmd = 'scanelf --quiet --recursive --mount --ldpath --path' + + @OETestID(966) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['pax-utils']) + def test_scanelf_textrel(self): + # print TEXTREL information + cmd = '%s --textrel' % self.scancmd + status, output = self.target.run(cmd) + msg = '\n'.join([cmd, output]) + self.assertEqual(output.strip(), '', msg=msg) + + @OETestID(967) + @OETestDepends(['scanelf.ScanelfTest.test_scanelf_textrel']) + def test_scanelf_rpath(self): + # print RPATH information + cmd = '%s --textrel --rpath' % self.scancmd + status, output = self.target.run(cmd) + msg = '\n'.join([cmd, output]) + self.assertEqual(output.strip(), '', msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/scp.py b/meta/lib/oeqa/runtime/cases/scp.py new file mode 100644 index 0000000000..f488a6175b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/scp.py @@ -0,0 +1,33 @@ +import os +from tempfile import mkstemp + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class ScpTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + cls.tmp_fd, cls.tmp_path = mkstemp() + with os.fdopen(cls.tmp_fd, 'w') as f: + f.seek(2 ** 22 -1) + f.write(os.linesep) + + @classmethod + def tearDownClass(cls): + os.remove(cls.tmp_path) + + @OETestID(220) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_scp_file(self): + dst = '/tmp/test_scp_file' + + (status, output) = self.target.copyTo(self.tmp_path, dst) + msg = 'File could not be copied. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + (status, output) = self.target.run('ls -la %s' % dst) + self.assertEqual(status, 0, msg = 'SCP test failed') + + self.target.run('rm %s' % dst) diff --git a/meta/lib/oeqa/runtime/cases/skeletoninit.py b/meta/lib/oeqa/runtime/cases/skeletoninit.py new file mode 100644 index 0000000000..4fdcf033a3 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/skeletoninit.py @@ -0,0 +1,33 @@ +# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=284 +# testcase. Image under test must have meta-skeleton layer in bblayers and +# IMAGE_INSTALL_append = " service" in local.conf +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +class SkeletonBasicTest(OERuntimeTestCase): + + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['service']) + @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', + 'Not appropiate for systemd image') + def test_skeleton_availability(self): + status, output = self.target.run('ls /etc/init.d/skeleton') + msg = 'skeleton init script not found. Output:\n%s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('ls /usr/sbin/skeleton-test') + msg = 'skeleton-test not found. Output:\n%s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(284) + @OETestDepends(['skeletoninit.SkeletonBasicTest.test_skeleton_availability']) + def test_skeleton_script(self): + output1 = self.target.run("/etc/init.d/skeleton start")[1] + cmd = '%s | grep [s]keleton-test' % self.tc.target_cmds['ps'] + status, output2 = self.target.run(cmd) + msg = ('Skeleton script could not be started:' + '\n%s\n%s' % (output1, output2)) + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ssh.py b/meta/lib/oeqa/runtime/cases/ssh.py new file mode 100644 index 0000000000..eca167969a --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/ssh.py @@ -0,0 +1,15 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class SSHTest(OERuntimeTestCase): + + @OETestID(224) + @OETestDepends(['ping.PingTest.test_ping']) + def test_ssh(self): + (status, output) = self.target.run('uname -a') + self.assertEqual(status, 0, msg='SSH Test failed: %s' % output) + (status, output) = self.target.run('cat /etc/masterimage') + msg = "This isn't the right image - /etc/masterimage " \ + "shouldn't be here %s" % output + self.assertEqual(status, 1, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/systemd.py b/meta/lib/oeqa/runtime/cases/systemd.py new file mode 100644 index 0000000000..db69384c8a --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/systemd.py @@ -0,0 +1,181 @@ +import re +import time + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar, skipIfNotDataVar +from oeqa.runtime.decorator.package import OEHasPackage +from oeqa.core.decorator.data import skipIfNotFeature + +class SystemdTest(OERuntimeTestCase): + + def systemctl(self, action='', target='', expected=0, verbose=False): + command = 'systemctl %s %s' % (action, target) + status, output = self.target.run(command) + message = '\n'.join([command, output]) + if status != expected and verbose: + cmd = 'systemctl status --full %s' % target + message += self.target.run(cmd)[1] + self.assertEqual(status, expected, message) + return output + + #TODO: use pyjournalctl instead + def journalctl(self, args='',l_match_units=None): + """ + Request for the journalctl output to the current target system + + Arguments: + -args, an optional argument pass through argument + -l_match_units, an optional list of units to filter the output + Returns: + -string output of the journalctl command + Raises: + -AssertionError, on remote commands that fail + -ValueError, on a journalctl call with filtering by l_match_units that + returned no entries + """ + + query_units='' + if l_match_units: + query_units = ['_SYSTEMD_UNIT='+unit for unit in l_match_units] + query_units = ' '.join(query_units) + command = 'journalctl %s %s' %(args, query_units) + status, output = self.target.run(command) + if status: + raise AssertionError("Command '%s' returned non-zero exit " + 'code %d:\n%s' % (command, status, output)) + if len(output) == 1 and "-- No entries --" in output: + raise ValueError('List of units to match: %s, returned no entries' + % l_match_units) + return output + +class SystemdBasicTests(SystemdTest): + + def settle(self): + """ + Block until systemd has finished activating any units being activated, + or until two minutes has elapsed. + + Returns a tuple, either (True, '') if all units have finished + activating, or (False, message string) if there are still units + activating (generally, failing units that restart). + """ + endtime = time.time() + (60 * 2) + while True: + status, output = self.target.run('systemctl --state=activating') + if "0 loaded units listed" in output: + return (True, '') + if time.time() >= endtime: + return (False, output) + time.sleep(10) + + @skipIfNotFeature('systemd', + 'Test requires systemd to be in DISTRO_FEATURES') + @skipIfNotDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', + 'systemd is not the init manager for this image') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_systemd_basic(self): + self.systemctl('--version') + + @OETestID(551) + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + def test_systemd_list(self): + self.systemctl('list-unit-files') + + @OETestID(550) + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + def test_systemd_failed(self): + settled, output = self.settle() + msg = "Timed out waiting for systemd to settle:\n%s" % output + self.assertTrue(settled, msg=msg) + + output = self.systemctl('list-units', '--failed') + match = re.search('0 loaded units listed', output) + if not match: + output += self.systemctl('status --full --failed') + self.assertTrue(match, msg='Some systemd units failed:\n%s' % output) + + +class SystemdServiceTests(SystemdTest): + + @OEHasPackage(['avahi-daemon']) + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + def test_systemd_status(self): + self.systemctl('status --full', 'avahi-daemon.service') + + @OETestID(695) + @OETestDepends(['systemd.SystemdServiceTests.test_systemd_status']) + def test_systemd_stop_start(self): + self.systemctl('stop', 'avahi-daemon.service') + self.systemctl('is-active', 'avahi-daemon.service', + expected=3, verbose=True) + self.systemctl('start','avahi-daemon.service') + self.systemctl('is-active', 'avahi-daemon.service', verbose=True) + + @OETestID(696) + @OETestDepends(['systemd.SystemdServiceTests.test_systemd_status']) + def test_systemd_disable_enable(self): + self.systemctl('disable', 'avahi-daemon.service') + self.systemctl('is-enabled', 'avahi-daemon.service', expected=1) + self.systemctl('enable', 'avahi-daemon.service') + self.systemctl('is-enabled', 'avahi-daemon.service') + +class SystemdJournalTests(SystemdTest): + + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + def test_systemd_journal(self): + status, output = self.target.run('journalctl') + self.assertEqual(status, 0, output) + + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + def test_systemd_boot_time(self, systemd_TimeoutStartSec=90): + """ + Get the target boot time from journalctl and log it + + Arguments: + -systemd_TimeoutStartSec, an optional argument containing systemd's + unit start timeout to compare against + """ + + # The expression chain that uniquely identifies the time boot message. + expr_items=['Startup finished', 'kernel', 'userspace','\.$'] + try: + output = self.journalctl(args='-o cat --reverse') + except AssertionError: + self.fail('Error occurred while calling journalctl') + if not len(output): + self.fail('Error, unable to get startup time from systemd journal') + + # Check for the regular expression items that match the startup time. + for line in output.split('\n'): + check_match = ''.join(re.findall('.*'.join(expr_items), line)) + if check_match: + break + # Put the startup time in the test log + if check_match: + self.tc.logger.info('%s' % check_match) + else: + self.skipTest('Error at obtaining the boot time from journalctl') + boot_time_sec = 0 + + # Get the numeric values from the string and convert them to seconds + # same data will be placed in list and string for manipulation. + l_boot_time = check_match.split(' ')[-2:] + s_boot_time = ' '.join(l_boot_time) + try: + # Obtain the minutes it took to boot. + if l_boot_time[0].endswith('min') and l_boot_time[0][0].isdigit(): + boot_time_min = s_boot_time.split('min')[0] + # Convert to seconds and accumulate it. + boot_time_sec += int(boot_time_min) * 60 + # Obtain the seconds it took to boot and accumulate. + boot_time_sec += float(l_boot_time[1].split('s')[0]) + except ValueError: + self.skipTest('Error when parsing time from boot string') + + # Assert the target boot time against systemd's unit start timeout. + if boot_time_sec > systemd_TimeoutStartSec: + msg = ("Target boot time %s exceeds systemd's TimeoutStartSec %s" + % (boot_time_sec, systemd_TimeoutStartSec)) + self.tc.logger.info(msg) diff --git a/meta/lib/oeqa/runtime/cases/x32lib.py b/meta/lib/oeqa/runtime/cases/x32lib.py new file mode 100644 index 0000000000..8da0154e7b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/x32lib.py @@ -0,0 +1,19 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotInDataVar + +class X32libTest(OERuntimeTestCase): + + @skipIfNotInDataVar('DEFAULTTUNE', 'x86-64-x32', + 'DEFAULTTUNE is not set to x86-64-x32') + @OETestID(281) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_x32_file(self): + cmd = 'readelf -h /bin/ls | grep Class | grep ELF32' + status1 = self.target.run(cmd)[0] + cmd = 'readelf -h /bin/ls | grep Machine | grep X86-64' + status2 = self.target.run(cmd)[0] + msg = ("/bin/ls isn't an X86-64 ELF32 binary. readelf says: %s" % + self.target.run("readelf -h /bin/ls")[1]) + self.assertTrue(status1 == 0 and status2 == 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/xorg.py b/meta/lib/oeqa/runtime/cases/xorg.py new file mode 100644 index 0000000000..2124813e3c --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/xorg.py @@ -0,0 +1,17 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class XorgTest(OERuntimeTestCase): + + @OETestID(1151) + @skipIfNotFeature('x11-base', + 'Test requires x11 to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_xorg_running(self): + cmd ='%s | grep -v xinit | grep [X]org' % self.tc.target_cmds['ps'] + status, output = self.target.run(cmd) + msg = ('Xorg does not appear to be running %s' % + self.target.run(self.tc.target_cmds['ps'])[1]) + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/connman.py b/meta/lib/oeqa/runtime/connman.py deleted file mode 100644 index cc537f7766..0000000000 --- a/meta/lib/oeqa/runtime/connman.py +++ /dev/null @@ -1,30 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("connman"): - skipModule("No connman package in image") - - -class ConnmanTest(oeRuntimeTest): - - def service_status(self, service): - if oeRuntimeTest.hasFeature("systemd"): - (status, output) = self.target.run('systemctl status -l %s' % service) - return output - else: - return "Unable to get status or logs for %s" % service - - @skipUnlessPassed('test_ssh') - def test_connmand_help(self): - (status, output) = self.target.run('/usr/sbin/connmand --help') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - - @testcase(221) - @skipUnlessPassed('test_connmand_help') - def test_connmand_running(self): - (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep [c]onnmand') - if status != 0: - print self.service_status("connman") - self.fail("No connmand process running") diff --git a/meta/lib/oeqa/runtime/context.py b/meta/lib/oeqa/runtime/context.py new file mode 100644 index 0000000000..c4cd76cf44 --- /dev/null +++ b/meta/lib/oeqa/runtime/context.py @@ -0,0 +1,220 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os + +from oeqa.core.context import OETestContext, OETestContextExecutor +from oeqa.core.target.ssh import OESSHTarget +from oeqa.core.target.qemu import OEQemuTarget +from oeqa.utils.dump import HostDumper + +from oeqa.runtime.loader import OERuntimeTestLoader + +class OERuntimeTestContext(OETestContext): + loaderClass = OERuntimeTestLoader + runtime_files_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "files") + + def __init__(self, td, logger, target, + host_dumper, image_packages, extract_dir): + super(OERuntimeTestContext, self).__init__(td, logger) + + self.target = target + self.image_packages = image_packages + self.host_dumper = host_dumper + self.extract_dir = extract_dir + self._set_target_cmds() + + def _set_target_cmds(self): + self.target_cmds = {} + + self.target_cmds['ps'] = 'ps' + if 'procps' in self.image_packages: + self.target_cmds['ps'] = self.target_cmds['ps'] + ' -ef' + +class OERuntimeTestContextExecutor(OETestContextExecutor): + _context_class = OERuntimeTestContext + + name = 'runtime' + help = 'runtime test component' + description = 'executes runtime tests over targets' + + default_cases = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'cases') + default_data = None + default_test_data = 'data/testdata.json' + default_tests = '' + + default_target_type = 'simpleremote' + default_manifest = 'data/manifest' + default_server_ip = '192.168.7.1' + default_target_ip = '192.168.7.2' + default_host_dumper_dir = '/tmp/oe-saved-tests' + default_extract_dir = 'packages/extracted' + + def register_commands(self, logger, subparsers): + super(OERuntimeTestContextExecutor, self).register_commands(logger, subparsers) + + runtime_group = self.parser.add_argument_group('runtime options') + + runtime_group.add_argument('--target-type', action='store', + default=self.default_target_type, choices=['simpleremote', 'qemu'], + help="Target type of device under test, default: %s" \ + % self.default_target_type) + runtime_group.add_argument('--target-ip', action='store', + default=self.default_target_ip, + help="IP address of device under test, default: %s" \ + % self.default_target_ip) + runtime_group.add_argument('--server-ip', action='store', + default=self.default_target_ip, + help="IP address of device under test, default: %s" \ + % self.default_server_ip) + + runtime_group.add_argument('--host-dumper-dir', action='store', + default=self.default_host_dumper_dir, + help="Directory where host status is dumped, if tests fails, default: %s" \ + % self.default_host_dumper_dir) + + runtime_group.add_argument('--packages-manifest', action='store', + default=self.default_manifest, + help="Package manifest of the image under testi, default: %s" \ + % self.default_manifest) + + runtime_group.add_argument('--extract-dir', action='store', + default=self.default_extract_dir, + help='Directory where extracted packages reside, default: %s' \ + % self.default_extract_dir) + + runtime_group.add_argument('--qemu-boot', action='store', + help="Qemu boot configuration, only needed when target_type is QEMU.") + + @staticmethod + def getTarget(target_type, logger, target_ip, server_ip, **kwargs): + target = None + + if target_type == 'simpleremote': + target = OESSHTarget(logger, target_ip, server_ip, **kwargs) + elif target_type == 'qemu': + target = OEQemuTarget(logger, target_ip, server_ip, **kwargs) + else: + # XXX: This code uses the old naming convention for controllers and + # targets, the idea it is to leave just targets as the controller + # most of the time was just a wrapper. + # XXX: This code tries to import modules from lib/oeqa/controllers + # directory and treat them as controllers, it will less error prone + # to use introspection to load such modules. + # XXX: Don't base your targets on this code it will be refactored + # in the near future. + # Custom target module loading + try: + target_modules_path = kwargs.get('target_modules_path', '') + controller = OERuntimeTestContextExecutor.getControllerModule(target_type, target_modules_path) + target = controller(logger, target_ip, server_ip, **kwargs) + except ImportError as e: + raise TypeError("Failed to import %s from available controller modules" % target_type) + + return target + + # Search oeqa.controllers module directory for and return a controller + # corresponding to the given target name. + # AttributeError raised if not found. + # ImportError raised if a provided module can not be imported. + @staticmethod + def getControllerModule(target, target_modules_path): + controllerslist = OERuntimeTestContextExecutor._getControllerModulenames(target_modules_path) + controller = OERuntimeTestContextExecutor._loadControllerFromName(target, controllerslist) + return controller + + # Return a list of all python modules in lib/oeqa/controllers for each + # layer in bbpath + @staticmethod + def _getControllerModulenames(target_modules_path): + + controllerslist = [] + + def add_controller_list(path): + if not os.path.exists(os.path.join(path, '__init__.py')): + raise OSError('Controllers directory %s exists but is missing __init__.py' % path) + files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) + for f in files: + module = 'oeqa.controllers.' + f[:-3] + if module not in controllerslist: + controllerslist.append(module) + else: + raise RuntimeError("Duplicate controller module found for %s. Layers should create unique controller module names" % module) + + extpath = target_modules_path.split(':') + for p in extpath: + controllerpath = os.path.join(p, 'lib', 'oeqa', 'controllers') + if os.path.exists(controllerpath): + add_controller_list(controllerpath) + return controllerslist + + # Search for and return a controller from given target name and + # set of module names. + # Raise AttributeError if not found. + # Raise ImportError if a provided module can not be imported + @staticmethod + def _loadControllerFromName(target, modulenames): + for name in modulenames: + obj = OERuntimeTestContextExecutor._loadControllerFromModule(target, name) + if obj: + return obj + raise AttributeError("Unable to load {0} from available modules: {1}".format(target, str(modulenames))) + + # Search for and return a controller or None from given module name + @staticmethod + def _loadControllerFromModule(target, modulename): + obj = None + # import module, allowing it to raise import exception + try: + module = __import__(modulename, globals(), locals(), [target]) + except Exception as e: + return obj + # look for target class in the module, catching any exceptions as it + # is valid that a module may not have the target class. + try: + obj = getattr(module, target) + except: + obj = None + return obj + + @staticmethod + def readPackagesManifest(manifest): + if not manifest or not os.path.exists(manifest): + raise OSError("Manifest file not exists: %s" % manifest) + + image_packages = set() + with open(manifest, 'r') as f: + for line in f.readlines(): + line = line.strip() + if line and not line.startswith("#"): + image_packages.add(line.split()[0]) + + return image_packages + + @staticmethod + def getHostDumper(cmds, directory): + return HostDumper(cmds, directory) + + def _process_args(self, logger, args): + if not args.packages_manifest: + raise TypeError('Manifest file not provided') + + super(OERuntimeTestContextExecutor, self)._process_args(logger, args) + + target_kwargs = {} + target_kwargs['qemuboot'] = args.qemu_boot + + self.tc_kwargs['init']['target'] = \ + OERuntimeTestContextExecutor.getTarget(args.target_type, + None, args.target_ip, args.server_ip, **target_kwargs) + self.tc_kwargs['init']['host_dumper'] = \ + OERuntimeTestContextExecutor.getHostDumper(None, + args.host_dumper_dir) + self.tc_kwargs['init']['image_packages'] = \ + OERuntimeTestContextExecutor.readPackagesManifest( + args.packages_manifest) + self.tc_kwargs['init']['extract_dir'] = args.extract_dir + +_executor_class = OERuntimeTestContextExecutor diff --git a/meta/lib/oeqa/runtime/date.py b/meta/lib/oeqa/runtime/date.py deleted file mode 100644 index 97e8ee42ad..0000000000 --- a/meta/lib/oeqa/runtime/date.py +++ /dev/null @@ -1,23 +0,0 @@ -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * -import re - -class DateTest(oeRuntimeTest): - - @testcase(211) - @skipUnlessPassed("test_ssh") - def test_date(self): - (status, output) = self.target.run('date +"%Y-%m-%d %T"') - self.assertEqual(status, 0, msg="Failed to get initial date, output: %s" % output) - oldDate = output - - sampleDate = '"2016-08-09 10:00:00"' - (status, output) = self.target.run("date -s %s" % sampleDate) - self.assertEqual(status, 0, msg="Date set failed, output: %s" % output) - - (status, output) = self.target.run("date -R") - p = re.match('Tue, 09 Aug 2016 10:00:.. \+0000', output) - self.assertTrue(p, msg="The date was not set correctly, output: %s" % output) - - (status, output) = self.target.run('date -s "%s"' % oldDate) - self.assertEqual(status, 0, msg="Failed to reset date, output: %s" % output) diff --git a/meta/lib/oeqa/runtime/decorator/package.py b/meta/lib/oeqa/runtime/decorator/package.py new file mode 100644 index 0000000000..aa6ecb68fa --- /dev/null +++ b/meta/lib/oeqa/runtime/decorator/package.py @@ -0,0 +1,53 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.decorator import OETestDecorator, registerDecorator +from oeqa.core.utils.misc import strToSet + +@registerDecorator +class OEHasPackage(OETestDecorator): + """ + Checks if image has packages (un)installed. + + The argument must be a string, set, or list of packages that must be + installed or not present in the image. + + The way to tell a package must not be in an image is using an + exclamation point ('!') before the name of the package. + + If test depends on pkg1 or pkg2 you need to use: + @OEHasPackage({'pkg1', 'pkg2'}) + + If test depends on pkg1 and pkg2 you need to use: + @OEHasPackage('pkg1') + @OEHasPackage('pkg2') + + If test depends on pkg1 but pkg2 must not be present use: + @OEHasPackage({'pkg1', '!pkg2'}) + """ + + attrs = ('need_pkgs',) + + def setUpDecorator(self): + need_pkgs = set() + unneed_pkgs = set() + pkgs = strToSet(self.need_pkgs) + for pkg in pkgs: + if pkg.startswith('!'): + unneed_pkgs.add(pkg[1:]) + else: + need_pkgs.add(pkg) + + if unneed_pkgs: + msg = 'Checking if %s is not installed' % ', '.join(unneed_pkgs) + self.logger.debug(msg) + if not self.case.tc.image_packages.isdisjoint(unneed_pkgs): + msg = "Test can't run with %s installed" % ', or'.join(unneed_pkgs) + self.case.skipTest(msg) + + if need_pkgs: + msg = 'Checking if at least one of %s is installed' % ', '.join(need_pkgs) + self.logger.debug(msg) + if self.case.tc.image_packages.isdisjoint(need_pkgs): + msg = "Test requires %s to be installed" % ', or'.join(need_pkgs) + self.case.skipTest(msg) diff --git a/meta/lib/oeqa/runtime/df.py b/meta/lib/oeqa/runtime/df.py deleted file mode 100644 index 09569d5ff6..0000000000 --- a/meta/lib/oeqa/runtime/df.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * - - -class DfTest(oeRuntimeTest): - - @testcase(234) - @skipUnlessPassed("test_ssh") - def test_df(self): - (status,output) = self.target.run("df / | sed -n '2p' | awk '{print $4}'") - self.assertTrue(int(output)>5120, msg="Not enough space on image. Current size is %s" % output) diff --git a/meta/lib/oeqa/runtime/dmesg.py b/meta/lib/oeqa/runtime/dmesg.py deleted file mode 100644 index 43e16c3f79..0000000000 --- a/meta/lib/oeqa/runtime/dmesg.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * - - -class DmesgTest(oeRuntimeTest): - - @testcase(215) - @skipUnlessPassed('test_ssh') - def test_dmesg(self): - (status, output) = self.target.run('dmesg | grep -v mmci-pl18x | grep -v "error changing net interface name" | grep -iv "dma timeout" | grep -i error') - self.assertEqual(status, 1, msg = "Error messages in dmesg log: %s" % output) diff --git a/meta/lib/oeqa/runtime/gcc.py b/meta/lib/oeqa/runtime/gcc.py deleted file mode 100644 index 08b3cf1230..0000000000 --- a/meta/lib/oeqa/runtime/gcc.py +++ /dev/null @@ -1,39 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - - -class GccCompileTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.c"), "/tmp/test.c") - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "testmakefile"), "/tmp/testmakefile") - - @testcase(203) - def test_gcc_compile(self): - (status, output) = self.target.run('gcc /tmp/test.c -o /tmp/test -lm') - self.assertEqual(status, 0, msg="gcc compile failed, output: %s" % output) - (status, output) = self.target.run('/tmp/test') - self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) - - @testcase(200) - def test_gpp_compile(self): - (status, output) = self.target.run('g++ /tmp/test.c -o /tmp/test -lm') - self.assertEqual(status, 0, msg="g++ compile failed, output: %s" % output) - (status, output) = self.target.run('/tmp/test') - self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) - - @testcase(204) - def test_make(self): - (status, output) = self.target.run('cd /tmp; make -f testmakefile') - self.assertEqual(status, 0, msg="running make failed, output %s" % output) - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run("rm /tmp/test.c /tmp/test.o /tmp/test /tmp/testmakefile") diff --git a/meta/lib/oeqa/runtime/kernelmodule.py b/meta/lib/oeqa/runtime/kernelmodule.py deleted file mode 100644 index 2e81720327..0000000000 --- a/meta/lib/oeqa/runtime/kernelmodule.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - - -class KernelModuleTest(oeRuntimeTest): - - def setUp(self): - self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod.c"), "/tmp/hellomod.c") - self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod_makefile"), "/tmp/Makefile") - - @testcase('316') - @skipUnlessPassed('test_ssh') - @skipUnlessPassed('test_gcc_compile') - def test_kernel_module(self): - cmds = [ - 'cd /usr/src/kernel && make scripts', - 'cd /tmp && make', - 'cd /tmp && insmod hellomod.ko', - 'lsmod | grep hellomod', - 'dmesg | grep Hello', - 'rmmod hellomod', 'dmesg | grep "Cleaning up hellomod"' - ] - for cmd in cmds: - (status, output) = self.target.run(cmd, 900) - self.assertEqual(status, 0, msg="\n".join([cmd, output])) - - def tearDown(self): - self.target.run('rm -f /tmp/Makefile /tmp/hellomod.c') diff --git a/meta/lib/oeqa/runtime/ldd.py b/meta/lib/oeqa/runtime/ldd.py deleted file mode 100644 index bce56c4270..0000000000 --- a/meta/lib/oeqa/runtime/ldd.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class LddTest(oeRuntimeTest): - - @skipUnlessPassed('test_ssh') - def test_ldd_exists(self): - (status, output) = self.target.run('which ldd') - self.assertEqual(status, 0, msg = "ldd does not exist in PATH: which ldd: %s" % output) - - @testcase(239) - @skipUnlessPassed('test_ldd_exists') - def test_ldd_rtldlist_check(self): - (status, output) = self.target.run('for i in $(which ldd | xargs cat | grep "^RTLDLIST"|cut -d\'=\' -f2|tr -d \'"\'); do test -f $i && echo $i && break; done') - self.assertEqual(status, 0, msg = "ldd path not correct or RTLDLIST files don't exist. ") diff --git a/meta/lib/oeqa/runtime/loader.py b/meta/lib/oeqa/runtime/loader.py new file mode 100644 index 0000000000..041ef976eb --- /dev/null +++ b/meta/lib/oeqa/runtime/loader.py @@ -0,0 +1,16 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.loader import OETestLoader +from oeqa.runtime.case import OERuntimeTestCase + +class OERuntimeTestLoader(OETestLoader): + caseClass = OERuntimeTestCase + + def _getTestCase(self, testCaseClass, tcName): + case = super(OERuntimeTestLoader, self)._getTestCase(testCaseClass, tcName) + + # Adds custom attributes to the OERuntimeTestCase + setattr(case, 'target', self.tc.target) + + return case diff --git a/meta/lib/oeqa/runtime/logrotate.py b/meta/lib/oeqa/runtime/logrotate.py deleted file mode 100644 index 86d791c300..0000000000 --- a/meta/lib/oeqa/runtime/logrotate.py +++ /dev/null @@ -1,28 +0,0 @@ -# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=289 testcase -# Note that the image under test must have logrotate installed - -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("logrotate"): - skipModule("No logrotate package in image") - - -class LogrotateTest(oeRuntimeTest): - - @skipUnlessPassed("test_ssh") - def test_1_logrotate_setup(self): - (status, output) = self.target.run('mkdir /home/root/logrotate_dir') - self.assertEqual(status, 0, msg = "Could not create logrotate_dir. Output: %s" % output) - (status, output) = self.target.run("sed -i 's#wtmp {#wtmp {\\n olddir /home/root/logrotate_dir#' /etc/logrotate.conf") - self.assertEqual(status, 0, msg = "Could not write to logrotate.conf file. Status and output: %s and %s)" % (status, output)) - - @testcase(289) - @skipUnlessPassed("test_1_logrotate_setup") - def test_2_logrotate(self): - (status, output) = self.target.run('logrotate -f /etc/logrotate.conf') - self.assertEqual(status, 0, msg = "logrotate service could not be reloaded. Status and output: %s and %s" % (status, output)) - output = self.target.run('ls -la /home/root/logrotate_dir/ | wc -l')[1] - self.assertTrue(int(output)>=3, msg = "new logfile could not be created. List of files within log directory: %s" %(self.target.run('ls -la /home/root/logrotate_dir')[1])) diff --git a/meta/lib/oeqa/runtime/multilib.py b/meta/lib/oeqa/runtime/multilib.py deleted file mode 100644 index ab0a6ccd69..0000000000 --- a/meta/lib/oeqa/runtime/multilib.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - multilibs = oeRuntimeTest.tc.d.getVar("MULTILIBS", True) or "" - if "multilib:lib32" not in multilibs: - skipModule("this isn't a multilib:lib32 image") - - -class MultilibTest(oeRuntimeTest): - - @testcase('279') - @skipUnlessPassed('test_ssh') - def test_file_connman(self): - self.assertTrue(oeRuntimeTest.hasPackage('connman-gnome'), msg="This test assumes connman-gnome is installed") - (status, output) = self.target.run("readelf -h /usr/bin/connman-applet | sed -n '3p' | awk '{print $2}'") - self.assertEqual(output, "ELF32", msg="connman-applet isn't an ELF32 binary. readelf says: %s" % self.target.run("readelf -h /usr/bin/connman-applet")[1]) diff --git a/meta/lib/oeqa/runtime/pam.py b/meta/lib/oeqa/runtime/pam.py deleted file mode 100644 index c8205c9abc..0000000000 --- a/meta/lib/oeqa/runtime/pam.py +++ /dev/null @@ -1,25 +0,0 @@ -# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=287 testcase -# Note that the image under test must have "pam" in DISTRO_FEATURES - -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("pam"): - skipModule("target doesn't have 'pam' in DISTRO_FEATURES") - - -class PamBasicTest(oeRuntimeTest): - - @testcase(287) - @skipUnlessPassed('test_ssh') - def test_pam(self): - (status, output) = self.target.run('login --help') - self.assertEqual(status, 1, msg = "login command does not work as expected. Status and output:%s and %s" %(status, output)) - (status, output) = self.target.run('passwd --help') - self.assertEqual(status, 0, msg = "passwd command does not work as expected. Status and output:%s and %s" %(status, output)) - (status, output) = self.target.run('su --help') - self.assertEqual(status, 0, msg = "su command does not work as expected. Status and output:%s and %s" %(status, output)) - (status, output) = self.target.run('useradd --help') - self.assertEqual(status, 0, msg = "useradd command does not work as expected. Status and output:%s and %s" %(status, output)) diff --git a/meta/lib/oeqa/runtime/perl.py b/meta/lib/oeqa/runtime/perl.py deleted file mode 100644 index 65da028d4b..0000000000 --- a/meta/lib/oeqa/runtime/perl.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("perl"): - skipModule("No perl package in the image") - - -class PerlTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.pl"), "/tmp/test.pl") - - def test_perl_exists(self): - (status, output) = self.target.run('which perl') - self.assertEqual(status, 0, msg="Perl binary not in PATH or not on target.") - - @testcase(208) - def test_perl_works(self): - (status, output) = self.target.run('perl /tmp/test.pl') - self.assertEqual(status, 0, msg="Exit status was not 0. Output: %s" % output) - self.assertEqual(output, "the value of a is 0.01", msg="Incorrect output: %s" % output) - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run("rm /tmp/test.pl") diff --git a/meta/lib/oeqa/runtime/ping.py b/meta/lib/oeqa/runtime/ping.py deleted file mode 100644 index a73c72402a..0000000000 --- a/meta/lib/oeqa/runtime/ping.py +++ /dev/null @@ -1,20 +0,0 @@ -import subprocess -import unittest -import sys -import time -from oeqa.oetest import oeRuntimeTest - -class PingTest(oeRuntimeTest): - - def test_ping(self): - output = '' - count = 0 - endtime = time.time() + 60 - while count < 5 and time.time() < endtime: - proc = subprocess.Popen("ping -c 1 %s" % self.target.ip, shell=True, stdout=subprocess.PIPE) - output += proc.communicate()[0] - if proc.poll() == 0: - count += 1 - else: - count = 0 - self.assertEqual(count, 5, msg = "Expected 5 consecutive replies, got %d.\nping output is:\n%s" % (count,output)) diff --git a/meta/lib/oeqa/runtime/python.py b/meta/lib/oeqa/runtime/python.py deleted file mode 100644 index 0387b9a03e..0000000000 --- a/meta/lib/oeqa/runtime/python.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("python"): - skipModule("No python package in the image") - - -class PythonTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.py"), "/tmp/test.py") - - def test_python_exists(self): - (status, output) = self.target.run('which python') - self.assertEqual(status, 0, msg="Python binary not in PATH or not on target.") - - @testcase(965) - def test_python_stdout(self): - (status, output) = self.target.run('python /tmp/test.py') - self.assertEqual(status, 0, msg="Exit status was not 0. Output: %s" % output) - self.assertEqual(output, "the value of a is 0.01", msg="Incorrect output: %s" % output) - - def test_python_testfile(self): - (status, output) = self.target.run('ls /tmp/testfile.python') - self.assertEqual(status, 0, msg="Python test file generate failed.") - - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run("rm /tmp/test.py /tmp/testfile.python") diff --git a/meta/lib/oeqa/runtime/rpm.py b/meta/lib/oeqa/runtime/rpm.py deleted file mode 100644 index b17e8b46a8..0000000000 --- a/meta/lib/oeqa/runtime/rpm.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest -import os -import fnmatch -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("package-management"): - skipModule("rpm module skipped: target doesn't have package-management in IMAGE_FEATURES") - if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: - skipModule("rpm module skipped: target doesn't have rpm as primary package manager") - - -class RpmBasicTest(oeRuntimeTest): - - @skipUnlessPassed('test_ssh') - def test_rpm_help(self): - (status, output) = self.target.run('rpm --help') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - - @testcase(191) - @skipUnlessPassed('test_rpm_help') - def test_rpm_query(self): - (status, output) = self.target.run('rpm -q rpm') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - -class RpmInstallRemoveTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - pkgarch = oeRuntimeTest.tc.d.getVar('TUNE_PKGARCH', True).replace("-", "_") - rpmdir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), "rpm", pkgarch) - # pick rpm-doc as a test file to get installed, because it's small and it will always be built for standard targets - for f in fnmatch.filter(os.listdir(rpmdir), "rpm-doc-*.%s.rpm" % pkgarch): - testrpmfile = f - oeRuntimeTest.tc.target.copy_to(os.path.join(rpmdir,testrpmfile), "/tmp/rpm-doc.rpm") - - @testcase(192) - @skipUnlessPassed('test_rpm_help') - def test_rpm_install(self): - (status, output) = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') - self.assertEqual(status, 0, msg="Failed to install rpm-doc package: %s" % output) - - @testcase(194) - @skipUnlessPassed('test_rpm_install') - def test_rpm_remove(self): - (status,output) = self.target.run('rpm -e rpm-doc') - self.assertEqual(status, 0, msg="Failed to remove rpm-doc package: %s" % output) - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run('rm -f /tmp/rpm-doc.rpm') - diff --git a/meta/lib/oeqa/runtime/scanelf.py b/meta/lib/oeqa/runtime/scanelf.py deleted file mode 100644 index 43a024ab9a..0000000000 --- a/meta/lib/oeqa/runtime/scanelf.py +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("pax-utils"): - skipModule("pax-utils package not installed") - -class ScanelfTest(oeRuntimeTest): - - def setUp(self): - self.scancmd = 'scanelf --quiet --recursive --mount --ldpath --path' - - @testcase(966) - @skipUnlessPassed('test_ssh') - def test_scanelf_textrel(self): - # print TEXTREL information - self.scancmd += " --textrel" - (status, output) = self.target.run(self.scancmd) - self.assertEqual(output.strip(), "", "\n".join([self.scancmd, output])) - - @testcase(967) - @skipUnlessPassed('test_ssh') - def test_scanelf_rpath(self): - # print RPATH information - self.scancmd += " --rpath" - (status, output) = self.target.run(self.scancmd) - self.assertEqual(output.strip(), "", "\n".join([self.scancmd, output])) diff --git a/meta/lib/oeqa/runtime/scp.py b/meta/lib/oeqa/runtime/scp.py deleted file mode 100644 index 48e87d2d0b..0000000000 --- a/meta/lib/oeqa/runtime/scp.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import skipUnlessPassed, testcase - -def setUpModule(): - if not (oeRuntimeTest.hasPackage("dropbear") or oeRuntimeTest.hasPackage("openssh-sshd")): - skipModule("No ssh package in image") - -class ScpTest(oeRuntimeTest): - - @testcase(220) - @skipUnlessPassed('test_ssh') - def test_scp_file(self): - test_log_dir = oeRuntimeTest.tc.d.getVar("TEST_LOG_DIR", True) - test_file_path = os.path.join(test_log_dir, 'test_scp_file') - with open(test_file_path, 'w') as test_scp_file: - test_scp_file.seek(2 ** 22 - 1) - test_scp_file.write(os.linesep) - (status, output) = self.target.copy_to(test_file_path, '/tmp/test_scp_file') - self.assertEqual(status, 0, msg = "File could not be copied. Output: %s" % output) - (status, output) = self.target.run("ls -la /tmp/test_scp_file") - self.assertEqual(status, 0, msg = "SCP test failed") diff --git a/meta/lib/oeqa/runtime/skeletoninit.py b/meta/lib/oeqa/runtime/skeletoninit.py deleted file mode 100644 index 7c7f402e5d..0000000000 --- a/meta/lib/oeqa/runtime/skeletoninit.py +++ /dev/null @@ -1,29 +0,0 @@ -# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=284 testcase -# Note that the image under test must have meta-skeleton layer in bblayers and IMAGE_INSTALL_append = " service" in local.conf - -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("service"): - skipModule("No service package in image") - - -class SkeletonBasicTest(oeRuntimeTest): - - @skipUnlessPassed('test_ssh') - @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager"), "Not appropiate for systemd image") - def test_skeleton_availability(self): - (status, output) = self.target.run('ls /etc/init.d/skeleton') - self.assertEqual(status, 0, msg = "skeleton init script not found. Output:\n%s " % output) - (status, output) = self.target.run('ls /usr/sbin/skeleton-test') - self.assertEqual(status, 0, msg = "skeleton-test not found. Output:\n%s" % output) - - @testcase(284) - @skipUnlessPassed('test_skeleton_availability') - @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager"), "Not appropiate for systemd image") - def test_skeleton_script(self): - output1 = self.target.run("/etc/init.d/skeleton start")[1] - (status, output2) = self.target.run(oeRuntimeTest.pscmd + ' | grep [s]keleton-test') - self.assertEqual(status, 0, msg = "Skeleton script could not be started:\n%s\n%s" % (output1, output2)) diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py deleted file mode 100644 index 3b49314df7..0000000000 --- a/meta/lib/oeqa/runtime/smart.py +++ /dev/null @@ -1,121 +0,0 @@ -import unittest -import re -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.httpserver import HTTPService - -def setUpModule(): - if not oeRuntimeTest.hasFeature("package-management"): - skipModule("Image doesn't have package management feature") - if not oeRuntimeTest.hasPackage("smart"): - skipModule("Image doesn't have smart installed") - if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: - skipModule("Rpm is not the primary package manager") - -class SmartTest(oeRuntimeTest): - - @skipUnlessPassed('test_smart_help') - def smart(self, command, expected = 0): - command = 'smart %s' % command - status, output = self.target.run(command, 1500) - message = os.linesep.join([command, output]) - self.assertEqual(status, expected, message) - self.assertFalse("Cannot allocate memory" in output, message) - return output - -class SmartBasicTest(SmartTest): - - @testcase(716) - @skipUnlessPassed('test_ssh') - def test_smart_help(self): - self.smart('--help') - - def test_smart_version(self): - self.smart('--version') - - @testcase(721) - def test_smart_info(self): - self.smart('info python-smartpm') - - @testcase(421) - def test_smart_query(self): - self.smart('query python-smartpm') - - @testcase(720) - def test_smart_search(self): - self.smart('search python-smartpm') - - @testcase(722) - def test_smart_stats(self): - self.smart('stats') - -class SmartRepoTest(SmartTest): - - @classmethod - def setUpClass(self): - self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip) - self.repo_server.start() - - @classmethod - def tearDownClass(self): - self.repo_server.stop() - - def test_smart_channel(self): - self.smart('channel', 1) - - @testcase(719) - def test_smart_channel_add(self): - image_pkgtype = self.tc.d.getVar('IMAGE_PKGTYPE', True) - deploy_url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, image_pkgtype) - pkgarchs = self.tc.d.getVar('PACKAGE_ARCHS', True).replace("-","_").split() - for arch in os.listdir('%s/%s' % (self.repo_server.root_dir, image_pkgtype)): - if arch in pkgarchs: - self.smart('channel -y --add {a} type=rpm-md baseurl={u}/{a}'.format(a=arch, u=deploy_url)) - self.smart('update') - - def test_smart_channel_help(self): - self.smart('channel --help') - - def test_smart_channel_list(self): - self.smart('channel --list') - - def test_smart_channel_show(self): - self.smart('channel --show') - - @testcase(717) - def test_smart_channel_rpmsys(self): - self.smart('channel --show rpmsys') - self.smart('channel --disable rpmsys') - self.smart('channel --enable rpmsys') - - @skipUnlessPassed('test_smart_channel_add') - def test_smart_install(self): - self.smart('remove -y psplash-default') - self.smart('install -y psplash-default') - - @testcase(728) - @skipUnlessPassed('test_smart_install') - def test_smart_install_dependency(self): - self.smart('remove -y psplash') - self.smart('install -y psplash-default') - - @testcase(723) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_install_from_disk(self): - self.smart('remove -y psplash-default') - self.smart('download psplash-default') - self.smart('install -y ./psplash-default*') - - @testcase(725) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_install_from_http(self): - output = self.smart('download --urls psplash-default') - url = re.search('(http://.*/psplash-default.*\.rpm)', output) - self.assertTrue(url, msg="Couln't find download url in %s" % output) - self.smart('remove -y psplash-default') - self.smart('install -y %s' % url.group(0)) - - @testcase(729) - @skipUnlessPassed('test_smart_install') - def test_smart_reinstall(self): - self.smart('reinstall -y psplash-default') diff --git a/meta/lib/oeqa/runtime/ssh.py b/meta/lib/oeqa/runtime/ssh.py deleted file mode 100644 index 0e76d5d512..0000000000 --- a/meta/lib/oeqa/runtime/ssh.py +++ /dev/null @@ -1,19 +0,0 @@ -import subprocess -import unittest -import sys -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not (oeRuntimeTest.hasPackage("dropbear") or oeRuntimeTest.hasPackage("openssh")): - skipModule("No ssh package in image") - -class SshTest(oeRuntimeTest): - - @testcase(224) - @skipUnlessPassed('test_ping') - def test_ssh(self): - (status, output) = self.target.run('uname -a') - self.assertEqual(status, 0, msg="SSH Test failed: %s" % output) - (status, output) = self.target.run('cat /etc/masterimage') - self.assertEqual(status, 1, msg="This isn't the right image - /etc/masterimage shouldn't be here %s" % output) diff --git a/meta/lib/oeqa/runtime/syslog.py b/meta/lib/oeqa/runtime/syslog.py deleted file mode 100644 index 7fa018e97f..0000000000 --- a/meta/lib/oeqa/runtime/syslog.py +++ /dev/null @@ -1,48 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("syslog"): - skipModule("No syslog package in image") - -class SyslogTest(oeRuntimeTest): - - @skipUnlessPassed("test_ssh") - def test_syslog_help(self): - (status,output) = self.target.run('/sbin/syslogd --help') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - - @testcase(201) - @skipUnlessPassed("test_syslog_help") - def test_syslog_running(self): - (status,output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -i [s]yslogd') - self.assertEqual(status, 0, msg="no syslogd process, ps output: %s" % self.target.run(oeRuntimeTest.pscmd)[1]) - - -class SyslogTestConfig(oeRuntimeTest): - - @skipUnlessPassed("test_syslog_running") - def test_syslog_logger(self): - (status,output) = self.target.run('logger foobar && test -e /var/log/messages && grep foobar /var/log/messages || logread | grep foobar') - self.assertEqual(status, 0, msg="Test log string not found in /var/log/messages. Output: %s " % output) - - @skipUnlessPassed("test_syslog_running") - def test_syslog_restart(self): - if "systemd" != oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager"): - (status,output) = self.target.run('/etc/init.d/syslog restart') - else: - (status,output) = self.target.run('systemctl restart syslog.service') - - @testcase(202) - @skipUnlessPassed("test_syslog_restart") - @skipUnlessPassed("test_syslog_logger") - @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager"), "Not appropiate for systemd image") - def test_syslog_startup_config(self): - self.target.run('echo "LOGFILE=/var/log/test" >> /etc/syslog-startup.conf') - (status,output) = self.target.run('/etc/init.d/syslog restart') - self.assertEqual(status, 0, msg="Could not restart syslog service. Status and output: %s and %s" % (status,output)) - (status,output) = self.target.run('logger foobar && grep foobar /var/log/test') - self.assertEqual(status, 0, msg="Test log string not found. Output: %s " % output) - self.target.run("sed -i 's#LOGFILE=/var/log/test##' /etc/syslog-startup.conf") - self.target.run('/etc/init.d/syslog restart') diff --git a/meta/lib/oeqa/runtime/systemd.py b/meta/lib/oeqa/runtime/systemd.py deleted file mode 100644 index 1451698bb3..0000000000 --- a/meta/lib/oeqa/runtime/systemd.py +++ /dev/null @@ -1,88 +0,0 @@ -import unittest -import re -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("systemd"): - skipModule("target doesn't have systemd in DISTRO_FEATURES") - if "systemd" != oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", True): - skipModule("systemd is not the init manager for this image") - - -class SystemdTest(oeRuntimeTest): - - def systemctl(self, action = '', target = '', expected = 0, verbose = False): - command = 'systemctl %s %s' % (action, target) - status, output = self.target.run(command) - message = '\n'.join([command, output]) - if status != expected and verbose: - message += self.target.run('systemctl status --full %s' % target)[1] - self.assertEqual(status, expected, message) - return output - - -class SystemdBasicTests(SystemdTest): - - @skipUnlessPassed('test_ssh') - def test_systemd_basic(self): - self.systemctl('--version') - - @testcase(551) - @skipUnlessPassed('test_system_basic') - def test_systemd_list(self): - self.systemctl('list-unit-files') - - def settle(self): - """ - Block until systemd has finished activating any units being activated, - or until two minutes has elapsed. - - Returns a tuple, either (True, '') if all units have finished - activating, or (False, message string) if there are still units - activating (generally, failing units that restart). - """ - import time - endtime = time.time() + (60 * 2) - while True: - status, output = self.target.run('systemctl --state=activating') - if "0 loaded units listed" in output: - return (True, '') - if time.time() >= endtime: - return (False, output) - time.sleep(10) - - @testcase(550) - @skipUnlessPassed('test_systemd_basic') - def test_systemd_failed(self): - settled, output = self.settle() - self.assertTrue(settled, msg="Timed out waiting for systemd to settle:\n" + output) - - output = self.systemctl('list-units', '--failed') - match = re.search("0 loaded units listed", output) - if not match: - output += self.systemctl('status --full --failed') - self.assertTrue(match, msg="Some systemd units failed:\n%s" % output) - - -class SystemdServiceTests(SystemdTest): - - @skipUnlessPassed('test_systemd_basic') - def test_systemd_status(self): - self.systemctl('status --full', 'avahi-daemon.service') - - @testcase(695) - @skipUnlessPassed('test_systemd_status') - def test_systemd_stop_start(self): - self.systemctl('stop', 'avahi-daemon.service') - self.systemctl('is-active', 'avahi-daemon.service', expected=3, verbose=True) - self.systemctl('start','avahi-daemon.service') - self.systemctl('is-active', 'avahi-daemon.service', verbose=True) - - @testcase(696) - @skipUnlessPassed('test_systemd_basic') - def test_systemd_disable_enable(self): - self.systemctl('disable', 'avahi-daemon.service') - self.systemctl('is-enabled', 'avahi-daemon.service', expected=1) - self.systemctl('enable', 'avahi-daemon.service') - self.systemctl('is-enabled', 'avahi-daemon.service') diff --git a/meta/lib/oeqa/runtime/utils/__init__.py b/meta/lib/oeqa/runtime/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/runtime/utils/__init__.py diff --git a/meta/lib/oeqa/runtime/utils/targetbuildproject.py b/meta/lib/oeqa/runtime/utils/targetbuildproject.py new file mode 100644 index 0000000000..5af55d736e --- /dev/null +++ b/meta/lib/oeqa/runtime/utils/targetbuildproject.py @@ -0,0 +1,39 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.utils.buildproject import BuildProject + +class TargetBuildProject(BuildProject): + + def __init__(self, target, uri, foldername=None, dl_dir=None): + self.target = target + self.targetdir = "~/" + BuildProject.__init__(self, uri, foldername, dl_dir=dl_dir) + + def download_archive(self): + self._download_archive() + + status, output = self.target.copyTo(self.localarchive, self.targetdir) + if status: + raise Exception('Failed to copy archive to target, ' + 'output: %s' % output) + + cmd = 'tar xf %s%s -C %s' % (self.targetdir, + self.archive, + self.targetdir) + status, output = self.target.run(cmd) + if status: + raise Exception('Failed to extract archive, ' + 'output: %s' % output) + + # Change targetdir to project folder + self.targetdir = self.targetdir + self.fname + + # The timeout parameter of target.run is set to 0 + # to make the ssh command run with no timeout. + def _run(self, cmd): + ret = self.target.run(cmd, 0) + msg = "Command %s failed with exit code %s: %s" % (cmd, ret[0], ret[1]) + if ret[0] != 0: + raise Exception(msg) + return ret[0] diff --git a/meta/lib/oeqa/runtime/vnc.py b/meta/lib/oeqa/runtime/vnc.py deleted file mode 100644 index f31deff306..0000000000 --- a/meta/lib/oeqa/runtime/vnc.py +++ /dev/null @@ -1,20 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModuleUnless -from oeqa.utils.decorators import * -import re - -def setUpModule(): - skipModuleUnless(oeRuntimeTest.hasPackage('x11vnc'), "No x11vnc package in image") - -class VNCTest(oeRuntimeTest): - - @testcase(213) - @skipUnlessPassed('test_ssh') - def test_vnc(self): - (status, output) = self.target.run('x11vnc -display :0 -bg -o x11vnc.log') - self.assertEqual(status, 0, msg="x11vnc server failed to start: %s" % output) - port = re.search('PORT=[0-9]*', output) - self.assertTrue(port, msg="Listening port not specified in command output: %s" %output) - - vncport = port.group(0).split('=')[1] - (status, output) = self.target.run('netstat -ntl | grep ":%s"' % vncport) - self.assertEqual(status, 0, msg="x11vnc server not running on port %s\n\n%s" % (vncport, self.target.run('netstat -ntl; cat x11vnc.log')[1])) diff --git a/meta/lib/oeqa/runtime/x32lib.py b/meta/lib/oeqa/runtime/x32lib.py deleted file mode 100644 index ce5e214035..0000000000 --- a/meta/lib/oeqa/runtime/x32lib.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - #check if DEFAULTTUNE is set and it's value is: x86-64-x32 - defaulttune = oeRuntimeTest.tc.d.getVar("DEFAULTTUNE", True) - if "x86-64-x32" not in defaulttune: - skipModule("DEFAULTTUNE is not set to x86-64-x32") - -class X32libTest(oeRuntimeTest): - - @testcase(281) - @skipUnlessPassed("test_ssh") - def test_x32_file(self): - status1 = self.target.run("readelf -h /bin/ls | grep Class | grep ELF32")[0] - status2 = self.target.run("readelf -h /bin/ls | grep Machine | grep X86-64")[0] - self.assertTrue(status1 == 0 and status2 == 0, msg="/bin/ls isn't an X86-64 ELF32 binary. readelf says: %s" % self.target.run("readelf -h /bin/ls")[1]) diff --git a/meta/lib/oeqa/runtime/xorg.py b/meta/lib/oeqa/runtime/xorg.py deleted file mode 100644 index 7aa61ad6ab..0000000000 --- a/meta/lib/oeqa/runtime/xorg.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("x11-base"): - skipModule("target doesn't have x11 in IMAGE_FEATURES") - - -class XorgTest(oeRuntimeTest): - - @skipUnlessPassed('test_ssh') - def test_xorg_running(self): - (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -v xinit | grep [X]org') - self.assertEqual(status, 0, msg="Xorg does not appear to be running %s" % self.target.run(oeRuntimeTest.pscmd)[1]) - - @testcase(972) - @skipUnlessPassed('test_ssh') - def test_xorg_error(self): - (status, output) = self.target.run('cat /var/log/Xorg.0.log | grep -v "(EE) error," | grep -v "PreInit" | grep -v "evdev:" | grep -v "glx" | grep "(EE)"') - self.assertEqual(status, 1, msg="Errors in Xorg log: %s" % output) - diff --git a/meta/lib/oeqa/sdk/__init__.py b/meta/lib/oeqa/sdk/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/sdk/__init__.py diff --git a/meta/lib/oeqa/sdk/case.py b/meta/lib/oeqa/sdk/case.py new file mode 100644 index 0000000000..963aa8d358 --- /dev/null +++ b/meta/lib/oeqa/sdk/case.py @@ -0,0 +1,12 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import subprocess + +from oeqa.core.case import OETestCase + +class OESDKTestCase(OETestCase): + def _run(self, cmd): + return subprocess.check_output(". %s > /dev/null; %s;" % \ + (self.tc.sdk_env, cmd), shell=True, + stderr=subprocess.STDOUT, universal_newlines=True) diff --git a/meta/lib/oeqa/sdk/cases/buildcpio.py b/meta/lib/oeqa/sdk/cases/buildcpio.py new file mode 100644 index 0000000000..333dc7c226 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/buildcpio.py @@ -0,0 +1,33 @@ +import unittest +from oeqa.sdk.case import OESDKTestCase +from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject + +class BuildCpioTest(OESDKTestCase): + td_vars = ['DATETIME'] + + @classmethod + def setUpClass(self): + dl_dir = self.td.get('DL_DIR', None) + + self.project = SDKBuildProject(self.tc.sdk_dir + "/cpio/", self.tc.sdk_env, + "https://ftp.gnu.org/gnu/cpio/cpio-2.12.tar.gz", + self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir) + self.project.download_archive() + + machine = self.td.get("MACHINE") + if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine): + raise unittest.SkipTest("SDK doesn't contain a cross-canadian toolchain") + + def test_cpio(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + self.assertEqual(self.project.run_install(), 0, + msg="Running make install failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/sdk/cases/buildgalculator.py b/meta/lib/oeqa/sdk/cases/buildgalculator.py new file mode 100644 index 0000000000..42e8ddb185 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/buildgalculator.py @@ -0,0 +1,35 @@ +import unittest + +from oeqa.sdk.case import OESDKTestCase +from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject + +class GalculatorTest(OESDKTestCase): + td_vars = ['DATETIME'] + + @classmethod + def setUpClass(self): + if not (self.tc.hasTargetPackage("gtk+3") or\ + self.tc.hasTargetPackage("libgtk-3.0")): + raise unittest.SkipTest("GalculatorTest class: SDK don't support gtk+3") + + def test_galculator(self): + dl_dir = self.td.get('DL_DIR', None) + project = None + try: + project = SDKBuildProject(self.tc.sdk_dir + "/galculator/", + self.tc.sdk_env, + "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2", + self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir) + + project.download_archive() + + # regenerate configure to get support for --with-libtool-sysroot + legacy_preconf=("autoreconf -i -f -I ${OECORE_TARGET_SYSROOT}/usr/share/aclocal -I m4;") + + self.assertEqual(project.run_configure(extra_cmds=legacy_preconf), + 0, msg="Running configure failed") + + self.assertEqual(project.run_make(), 0, + msg="Running make failed") + finally: + project.clean() diff --git a/meta/lib/oeqa/sdk/cases/buildiptables.py b/meta/lib/oeqa/sdk/cases/buildiptables.py new file mode 100644 index 0000000000..0bd00d125a --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/buildiptables.py @@ -0,0 +1,35 @@ +import unittest +from oeqa.sdk.case import OESDKTestCase +from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject + + +class BuildIptablesTest(OESDKTestCase): + td_vars = ['DATETIME'] + + @classmethod + def setUpClass(self): + dl_dir = self.td.get('DL_DIR', None) + + self.project = SDKBuildProject(self.tc.sdk_dir + "/iptables/", self.tc.sdk_env, + "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2", + self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir) + self.project.download_archive() + + machine = self.td.get("MACHINE") + + if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine): + raise unittest.SkipTest("SDK doesn't contain a cross-canadian toolchain") + + def test_iptables(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + self.assertEqual(self.project.run_install(), 0, + msg="Running make install failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/sdk/cases/gcc.py b/meta/lib/oeqa/sdk/cases/gcc.py new file mode 100644 index 0000000000..74ad2a2f2b --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/gcc.py @@ -0,0 +1,42 @@ +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase + +class GccCompileTest(OESDKTestCase): + td_vars = ['MACHINE'] + + @classmethod + def setUpClass(self): + files = {'test.c' : self.tc.files_dir, 'test.cpp' : self.tc.files_dir, + 'testsdkmakefile' : self.tc.sdk_files_dir} + for f in files: + shutil.copyfile(os.path.join(files[f], f), + os.path.join(self.tc.sdk_dir, f)) + + def setUp(self): + machine = self.td.get("MACHINE") + if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine): + raise unittest.SkipTest("GccCompileTest class: SDK doesn't contain a cross-canadian toolchain") + + def test_gcc_compile(self): + self._run('$CC %s/test.c -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir)) + + def test_gpp_compile(self): + self._run('$CXX %s/test.c -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir)) + + def test_gpp2_compile(self): + self._run('$CXX %s/test.cpp -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir)) + + def test_make(self): + self._run('cd %s; make -f testsdkmakefile' % self.tc.sdk_dir) + + @classmethod + def tearDownClass(self): + files = [os.path.join(self.tc.sdk_dir, f) \ + for f in ['test.c', 'test.cpp', 'test.o', 'test', + 'testsdkmakefile']] + for f in files: + remove_safe(f) diff --git a/meta/lib/oeqa/sdk/cases/perl.py b/meta/lib/oeqa/sdk/cases/perl.py new file mode 100644 index 0000000000..e1bded2ff2 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/perl.py @@ -0,0 +1,27 @@ +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase + +class PerlTest(OESDKTestCase): + @classmethod + def setUpClass(self): + if not self.tc.hasHostPackage("nativesdk-perl"): + raise unittest.SkipTest("No perl package in the SDK") + + for f in ['test.pl']: + shutil.copyfile(os.path.join(self.tc.files_dir, f), + os.path.join(self.tc.sdk_dir, f)) + self.testfile = os.path.join(self.tc.sdk_dir, "test.pl") + + def test_perl_exists(self): + self._run('which perl') + + def test_perl_works(self): + self._run('perl %s' % self.testfile) + + @classmethod + def tearDownClass(self): + remove_safe(self.testfile) diff --git a/meta/lib/oeqa/sdk/cases/python.py b/meta/lib/oeqa/sdk/cases/python.py new file mode 100644 index 0000000000..94a296f0ec --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/python.py @@ -0,0 +1,31 @@ +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase + +class PythonTest(OESDKTestCase): + @classmethod + def setUpClass(self): + if not self.tc.hasHostPackage("nativesdk-python"): + raise unittest.SkipTest("No python package in the SDK") + + for f in ['test.py']: + shutil.copyfile(os.path.join(self.tc.files_dir, f), + os.path.join(self.tc.sdk_dir, f)) + + def test_python_exists(self): + self._run('which python') + + def test_python_stdout(self): + output = self._run('python %s/test.py' % self.tc.sdk_dir) + self.assertEqual(output.strip(), "the value of a is 0.01", msg="Incorrect output: %s" % output) + + def test_python_testfile(self): + self._run('ls /tmp/testfile.python') + + @classmethod + def tearDownClass(self): + remove_safe("%s/test.py" % self.tc.sdk_dir) + remove_safe("/tmp/testfile.python") diff --git a/meta/lib/oeqa/sdk/context.py b/meta/lib/oeqa/sdk/context.py new file mode 100644 index 0000000000..0189ed851e --- /dev/null +++ b/meta/lib/oeqa/sdk/context.py @@ -0,0 +1,133 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import glob +import re + +from oeqa.core.context import OETestContext, OETestContextExecutor + +class OESDKTestContext(OETestContext): + sdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + + def __init__(self, td=None, logger=None, sdk_dir=None, sdk_env=None, + target_pkg_manifest=None, host_pkg_manifest=None): + super(OESDKTestContext, self).__init__(td, logger) + + self.sdk_dir = sdk_dir + self.sdk_env = sdk_env + self.target_pkg_manifest = target_pkg_manifest + self.host_pkg_manifest = host_pkg_manifest + + def _hasPackage(self, manifest, pkg): + for host_pkg in manifest.keys(): + if re.search(pkg, host_pkg): + return True + return False + + def hasHostPackage(self, pkg): + return self._hasPackage(self.host_pkg_manifest, pkg) + + def hasTargetPackage(self, pkg): + return self._hasPackage(self.target_pkg_manifest, pkg) + +class OESDKTestContextExecutor(OETestContextExecutor): + _context_class = OESDKTestContext + + name = 'sdk' + help = 'sdk test component' + description = 'executes sdk tests' + + default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'cases')] + default_test_data = None + + def register_commands(self, logger, subparsers): + import argparse_oe + + super(OESDKTestContextExecutor, self).register_commands(logger, subparsers) + + sdk_group = self.parser.add_argument_group('sdk options') + sdk_group.add_argument('--sdk-env', action='store', + help='sdk environment') + sdk_group.add_argument('--target-manifest', action='store', + help='sdk target manifest') + sdk_group.add_argument('--host-manifest', action='store', + help='sdk host manifest') + + sdk_dgroup = self.parser.add_argument_group('sdk display options') + sdk_dgroup.add_argument('--list-sdk-env', action='store_true', + default=False, help='sdk list available environment') + + # XXX this option is required but argparse_oe has a bug handling + # required options, seems that don't keep track of already parsed + # options + sdk_rgroup = self.parser.add_argument_group('sdk required options') + sdk_rgroup.add_argument('--sdk-dir', required=False, action='store', + help='sdk installed directory') + + @staticmethod + def _load_manifest(manifest): + pkg_manifest = {} + if manifest: + with open(manifest) as f: + for line in f: + (pkg, arch, version) = line.strip().split() + pkg_manifest[pkg] = (version, arch) + + return pkg_manifest + + def _process_args(self, logger, args): + super(OESDKTestContextExecutor, self)._process_args(logger, args) + + self.tc_kwargs['init']['sdk_dir'] = args.sdk_dir + self.tc_kwargs['init']['sdk_env'] = self.sdk_env + self.tc_kwargs['init']['target_pkg_manifest'] = \ + OESDKTestContextExecutor._load_manifest(args.target_manifest) + self.tc_kwargs['init']['host_pkg_manifest'] = \ + OESDKTestContextExecutor._load_manifest(args.host_manifest) + + @staticmethod + def _get_sdk_environs(sdk_dir): + sdk_env = {} + + environ_pattern = sdk_dir + '/environment-setup-*' + full_sdk_env = glob.glob(sdk_dir + '/environment-setup-*') + for env in full_sdk_env: + m = re.search('environment-setup-(.*)', env) + if m: + sdk_env[m.group(1)] = env + + return sdk_env + + def _display_sdk_envs(self, log, args, sdk_envs): + log("Available SDK environments at directory %s:" \ + % args.sdk_dir) + log("") + for env in sdk_envs: + log(env) + + def run(self, logger, args): + if not args.sdk_dir: + raise argparse_oe.ArgumentUsageError("No SDK directory "\ + "specified please do, --sdk-dir SDK_DIR", self.name) + + sdk_envs = OESDKTestContextExecutor._get_sdk_environs(args.sdk_dir) + if not sdk_envs: + raise argparse_oe.ArgumentUsageError("No available SDK "\ + "enviroments found at %s" % args.sdk_dir, self.name) + + if args.list_sdk_env: + self._display_sdk_envs(logger.info, args, sdk_envs) + sys.exit(0) + + if not args.sdk_env in sdk_envs: + self._display_sdk_envs(logger.error, args, sdk_envs) + raise argparse_oe.ArgumentUsageError("No valid SDK "\ + "environment (%s) specified" % args.sdk_env, self.name) + + self.sdk_env = sdk_envs[args.sdk_env] + super(OESDKTestContextExecutor, self).run(logger, args) + +_executor_class = OESDKTestContextExecutor diff --git a/meta/lib/oeqa/sdk/files/testsdkmakefile b/meta/lib/oeqa/sdk/files/testsdkmakefile new file mode 100644 index 0000000000..fb05f822f3 --- /dev/null +++ b/meta/lib/oeqa/sdk/files/testsdkmakefile @@ -0,0 +1,5 @@ +test: test.o + $(CC) -o test test.o -lm +test.o: test.c + $(CC) -c test.c + diff --git a/meta/lib/oeqa/sdk/utils/__init__.py b/meta/lib/oeqa/sdk/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/sdk/utils/__init__.py diff --git a/meta/lib/oeqa/sdk/utils/sdkbuildproject.py b/meta/lib/oeqa/sdk/utils/sdkbuildproject.py new file mode 100644 index 0000000000..4e251142d7 --- /dev/null +++ b/meta/lib/oeqa/sdk/utils/sdkbuildproject.py @@ -0,0 +1,45 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import subprocess + +from oeqa.utils.buildproject import BuildProject + +class SDKBuildProject(BuildProject): + def __init__(self, testpath, sdkenv, uri, testlogdir, builddatetime, + foldername=None, dl_dir=None): + self.sdkenv = sdkenv + self.testdir = testpath + self.targetdir = testpath + os.makedirs(testpath, exist_ok=True) + self.datetime = builddatetime + self.testlogdir = testlogdir + os.makedirs(self.testlogdir, exist_ok=True) + self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime) + BuildProject.__init__(self, uri, foldername, tmpdir=testpath, dl_dir=dl_dir) + + def download_archive(self): + + self._download_archive() + + cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir) + subprocess.check_output(cmd, shell=True) + + #Change targetdir to project folder + self.targetdir = os.path.join(self.targetdir, self.fname) + + def run_configure(self, configure_args='', extra_cmds=''): + return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=extra_cmds) + + def run_install(self, install_args=''): + return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir)) + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s\n" % msg) + + def _run(self, cmd): + self.log("Running . %s; " % self.sdkenv + cmd) + return subprocess.call(". %s; " % self.sdkenv + cmd, shell=True) diff --git a/meta/lib/oeqa/sdkext/__init__.py b/meta/lib/oeqa/sdkext/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/sdkext/__init__.py diff --git a/meta/lib/oeqa/sdkext/case.py b/meta/lib/oeqa/sdkext/case.py new file mode 100644 index 0000000000..21b718831c --- /dev/null +++ b/meta/lib/oeqa/sdkext/case.py @@ -0,0 +1,21 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import subprocess + +from oeqa.utils import avoid_paths_in_environ +from oeqa.sdk.case import OESDKTestCase + +class OESDKExtTestCase(OESDKTestCase): + def _run(self, cmd): + # extensible sdk shows a warning if found bitbake in the path + # because can cause contamination, i.e. use devtool from + # poky/scripts instead of eSDK one. + env = os.environ.copy() + paths_to_avoid = ['bitbake/bin', 'poky/scripts'] + env['PATH'] = avoid_paths_in_environ(paths_to_avoid) + + return subprocess.check_output(". %s > /dev/null;"\ + " %s;" % (self.tc.sdk_env, cmd), stderr=subprocess.STDOUT, + shell=True, env=env, universal_newlines=True) diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py new file mode 100644 index 0000000000..a01bc0bfe2 --- /dev/null +++ b/meta/lib/oeqa/sdkext/cases/devtool.py @@ -0,0 +1,97 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import shutil +import subprocess + +from oeqa.sdkext.case import OESDKExtTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class DevtoolTest(OESDKExtTestCase): + @classmethod + def setUpClass(cls): + myapp_src = os.path.join(cls.tc.esdk_files_dir, "myapp") + cls.myapp_dst = os.path.join(cls.tc.sdk_dir, "myapp") + shutil.copytree(myapp_src, cls.myapp_dst) + + myapp_cmake_src = os.path.join(cls.tc.esdk_files_dir, "myapp_cmake") + cls.myapp_cmake_dst = os.path.join(cls.tc.sdk_dir, "myapp_cmake") + shutil.copytree(myapp_cmake_src, cls.myapp_cmake_dst) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.myapp_dst) + shutil.rmtree(cls.myapp_cmake_dst) + + def _test_devtool_build(self, directory): + self._run('devtool add myapp %s' % directory) + try: + self._run('devtool build myapp') + finally: + self._run('devtool reset myapp') + + def _test_devtool_build_package(self, directory): + self._run('devtool add myapp %s' % directory) + try: + self._run('devtool package myapp') + finally: + self._run('devtool reset myapp') + + def test_devtool_location(self): + output = self._run('which devtool') + self.assertEqual(output.startswith(self.tc.sdk_dir), True, \ + msg="Seems that devtool isn't the eSDK one: %s" % output) + + @OETestDepends(['test_devtool_location']) + def test_devtool_add_reset(self): + self._run('devtool add myapp %s' % self.myapp_dst) + self._run('devtool reset myapp') + + @OETestID(1605) + @OETestDepends(['test_devtool_location']) + def test_devtool_build_make(self): + self._test_devtool_build(self.myapp_dst) + + @OETestID(1606) + @OETestDepends(['test_devtool_location']) + def test_devtool_build_esdk_package(self): + self._test_devtool_build_package(self.myapp_dst) + + @OETestID(1607) + @OETestDepends(['test_devtool_location']) + def test_devtool_build_cmake(self): + self._test_devtool_build(self.myapp_cmake_dst) + + @OETestID(1608) + @OETestDepends(['test_devtool_location']) + def test_extend_autotools_recipe_creation(self): + req = 'https://github.com/rdfa/librdfa' + recipe = "librdfa" + self._run('devtool sdk-install libxml2') + self._run('devtool add %s %s' % (recipe, req) ) + try: + self._run('devtool build %s' % recipe) + finally: + self._run('devtool reset %s' % recipe) + + @OETestID(1609) + @OETestDepends(['test_devtool_location']) + def test_devtool_kernelmodule(self): + docfile = 'https://github.com/umlaeute/v4l2loopback.git' + recipe = 'v4l2loopback-driver' + self._run('devtool add %s %s' % (recipe, docfile) ) + try: + self._run('devtool build %s' % recipe) + finally: + self._run('devtool reset %s' % recipe) + + @OETestID(1610) + @OETestDepends(['test_devtool_location']) + def test_recipes_for_nodejs(self): + package_nodejs = "npm://registry.npmjs.org;name=winston;version=2.2.0" + self._run('devtool add %s ' % package_nodejs) + try: + self._run('devtool build %s ' % package_nodejs) + finally: + self._run('devtool reset %s '% package_nodejs) diff --git a/meta/lib/oeqa/sdkext/cases/sdk_update.py b/meta/lib/oeqa/sdkext/cases/sdk_update.py new file mode 100644 index 0000000000..2f8598bbe5 --- /dev/null +++ b/meta/lib/oeqa/sdkext/cases/sdk_update.py @@ -0,0 +1,39 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import shutil +import subprocess + +from oeqa.sdkext.case import OESDKExtTestCase +from oeqa.utils.httpserver import HTTPService + +class SdkUpdateTest(OESDKExtTestCase): + @classmethod + def setUpClass(self): + self.publish_dir = os.path.join(self.tc.sdk_dir, 'esdk_publish') + if os.path.exists(self.publish_dir): + shutil.rmtree(self.publish_dir) + os.mkdir(self.publish_dir) + + base_tcname = "%s/%s" % (self.td.get("SDK_DEPLOY", ''), + self.td.get("TOOLCHAINEXT_OUTPUTNAME", '')) + tcname_new = "%s-new.sh" % base_tcname + if not os.path.exists(tcname_new): + tcname_new = "%s.sh" % base_tcname + + cmd = 'oe-publish-sdk %s %s' % (tcname_new, self.publish_dir) + subprocess.check_output(cmd, shell=True) + + self.http_service = HTTPService(self.publish_dir) + self.http_service.start() + + self.http_url = "http://127.0.0.1:%d" % self.http_service.port + + def test_sdk_update_http(self): + output = self._run("devtool sdk-update \"%s\"" % self.http_url) + + @classmethod + def tearDownClass(self): + self.http_service.stop() + shutil.rmtree(self.publish_dir) diff --git a/meta/lib/oeqa/sdkext/context.py b/meta/lib/oeqa/sdkext/context.py new file mode 100644 index 0000000000..65da4c6e1b --- /dev/null +++ b/meta/lib/oeqa/sdkext/context.py @@ -0,0 +1,29 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor + +class OESDKExtTestContext(OESDKTestContext): + esdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + + # FIXME - We really need to do better mapping of names here, this at + # least allows some tests to run + def hasHostPackage(self, pkg): + # We force a toolchain to be installed into the eSDK even if its minimal + if pkg.startswith("packagegroup-cross-canadian-"): + return True + return self._hasPackage(self.host_pkg_manifest, pkg) + +class OESDKExtTestContextExecutor(OESDKTestContextExecutor): + _context_class = OESDKExtTestContext + + name = 'esdk' + help = 'esdk test component' + description = 'executes esdk tests' + + default_cases = OESDKTestContextExecutor.default_cases + \ + [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'cases')] + default_test_data = None + +_executor_class = OESDKExtTestContextExecutor diff --git a/meta/lib/oeqa/sdkext/files/myapp/Makefile b/meta/lib/oeqa/sdkext/files/myapp/Makefile new file mode 100644 index 0000000000..abd91bea68 --- /dev/null +++ b/meta/lib/oeqa/sdkext/files/myapp/Makefile @@ -0,0 +1,10 @@ +all: myapp + +myapp: myapp.o + $(CC) $(LDFLAGS) $< -o $@ + +myapp.o: myapp.c + $(CC) $(CFLAGS) -c $< -o $@ + +clean: + rm -rf myapp.o myapp diff --git a/meta/lib/oeqa/sdkext/files/myapp/myapp.c b/meta/lib/oeqa/sdkext/files/myapp/myapp.c new file mode 100644 index 0000000000..f0b63f03f3 --- /dev/null +++ b/meta/lib/oeqa/sdkext/files/myapp/myapp.c @@ -0,0 +1,9 @@ +#include <stdio.h> + +int +main(int argc, char *argv[]) +{ + printf("Hello world\n"); + + return 0; +} diff --git a/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt b/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt new file mode 100644 index 0000000000..19d773dd63 --- /dev/null +++ b/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required (VERSION 2.6) +project (myapp) +# The version number. +set (myapp_VERSION_MAJOR 1) +set (myapp_VERSION_MINOR 0) + +# add the executable +add_executable (myapp myapp.c) + +install(TARGETS myapp + RUNTIME DESTINATION bin) diff --git a/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c b/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c new file mode 100644 index 0000000000..f0b63f03f3 --- /dev/null +++ b/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c @@ -0,0 +1,9 @@ +#include <stdio.h> + +int +main(int argc, char *argv[]) +{ + printf("Hello world\n"); + + return 0; +} diff --git a/meta/lib/oeqa/selftest/_toaster.py b/meta/lib/oeqa/selftest/_toaster.py deleted file mode 100644 index 1cf28a0144..0000000000 --- a/meta/lib/oeqa/selftest/_toaster.py +++ /dev/null @@ -1,445 +0,0 @@ -import unittest -import os -import sys -import shlex, subprocess -import urllib, commands, time, getpass, re, json, shlex - -import oeqa.utils.ftools as ftools -from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../', 'bitbake/lib/toaster'))) -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings") - -import toastermain.settings -from django.db.models import Q -from orm.models import * -from oeqa.utils.decorators import testcase - -class ToasterSetup(oeSelfTest): - - def recipe_parse(self, file_path, var): - for line in open(file_path,'r'): - if line.find(var) > -1: - val = line.split(" = ")[1].replace("\"", "").strip() - return val - - def fix_file_path(self, file_path): - if ":" in file_path: - file_path=file_path.split(":")[2] - return file_path - -class Toaster_DB_Tests(ToasterSetup): - - # Check if build name is unique - tc_id=795 - @testcase(795) - def test_Build_Unique_Name(self): - all_builds = Build.objects.all().count() - distinct_builds = Build.objects.values('id').distinct().count() - self.assertEqual(distinct_builds, all_builds, msg = 'Build name is not unique') - - # Check if build coocker log path is unique - tc_id=819 - @testcase(819) - def test_Build_Unique_Cooker_Log_Path(self): - distinct_path = Build.objects.values('cooker_log_path').distinct().count() - total_builds = Build.objects.values('id').count() - self.assertEqual(distinct_path, total_builds, msg = 'Build coocker log path is not unique') - - # Check if the number of errors matches the number of orm_logmessage.level entries with value 2 - tc_id=820 - @testcase(820) - def test_Build_Errors_No(self): - builds = Build.objects.values('id', 'errors_no') - cnt_err = [] - for build in builds: - log_mess_err_no = LogMessage.objects.filter(build = build['id'], level = 2).count() - if (build['errors_no'] != log_mess_err_no): - cnt_err.append(build['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) - - # Check if the number of warnings matches the number of orm_logmessage.level entries with value 1 - tc=821 - @testcase(821) - def test_Build_Warnings_No(self): - builds = Build.objects.values('id', 'warnings_no') - cnt_err = [] - for build in builds: - log_mess_warn_no = LogMessage.objects.filter(build = build['id'], level = 1).count() - if (build['warnings_no'] != log_mess_warn_no): - cnt_err.append(build['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) - - # Check if the build succeeded then the errors_no is 0 - tc_id=822 - @testcase(822) - def test_Build_Suceeded_Errors_No(self): - builds = Build.objects.filter(outcome = 0).values('id', 'errors_no') - cnt_err = [] - for build in builds: - if (build['errors_no'] != 0): - cnt_err.append(build['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) - - # Check if task order is unique for one build - tc=824 - @testcase(824) - def test_Task_Unique_Order(self): - builds = Build.objects.values('id') - cnt_err = [] - for build in builds: - total_task_order = Task.objects.filter(build = build['id']).values('order').count() - distinct_task_order = Task.objects.filter(build = build['id']).values('order').distinct().count() - if (total_task_order != distinct_task_order): - cnt_err.append(build['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) - - # Check task order sequence for one build - tc=825 - @testcase(825) - def test_Task_Order_Sequence(self): - builds = builds = Build.objects.values('id') - cnt_err = [] - for build in builds: - tasks = Task.objects.filter(Q(build = build['id']), ~Q(order = None), ~Q(task_name__contains = '_setscene')).values('id', 'order').order_by("order") - cnt_tasks = 0 - for task in tasks: - cnt_tasks += 1 - if (task['order'] != cnt_tasks): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if disk_io matches the difference between EndTimeIO and StartTimeIO in build stats - tc=828 - ### this needs to be updated ### - #def test_Task_Disk_IO_TC828(self): - - # Check if outcome = 2 (SSTATE) then sstate_result must be 3 (RESTORED) - tc=832 - @testcase(832) - def test_Task_If_Outcome_2_Sstate_Result_Must_Be_3(self): - tasks = Task.objects.filter(outcome = 2).values('id', 'sstate_result') - cnt_err = [] - for task in tasks: - if (row['sstate_result'] != 3): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if outcome = 1 (COVERED) or 3 (EXISTING) then sstate_result must be 0 (SSTATE_NA) - tc=833 - @testcase(833) - def test_Task_If_Outcome_1_3_Sstate_Result_Must_Be_0(self): - tasks = Task.objects.filter(outcome__in = (1, 3)).values('id', 'sstate_result') - cnt_err = [] - for task in tasks: - if (task['sstate_result'] != 0): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if outcome is 0 (SUCCESS) or 4 (FAILED) then sstate_result must be 0 (NA), 1 (MISS) or 2 (FAILED) - tc=834 - @testcase(834) - def test_Task_If_Outcome_0_4_Sstate_Result_Must_Be_0_1_2(self): - tasks = Task.objects.filter(outcome__in = (0, 4)).values('id', 'sstate_result') - cnt_err = [] - for task in tasks: - if (task['sstate_result'] not in [0, 1, 2]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = TRUE (1), script_type must be 0 (CODING_NA), 2 (CODING_PYTHON), 3 (CODING_SHELL) - tc=891 - @testcase(891) - def test_Task_If_Task_Executed_True_Script_Type_0_2_3(self): - tasks = Task.objects.filter(task_executed = 1).values('id', 'script_type') - cnt_err = [] - for task in tasks: - if (task['script_type'] not in [0, 2, 3]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = TRUE (1), outcome must be 0 (SUCCESS) or 4 (FAILED) - tc=836 - @testcase(836) - def test_Task_If_Task_Executed_True_Outcome_0_4(self): - tasks = Task.objects.filter(task_executed = 1).values('id', 'outcome') - cnt_err = [] - for task in tasks: - if (task['outcome'] not in [0, 4]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = FALSE (0), script_type must be 0 - tc=890 - @testcase(890) - def test_Task_If_Task_Executed_False_Script_Type_0(self): - tasks = Task.objects.filter(task_executed = 0).values('id', 'script_type') - cnt_err = [] - for task in tasks: - if (task['script_type'] != 0): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = FALSE (0) and build outcome = SUCCEEDED (0), task outcome must be 1 (COVERED), 2 (CACHED), 3 (PREBUILT), 5 (EMPTY) - tc=837 - @testcase(837) - def test_Task_If_Task_Executed_False_Outcome_1_2_3_5(self): - builds = Build.objects.filter(outcome = 0).values('id') - cnt_err = [] - for build in builds: - tasks = Task.objects.filter(build = build['id'], task_executed = 0).values('id', 'outcome') - for task in tasks: - if (task['outcome'] not in [1, 2, 3, 5]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Key verification - tc=888 - @testcase(888) - def test_Target_Installed_Package(self): - rows = Target_Installed_Package.objects.values('id', 'target_id', 'package_id') - cnt_err = [] - for row in rows: - target = Target.objects.filter(id = row['target_id']).values('id') - package = Package.objects.filter(id = row['package_id']).values('id') - if (not target or not package): - cnt_err.append(row['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for target installed package id: %s' % cnt_err) - - # Key verification - tc=889 - @testcase(889) - def test_Task_Dependency(self): - rows = Task_Dependency.objects.values('id', 'task_id', 'depends_on_id') - cnt_err = [] - for row in rows: - task_id = Task.objects.filter(id = row['task_id']).values('id') - depends_on_id = Task.objects.filter(id = row['depends_on_id']).values('id') - if (not task_id or not depends_on_id): - cnt_err.append(row['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task dependency id: %s' % cnt_err) - - # Check if build target file_name is populated only if is_image=true AND orm_build.outcome=0 then if the file exists and its size matches the file_size value - ### Need to add the tc in the test run - @testcase(1037) - def test_Target_File_Name_Populated(self): - builds = Build.objects.filter(outcome = 0).values('id') - for build in builds: - targets = Target.objects.filter(build_id = build['id'], is_image = 1).values('id') - for target in targets: - target_files = Target_Image_File.objects.filter(target_id = target['id']).values('id', 'file_name', 'file_size') - cnt_err = [] - for file_info in target_files: - target_id = file_info['id'] - target_file_name = file_info['file_name'] - target_file_size = file_info['file_size'] - if (not target_file_name or not target_file_size): - cnt_err.append(target_id) - else: - if (not os.path.exists(target_file_name)): - cnt_err.append(target_id) - else: - if (os.path.getsize(target_file_name) != target_file_size): - cnt_err.append(target_id) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for target image file id: %s' % cnt_err) - - # Key verification - tc=884 - @testcase(884) - def test_Package_Dependency(self): - cnt_err = [] - deps = Package_Dependency.objects.values('id', 'package_id', 'depends_on_id') - for dep in deps: - if (dep['package_id'] == dep['depends_on_id']): - cnt_err.append(dep['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package dependency id: %s' % cnt_err) - - # Check if recipe name does not start with a number (0-9) - tc=838 - @testcase(838) - def test_Recipe_Name(self): - recipes = Recipe.objects.values('id', 'name') - cnt_err = [] - for recipe in recipes: - if (recipe['name'][0].isdigit() is True): - cnt_err.append(recipe['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) - - # Check if recipe section matches the content of the SECTION variable (if set) in file_path - tc=839 - @testcase(839) - def test_Recipe_DB_Section_Match_Recipe_File_Section(self): - recipes = Recipe.objects.values('id', 'section', 'file_path') - cnt_err = [] - for recipe in recipes: - file_path = self.fix_file_path(recipe['file_path']) - file_exists = os.path.isfile(file_path) - if (not file_path or (file_exists is False)): - cnt_err.append(recipe['id']) - else: - file_section = self.recipe_parse(file_path, "SECTION = ") - db_section = recipe['section'] - if file_section: - if (db_section != file_section): - cnt_err.append(recipe['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) - - # Check if recipe license matches the content of the LICENSE variable (if set) in file_path - tc=840 - @testcase(840) - def test_Recipe_DB_License_Match_Recipe_File_License(self): - recipes = Recipe.objects.values('id', 'license', 'file_path') - cnt_err = [] - for recipe in recipes: - file_path = self.fix_file_path(recipe['file_path']) - file_exists = os.path.isfile(file_path) - if (not file_path or (file_exists is False)): - cnt_err.append(recipe['id']) - else: - file_license = self.recipe_parse(file_path, "LICENSE = ") - db_license = recipe['license'] - if file_license: - if (db_license != file_license): - cnt_err.append(recipe['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) - - # Check if recipe homepage matches the content of the HOMEPAGE variable (if set) in file_path - tc=841 - @testcase(841) - def test_Recipe_DB_Homepage_Match_Recipe_File_Homepage(self): - recipes = Recipe.objects.values('id', 'homepage', 'file_path') - cnt_err = [] - for recipe in recipes: - file_path = self.fix_file_path(recipe['file_path']) - file_exists = os.path.isfile(file_path) - if (not file_path or (file_exists is False)): - cnt_err.append(recipe['id']) - else: - file_homepage = self.recipe_parse(file_path, "HOMEPAGE = ") - db_homepage = recipe['homepage'] - if file_homepage: - if (db_homepage != file_homepage): - cnt_err.append(recipe['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) - - # Check if recipe bugtracker matches the content of the BUGTRACKER variable (if set) in file_path - tc=842 - @testcase(842) - def test_Recipe_DB_Bugtracker_Match_Recipe_File_Bugtracker(self): - recipes = Recipe.objects.values('id', 'bugtracker', 'file_path') - cnt_err = [] - for recipe in recipes: - file_path = self.fix_file_path(recipe['file_path']) - file_exists = os.path.isfile(file_path) - if (not file_path or (file_exists is False)): - cnt_err.append(recipe['id']) - else: - file_bugtracker = self.recipe_parse(file_path, "BUGTRACKER = ") - db_bugtracker = recipe['bugtracker'] - if file_bugtracker: - if (db_bugtracker != file_bugtracker): - cnt_err.append(recipe['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe id: %s' % cnt_err) - - # Recipe key verification, recipe name does not depends on a recipe having the same name - tc=883 - @testcase(883) - def test_Recipe_Dependency(self): - deps = Recipe_Dependency.objects.values('id', 'recipe_id', 'depends_on_id') - cnt_err = [] - for dep in deps: - if (not dep['recipe_id'] or not dep['depends_on_id']): - cnt_err.append(dep['id']) - else: - name = Recipe.objects.filter(id = dep['recipe_id']).values('name') - dep_name = Recipe.objects.filter(id = dep['depends_on_id']).values('name') - if (name == dep_name): - cnt_err.append(dep['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe dependency id: %s' % cnt_err) - - # Check if package name does not start with a number (0-9) - tc=846 - @testcase(846) - def test_Package_Name_For_Number(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'name') - cnt_err = [] - for package in packages: - if (package['name'][0].isdigit() is True): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if package version starts with a number (0-9) - tc=847 - @testcase(847) - def test_Package_Version_Starts_With_Number(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'version') - cnt_err = [] - for package in packages: - if (package['version'][0].isdigit() is False): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if package revision starts with 'r' - tc=848 - @testcase(848) - def test_Package_Revision_Starts_With_r(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'revision') - cnt_err = [] - for package in packages: - if (package['revision'][0].startswith("r") is False): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check the validity of the package build_id - ### TC must be added in test run - @testcase(1038) - def test_Package_Build_Id(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'build_id') - cnt_err = [] - for package in packages: - build_id = Build.objects.filter(id = package['build_id']).values('id') - if (not build_id): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check the validity of package recipe_id - ### TC must be added in test run - @testcase(1039) - def test_Package_Recipe_Id(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'recipe_id') - cnt_err = [] - for package in packages: - recipe_id = Recipe.objects.filter(id = package['recipe_id']).values('id') - if (not recipe_id): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if package installed_size field is not null - ### TC must be aded in test run - @testcase(1040) - def test_Package_Installed_Size_Not_NULL(self): - packages = Package.objects.filter(installed_size__isnull = True).values('id') - cnt_err = [] - for package in packages: - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if all layers requests return exit code is 200 - tc=843 - @testcase(843) - def test_Layers_Requests_Exit_Code(self): - layers = Layer.objects.values('id', 'layer_index_url') - cnt_err = [] - for layer in layers: - resp = urllib.urlopen(layer['layer_index_url']) - if (resp.getcode() != 200): - cnt_err.append(layer['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) - - # Check if the output of bitbake-layers show_layers matches the info from database - tc=895 - @testcase(895) - def test_Layers_Show_Layers(self): - layers = Layer.objects.values('id', 'name', 'local_path') - cmd = commands.getoutput('bitbake-layers show_layers') - cnt_err = [] - for layer in layers: - if (layer['name'] or layer['local_path']) not in cmd: - cnt_err.append(layer['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) - - # Check if django server starts regardless of the timezone set on the machine - tc=905 - @testcase(905) - def test_Start_Django_Timezone(self): - current_path = os.getcwd() - zonefilelist = [] - ZONEINFOPATH = '/usr/share/zoneinfo/' - os.chdir("../bitbake/lib/toaster/") - cnt_err = 0 - for filename in os.listdir(ZONEINFOPATH): - if os.path.isfile(os.path.join(ZONEINFOPATH, filename)): - zonefilelist.append(filename) - for k in range(len(zonefilelist)): - if k <= 5: - files = zonefilelist[k] - os.system("export TZ="+str(files)+"; python manage.py runserver > /dev/null 2>&1 &") - time.sleep(3) - pid = subprocess.check_output("ps aux | grep '[/u]sr/bin/python manage.py runserver' | awk '{print $2}'", shell = True) - if pid: - os.system("kill -9 "+str(pid)) - else: - cnt_err.append(zonefilelist[k]) - self.assertEqual(cnt_err, 0, msg = 'Errors django server does not start with timezone: %s' % cnt_err) - os.chdir(current_path) diff --git a/meta/lib/oeqa/selftest/archiver.py b/meta/lib/oeqa/selftest/archiver.py new file mode 100644 index 0000000000..d7f215cbf6 --- /dev/null +++ b/meta/lib/oeqa/selftest/archiver.py @@ -0,0 +1,43 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake, get_bb_vars +from oeqa.utils.decorators import testcase +import glob +import os +import shutil + + +class Archiver(oeSelfTest): + + @testcase(1345) + def test_archiver_allows_to_filter_on_recipe_name(self): + """ + Summary: The archiver should offer the possibility to filter on the recipe. (#6929) + Expected: 1. Included recipe (busybox) should be included + 2. Excluded recipe (zlib) should be excluded + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + include_recipe = 'busybox' + exclude_recipe = 'zlib' + + features = 'INHERIT += "archiver"\n' + features += 'ARCHIVER_MODE[src] = "original"\n' + features += 'COPYLEFT_PN_INCLUDE = "%s"\n' % include_recipe + features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % exclude_recipe + self.write_config(features) + + bitbake('-c clean %s %s' % (include_recipe, exclude_recipe)) + bitbake("%s %s" % (include_recipe, exclude_recipe)) + + bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS']) + src_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS']) + + # Check that include_recipe was included + included_present = len(glob.glob(src_path + '/%s-*' % include_recipe)) + self.assertTrue(included_present, 'Recipe %s was not included.' % include_recipe) + + # Check that exclude_recipe was excluded + excluded_present = len(glob.glob(src_path + '/%s-*' % exclude_recipe)) + self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % exclude_recipe) diff --git a/meta/lib/oeqa/selftest/base.py b/meta/lib/oeqa/selftest/base.py index 80b9b4b312..47a8ea8271 100644 --- a/meta/lib/oeqa/selftest/base.py +++ b/meta/lib/oeqa/selftest/base.py @@ -4,7 +4,7 @@ # DESCRIPTION -# Base class inherited by test classes in meta/lib/selftest +# Base class inherited by test classes in meta/lib/oeqa/selftest import unittest import os @@ -16,6 +16,8 @@ import errno import oeqa.utils.ftools as ftools from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer from oeqa.utils.decorators import LogResults +from random import choice +import glob @LogResults class oeSelfTest(unittest.TestCase): @@ -26,14 +28,47 @@ class oeSelfTest(unittest.TestCase): def __init__(self, methodName="runTest"): self.builddir = os.environ.get("BUILDDIR") self.localconf_path = os.path.join(self.builddir, "conf/local.conf") + self.localconf_backup = os.path.join(self.builddir, "conf/local.bk") self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc") + self.local_bblayers_path = os.path.join(self.builddir, "conf/bblayers.conf") + self.local_bblayers_backup = os.path.join(self.builddir, + "conf/bblayers.bk") + self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc") + self.machineinc_path = os.path.join(self.builddir, "conf/machine.inc") self.testlayer_path = oeSelfTest.testlayer_path self._extra_tear_down_commands = [] - self._track_for_cleanup = [] + self._track_for_cleanup = [ + self.testinc_path, self.testinc_bblayers_path, + self.machineinc_path, self.localconf_backup, + self.local_bblayers_backup] super(oeSelfTest, self).__init__(methodName) def setUp(self): os.chdir(self.builddir) + # Check if local.conf or bblayers.conf files backup exists + # from a previous failed test and restore them + if os.path.isfile(self.localconf_backup) or os.path.isfile( + self.local_bblayers_backup): + self.log.debug("Found a local.conf and/or bblayers.conf backup \ +from a previously aborted test. Restoring these files now, but tests should \ +be re-executed from a clean environment to ensure accurate results.") + try: + shutil.copyfile(self.localconf_backup, self.localconf_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + try: + shutil.copyfile(self.local_bblayers_backup, + self.local_bblayers_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + else: + # backup local.conf and bblayers.conf + shutil.copyfile(self.localconf_path, self.localconf_backup) + shutil.copyfile(self.local_bblayers_path, + self.local_bblayers_backup) + self.log.debug("Creating local.conf and bblayers.conf backups.") # we don't know what the previous test left around in config or inc files # if it failed so we need a fresh start try: @@ -45,6 +80,25 @@ class oeSelfTest(unittest.TestCase): for f in files: if f == 'test_recipe.inc': os.remove(os.path.join(root, f)) + + for incl_file in [self.testinc_bblayers_path, self.machineinc_path]: + try: + os.remove(incl_file) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + # Get CUSTOMMACHINE from env (set by --machine argument to oe-selftest) + custommachine = os.getenv('CUSTOMMACHINE') + if custommachine: + if custommachine == 'random': + machine = get_random_machine() + else: + machine = custommachine + machine_conf = 'MACHINE ??= "%s"\n' % machine + self.set_machine_config(machine_conf) + print('MACHINE: %s' % machine) + # tests might need their own setup # but if they overwrite this one they have to call # super each time, so let's give them an alternative @@ -92,14 +146,24 @@ class oeSelfTest(unittest.TestCase): self.log.debug("Writing to: %s\n%s\n" % (self.testinc_path, data)) ftools.write_file(self.testinc_path, data) + custommachine = os.getenv('CUSTOMMACHINE') + if custommachine and 'MACHINE' in data: + machine = get_bb_var('MACHINE') + self.log.warning('MACHINE overridden: %s' % machine) + # append to <builddir>/conf/selftest.inc def append_config(self, data): self.log.debug("Appending to: %s\n%s\n" % (self.testinc_path, data)) ftools.append_file(self.testinc_path, data) + custommachine = os.getenv('CUSTOMMACHINE') + if custommachine and 'MACHINE' in data: + machine = get_bb_var('MACHINE') + self.log.warning('MACHINE overridden: %s' % machine) + # remove data from <builddir>/conf/selftest.inc def remove_config(self, data): - self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data)) + self.log.debug("Removing from: %s\n%s\n" % (self.testinc_path, data)) ftools.remove_from_file(self.testinc_path, data) # write to meta-sefltest/recipes-test/<recipe>/test_recipe.inc @@ -129,3 +193,43 @@ class oeSelfTest(unittest.TestCase): except OSError as e: if e.errno != errno.ENOENT: raise + + # write to <builddir>/conf/bblayers.inc + def write_bblayers_config(self, data): + self.log.debug("Writing to: %s\n%s\n" % (self.testinc_bblayers_path, data)) + ftools.write_file(self.testinc_bblayers_path, data) + + # append to <builddir>/conf/bblayers.inc + def append_bblayers_config(self, data): + self.log.debug("Appending to: %s\n%s\n" % (self.testinc_bblayers_path, data)) + ftools.append_file(self.testinc_bblayers_path, data) + + # remove data from <builddir>/conf/bblayers.inc + def remove_bblayers_config(self, data): + self.log.debug("Removing from: %s\n%s\n" % (self.testinc_bblayers_path, data)) + ftools.remove_from_file(self.testinc_bblayers_path, data) + + # write to <builddir>/conf/machine.inc + def set_machine_config(self, data): + self.log.debug("Writing to: %s\n%s\n" % (self.machineinc_path, data)) + ftools.write_file(self.machineinc_path, data) + + +def get_available_machines(): + # Get a list of all available machines + bbpath = get_bb_var('BBPATH').split(':') + machines = [] + + for path in bbpath: + found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf')) + if found_machines: + for i in found_machines: + # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf' + machines.append(os.path.splitext(os.path.basename(i))[0]) + + return machines + + +def get_random_machine(): + # Get a random machine + return choice(get_available_machines()) diff --git a/meta/lib/oeqa/selftest/bblayers.py b/meta/lib/oeqa/selftest/bblayers.py index 1ead8e8671..cd658c5d4e 100644 --- a/meta/lib/oeqa/selftest/bblayers.py +++ b/meta/lib/oeqa/selftest/bblayers.py @@ -6,7 +6,7 @@ import shutil import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd +from oeqa.utils.commands import runCmd, get_bb_var from oeqa.utils.decorators import testcase class BitbakeLayers(oeSelfTest): @@ -14,30 +14,86 @@ class BitbakeLayers(oeSelfTest): @testcase(756) def test_bitbakelayers_showcrossdepends(self): result = runCmd('bitbake-layers show-cross-depends') - self.assertTrue('aspell' in result.output) + self.assertTrue('aspell' in result.output, msg = "No dependencies were shown. bitbake-layers show-cross-depends output: %s" % result.output) @testcase(83) def test_bitbakelayers_showlayers(self): - result = runCmd('bitbake-layers show_layers') - self.assertTrue('meta-selftest' in result.output) + result = runCmd('bitbake-layers show-layers') + self.assertTrue('meta-selftest' in result.output, msg = "No layers were shown. bitbake-layers show-layers output: %s" % result.output) @testcase(93) def test_bitbakelayers_showappends(self): - result = runCmd('bitbake-layers show_appends') - self.assertTrue('xcursor-transparent-theme_0.1.1.bbappend' in result.output, msg='xcursor-transparent-theme_0.1.1.bbappend file was not recognised') + recipe = "xcursor-transparent-theme" + bb_file = self.get_recipe_basename(recipe) + result = runCmd('bitbake-layers show-appends') + self.assertTrue(bb_file in result.output, msg="%s file was not recognised. bitbake-layers show-appends output: %s" % (bb_file, result.output)) @testcase(90) def test_bitbakelayers_showoverlayed(self): - result = runCmd('bitbake-layers show_overlayed') - self.assertTrue('aspell' in result.output, msg='xcursor-transparent-theme_0.1.1.bbappend file was not recognised') + result = runCmd('bitbake-layers show-overlayed') + self.assertTrue('aspell' in result.output, msg="aspell overlayed recipe was not recognised bitbake-layers show-overlayed %s" % result.output) @testcase(95) def test_bitbakelayers_flatten(self): - self.assertFalse(os.path.isdir(os.path.join(self.builddir, 'test'))) - result = runCmd('bitbake-layers flatten test') - bb_file = os.path.join(self.builddir, 'test/recipes-graphics/xcursor-transparent-theme/xcursor-transparent-theme_0.1.1.bb') - self.assertTrue(os.path.isfile(bb_file)) + recipe = "xcursor-transparent-theme" + recipe_path = "recipes-graphics/xcursor-transparent-theme" + recipe_file = self.get_recipe_basename(recipe) + testoutdir = os.path.join(self.builddir, 'test_bitbakelayers_flatten') + self.assertFalse(os.path.isdir(testoutdir), msg = "test_bitbakelayers_flatten should not exist at this point in time") + self.track_for_cleanup(testoutdir) + result = runCmd('bitbake-layers flatten %s' % testoutdir) + bb_file = os.path.join(testoutdir, recipe_path, recipe_file) + self.assertTrue(os.path.isfile(bb_file), msg = "Cannot find xcursor-transparent-theme_0.1.1.bb in the test_bitbakelayers_flatten local dir.") contents = ftools.read_file(bb_file) find_in_contents = re.search("##### bbappended from meta-selftest #####\n(.*\n)*include test_recipe.inc", contents) - shutil.rmtree(os.path.join(self.builddir, 'test')) - self.assertTrue(find_in_contents) + self.assertTrue(find_in_contents, msg = "Flattening layers did not work. bitbake-layers flatten output: %s" % result.output) + + @testcase(1195) + def test_bitbakelayers_add_remove(self): + test_layer = os.path.join(get_bb_var('COREBASE'), 'meta-skeleton') + result = runCmd('bitbake-layers show-layers') + self.assertNotIn('meta-skeleton', result.output, "This test cannot run with meta-skeleton in bblayers.conf. bitbake-layers show-layers output: %s" % result.output) + result = runCmd('bitbake-layers add-layer %s' % test_layer) + result = runCmd('bitbake-layers show-layers') + self.assertIn('meta-skeleton', result.output, msg = "Something wrong happened. meta-skeleton layer was not added to conf/bblayers.conf. bitbake-layers show-layers output: %s" % result.output) + result = runCmd('bitbake-layers remove-layer %s' % test_layer) + result = runCmd('bitbake-layers show-layers') + self.assertNotIn('meta-skeleton', result.output, msg = "meta-skeleton should have been removed at this step. bitbake-layers show-layers output: %s" % result.output) + result = runCmd('bitbake-layers add-layer %s' % test_layer) + result = runCmd('bitbake-layers show-layers') + self.assertIn('meta-skeleton', result.output, msg = "Something wrong happened. meta-skeleton layer was not added to conf/bblayers.conf. bitbake-layers show-layers output: %s" % result.output) + result = runCmd('bitbake-layers remove-layer */meta-skeleton') + result = runCmd('bitbake-layers show-layers') + self.assertNotIn('meta-skeleton', result.output, msg = "meta-skeleton should have been removed at this step. bitbake-layers show-layers output: %s" % result.output) + + @testcase(1384) + def test_bitbakelayers_showrecipes(self): + result = runCmd('bitbake-layers show-recipes') + self.assertIn('aspell:', result.output) + self.assertIn('mtd-utils:', result.output) + self.assertIn('core-image-minimal:', result.output) + result = runCmd('bitbake-layers show-recipes mtd-utils') + self.assertIn('mtd-utils:', result.output) + self.assertNotIn('aspell:', result.output) + result = runCmd('bitbake-layers show-recipes -i image') + self.assertIn('core-image-minimal', result.output) + self.assertNotIn('mtd-utils:', result.output) + result = runCmd('bitbake-layers show-recipes -i cmake,pkgconfig') + self.assertIn('libproxy:', result.output) + self.assertNotIn('mtd-utils:', result.output) # doesn't inherit either + self.assertNotIn('wget:', result.output) # doesn't inherit cmake + self.assertNotIn('waffle:', result.output) # doesn't inherit pkgconfig + result = runCmd('bitbake-layers show-recipes -i nonexistentclass', ignore_status=True) + self.assertNotEqual(result.status, 0, 'bitbake-layers show-recipes -i nonexistentclass should have failed') + self.assertIn('ERROR:', result.output) + + def get_recipe_basename(self, recipe): + recipe_file = "" + result = runCmd("bitbake-layers show-recipes -f %s" % recipe) + for line in result.output.splitlines(): + if recipe in line: + recipe_file = line + break + + self.assertTrue(os.path.isfile(recipe_file), msg = "Can't find recipe file for %s" % recipe) + return os.path.basename(recipe_file) diff --git a/meta/lib/oeqa/selftest/bbtests.py b/meta/lib/oeqa/selftest/bbtests.py index e765e366c1..46e09f509f 100644 --- a/meta/lib/oeqa/selftest/bbtests.py +++ b/meta/lib/oeqa/selftest/bbtests.py @@ -1,56 +1,57 @@ -import unittest import os -import logging import re -import shutil import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars from oeqa.utils.decorators import testcase class BitbakeTests(oeSelfTest): + def getline(self, res, line): + for l in res.output.split('\n'): + if line in l: + return l + @testcase(789) def test_run_bitbake_from_dir_1(self): os.chdir(os.path.join(self.builddir, 'conf')) - bitbake('-e') + self.assertEqual(bitbake('-e').status, 0, msg = "bitbake couldn't run from \"conf\" dir") @testcase(790) def test_run_bitbake_from_dir_2(self): my_env = os.environ.copy() my_env['BBPATH'] = my_env['BUILDDIR'] os.chdir(os.path.dirname(os.environ['BUILDDIR'])) - bitbake('-e', env=my_env) + self.assertEqual(bitbake('-e', env=my_env).status, 0, msg = "bitbake couldn't run from builddir") @testcase(806) def test_event_handler(self): self.write_config("INHERIT += \"test_events\"") result = bitbake('m4-native') - find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Preparing runqueue", result.output) + find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Executing RunQueue Tasks", result.output) find_build_completed = re.search("Tasks Summary:.*(\n.*)*NOTE: Test for bb\.event\.BuildCompleted", result.output) self.assertTrue(find_build_started, msg = "Match failed in:\n%s" % result.output) self.assertTrue(find_build_completed, msg = "Match failed in:\n%s" % result.output) - self.assertFalse('Test for bb.event.InvalidEvent' in result.output) + self.assertFalse('Test for bb.event.InvalidEvent' in result.output, msg = "\"Test for bb.event.InvalidEvent\" message found during bitbake process. bitbake output: %s" % result.output) @testcase(103) def test_local_sstate(self): - bitbake('m4-native -ccleansstate') bitbake('m4-native') bitbake('m4-native -cclean') result = bitbake('m4-native') find_setscene = re.search("m4-native.*do_.*_setscene", result.output) - self.assertTrue(find_setscene) + self.assertTrue(find_setscene, msg = "No \"m4-native.*do_.*_setscene\" message found during bitbake m4-native. bitbake output: %s" % result.output ) @testcase(105) def test_bitbake_invalid_recipe(self): result = bitbake('-b asdf', ignore_status=True) - self.assertTrue("ERROR: Unable to find any recipe file matching 'asdf'" in result.output) + self.assertTrue("ERROR: Unable to find any recipe file matching 'asdf'" in result.output, msg = "Though asdf recipe doesn't exist, bitbake didn't output any err. message. bitbake output: %s" % result.output) @testcase(107) def test_bitbake_invalid_target(self): result = bitbake('asdf', ignore_status=True) - self.assertTrue("ERROR: Nothing PROVIDES 'asdf'" in result.output) + self.assertTrue("ERROR: Nothing PROVIDES 'asdf'" in result.output, msg = "Though no 'asdf' target exists, bitbake didn't output any err. message. bitbake output: %s" % result.output) @testcase(106) def test_warnings_errors(self): @@ -62,117 +63,216 @@ class BitbakeTests(oeSelfTest): @testcase(108) def test_invalid_patch(self): + # This patch already exists in SRC_URI so adding it again will cause the + # patch to fail. self.write_recipeinc('man', 'SRC_URI += "file://man-1.5h1-make.patch"') + self.write_config("INHERIT_remove = \"report-error\"") result = bitbake('man -c patch', ignore_status=True) self.delete_recipeinc('man') bitbake('-cclean man') - self.assertTrue("ERROR: Function failed: patch_do_patch" in result.output) + line = self.getline(result, "Function failed: patch_do_patch") + self.assertTrue(line and line.startswith("ERROR:"), msg = "Repeated patch application didn't fail. bitbake output: %s" % result.output) + + @testcase(1354) + def test_force_task_1(self): + # test 1 from bug 5875 + test_recipe = 'zlib' + test_data = "Microsoft Made No Profit From Anyone's Zunes Yo" + bb_vars = get_bb_vars(['D', 'PKGDEST', 'mandir'], test_recipe) + image_dir = bb_vars['D'] + pkgsplit_dir = bb_vars['PKGDEST'] + man_dir = bb_vars['mandir'] + + bitbake('-c clean %s' % test_recipe) + bitbake('-c package -f %s' % test_recipe) + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + + man_file = os.path.join(image_dir + man_dir, 'man3/zlib.3') + ftools.append_file(man_file, test_data) + bitbake('-c package -f %s' % test_recipe) + + man_split_file = os.path.join(pkgsplit_dir, 'zlib-doc' + man_dir, 'man3/zlib.3') + man_split_content = ftools.read_file(man_split_file) + self.assertIn(test_data, man_split_content, 'The man file has not changed in packages-split.') + + ret = bitbake(test_recipe) + self.assertIn('task do_package_write_rpm:', ret.output, 'Task do_package_write_rpm did not re-executed.') @testcase(163) - def test_force_task(self): - bitbake('m4-native') - result = bitbake('-C compile m4-native') - look_for_tasks = ['do_compile', 'do_install', 'do_populate_sysroot'] + def test_force_task_2(self): + # test 2 from bug 5875 + test_recipe = 'zlib' + + bitbake(test_recipe) + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + + result = bitbake('-C compile %s' % test_recipe) + look_for_tasks = ['do_compile:', 'do_install:', 'do_populate_sysroot:', 'do_package:'] for task in look_for_tasks: - find_task = re.search("m4-native.*%s" % task, result.output) - self.assertTrue(find_task) + self.assertIn(task, result.output, msg="Couldn't find %s task.") @testcase(167) def test_bitbake_g(self): - result = bitbake('-g core-image-full-cmdline') - self.assertTrue('NOTE: PN build list saved to \'pn-buildlist\'' in result.output) - self.assertTrue('openssh' in ftools.read_file(os.path.join(self.builddir, 'pn-buildlist'))) - for f in ['pn-buildlist', 'pn-depends.dot', 'package-depends.dot', 'task-depends.dot']: - os.remove(f) + result = bitbake('-g core-image-minimal') + for f in ['pn-buildlist', 'recipe-depends.dot', 'task-depends.dot']: + self.addCleanup(os.remove, f) + self.assertTrue('Task dependencies saved to \'task-depends.dot\'' in result.output, msg = "No task dependency \"task-depends.dot\" file was generated for the given task target. bitbake output: %s" % result.output) + self.assertTrue('busybox' in ftools.read_file(os.path.join(self.builddir, 'task-depends.dot')), msg = "No \"busybox\" dependency found in task-depends.dot file.") @testcase(899) def test_image_manifest(self): bitbake('core-image-minimal') - deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") - imagename = get_bb_var("IMAGE_LINK_NAME", target="core-image-minimal") + bb_vars = get_bb_vars(["DEPLOY_DIR_IMAGE", "IMAGE_LINK_NAME"], "core-image-minimal") + deploydir = bb_vars["DEPLOY_DIR_IMAGE"] + imagename = bb_vars["IMAGE_LINK_NAME"] manifest = os.path.join(deploydir, imagename + ".manifest") - self.assertTrue(os.path.islink(manifest), msg="No manifest file created for image") + self.assertTrue(os.path.islink(manifest), msg="No manifest file created for image. It should have been created in %s" % manifest) @testcase(168) def test_invalid_recipe_src_uri(self): data = 'SRC_URI = "file://invalid"' self.write_recipeinc('man', data) + self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" +SSTATE_DIR = \"${TOPDIR}/download-selftest\" +INHERIT_remove = \"report-error\" +""") + self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) + bitbake('-ccleanall man') result = bitbake('-c fetch man', ignore_status=True) bitbake('-ccleanall man') self.delete_recipeinc('man') - self.assertEqual(result.status, 1, msg='Command succeded when it should have failed') - self.assertTrue('ERROR: Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output) - self.assertTrue('ERROR: Function failed: Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.' in result.output) + self.assertEqual(result.status, 1, msg="Command succeded when it should have failed. bitbake output: %s" % result.output) + self.assertTrue('Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output, msg = "\"invalid\" file \ +doesn't exist, yet no error message encountered. bitbake output: %s" % result.output) + line = self.getline(result, 'Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.') + self.assertTrue(line and line.startswith("ERROR:"), msg = "\"invalid\" file \ +doesn't exist, yet fetcher didn't report any error. bitbake output: %s" % result.output) @testcase(171) def test_rename_downloaded_file(self): - data = 'SRC_URI_append = ";downloadfilename=test-aspell.tar.gz"' + # TODO unique dldir instead of using cleanall + # TODO: need to set sstatedir? + self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" +SSTATE_DIR = \"${TOPDIR}/download-selftest\" +""") + self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) + + data = 'SRC_URI = "${GNU_MIRROR}/aspell/aspell-${PV}.tar.gz;downloadfilename=test-aspell.tar.gz"' self.write_recipeinc('aspell', data) - bitbake('-ccleanall aspell') - result = bitbake('-c fetch aspell', ignore_status=True) + result = bitbake('-f -c fetch aspell', ignore_status=True) self.delete_recipeinc('aspell') - self.assertEqual(result.status, 0) - self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz'))) - self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz.done'))) - bitbake('-ccleanall aspell') + self.assertEqual(result.status, 0, msg = "Couldn't fetch aspell. %s" % result.output) + dl_dir = get_bb_var("DL_DIR") + self.assertTrue(os.path.isfile(os.path.join(dl_dir, 'test-aspell.tar.gz')), msg = "File rename failed. No corresponding test-aspell.tar.gz file found under %s" % dl_dir) + self.assertTrue(os.path.isfile(os.path.join(dl_dir, 'test-aspell.tar.gz.done')), "File rename failed. No corresponding test-aspell.tar.gz.done file found under %s" % dl_dir) @testcase(1028) def test_environment(self): - self.append_config("TEST_ENV=\"localconf\"") - result = runCmd('bitbake -e | grep TEST_ENV=') - self.assertTrue('localconf' in result.output) - self.remove_config("TEST_ENV=\"localconf\"") + self.write_config("TEST_ENV=\"localconf\"") + result = runCmd('bitbake -e | grep TEST_ENV=') + self.assertTrue('localconf' in result.output, msg = "bitbake didn't report any value for TEST_ENV variable. To test, run 'bitbake -e | grep TEST_ENV='") @testcase(1029) def test_dry_run(self): - result = runCmd('bitbake -n m4-native') - self.assertEqual(0, result.status) + result = runCmd('bitbake -n m4-native') + self.assertEqual(0, result.status, "bitbake dry run didn't run as expected. %s" % result.output) @testcase(1030) def test_just_parse(self): - result = runCmd('bitbake -p') - self.assertEqual(0, result.status) + result = runCmd('bitbake -p') + self.assertEqual(0, result.status, "errors encountered when parsing recipes. %s" % result.output) @testcase(1031) def test_version(self): - result = runCmd('bitbake -s | grep wget') - find = re.search("wget *:([0-9a-zA-Z\.\-]+)", result.output) - self.assertTrue(find) + result = runCmd('bitbake -s | grep wget') + find = re.search("wget *:([0-9a-zA-Z\.\-]+)", result.output) + self.assertTrue(find, "No version returned for searched recipe. bitbake output: %s" % result.output) @testcase(1032) def test_prefile(self): - preconf = os.path.join(self.builddir, 'conf/prefile.conf') - self.track_for_cleanup(preconf) - ftools.write_file(preconf ,"TEST_PREFILE=\"prefile\"") - result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=') - self.assertTrue('prefile' in result.output) - self.append_config("TEST_PREFILE=\"localconf\"") - result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=') - self.assertTrue('localconf' in result.output) - self.remove_config("TEST_PREFILE=\"localconf\"") + preconf = os.path.join(self.builddir, 'conf/prefile.conf') + self.track_for_cleanup(preconf) + ftools.write_file(preconf ,"TEST_PREFILE=\"prefile\"") + result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=') + self.assertTrue('prefile' in result.output, "Preconfigure file \"prefile.conf\"was not taken into consideration. ") + self.write_config("TEST_PREFILE=\"localconf\"") + result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=') + self.assertTrue('localconf' in result.output, "Preconfigure file \"prefile.conf\"was not taken into consideration.") @testcase(1033) def test_postfile(self): - postconf = os.path.join(self.builddir, 'conf/postfile.conf') - self.track_for_cleanup(postconf) - ftools.write_file(postconf , "TEST_POSTFILE=\"postfile\"") - self.append_config("TEST_POSTFILE=\"localconf\"") - result = runCmd('bitbake -R conf/postfile.conf -e | grep TEST_POSTFILE=') - self.assertTrue('postfile' in result.output) - self.remove_config("TEST_POSTFILE=\"localconf\"") + postconf = os.path.join(self.builddir, 'conf/postfile.conf') + self.track_for_cleanup(postconf) + ftools.write_file(postconf , "TEST_POSTFILE=\"postfile\"") + self.write_config("TEST_POSTFILE=\"localconf\"") + result = runCmd('bitbake -R conf/postfile.conf -e | grep TEST_POSTFILE=') + self.assertTrue('postfile' in result.output, "Postconfigure file \"postfile.conf\"was not taken into consideration.") @testcase(1034) def test_checkuri(self): - result = runCmd('bitbake -c checkuri m4') - self.assertEqual(0, result.status) + result = runCmd('bitbake -c checkuri m4') + self.assertEqual(0, result.status, msg = "\"checkuri\" task was not executed. bitbake output: %s" % result.output) @testcase(1035) def test_continue(self): - self.write_recipeinc('man',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" ) - runCmd('bitbake -c cleanall man xcursor-transparent-theme') - result = runCmd('bitbake man xcursor-transparent-theme -k', ignore_status=True) - errorpos = result.output.find('ERROR: Function failed: do_fail_task') - manver = re.search("NOTE: recipe xcursor-transparent-theme-(.*?): task do_unpack: Started", result.output) - continuepos = result.output.find('NOTE: recipe xcursor-transparent-theme-%s: task do_unpack: Started' % manver.group(1)) - self.assertLess(errorpos,continuepos) + self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" +SSTATE_DIR = \"${TOPDIR}/download-selftest\" +INHERIT_remove = \"report-error\" +""") + self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) + self.write_recipeinc('man',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" ) + runCmd('bitbake -c cleanall man xcursor-transparent-theme') + result = runCmd('bitbake -c unpack -k man xcursor-transparent-theme', ignore_status=True) + errorpos = result.output.find('ERROR: Function failed: do_fail_task') + manver = re.search("NOTE: recipe xcursor-transparent-theme-(.*?): task do_unpack: Started", result.output) + continuepos = result.output.find('NOTE: recipe xcursor-transparent-theme-%s: task do_unpack: Started' % manver.group(1)) + self.assertLess(errorpos,continuepos, msg = "bitbake didn't pass do_fail_task. bitbake output: %s" % result.output) + + @testcase(1119) + def test_non_gplv3(self): + self.write_config('INCOMPATIBLE_LICENSE = "GPLv3"') + result = bitbake('selftest-ed', ignore_status=True) + self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output)) + lic_dir = get_bb_var('LICENSE_DIRECTORY') + self.assertFalse(os.path.isfile(os.path.join(lic_dir, 'selftest-ed/generic_GPLv3'))) + self.assertTrue(os.path.isfile(os.path.join(lic_dir, 'selftest-ed/generic_GPLv2'))) + + @testcase(1422) + def test_setscene_only(self): + """ Bitbake option to restore from sstate only within a build (i.e. execute no real tasks, only setscene)""" + test_recipe = 'ed' + + bitbake(test_recipe) + bitbake('-c clean %s' % test_recipe) + ret = bitbake('--setscene-only %s' % test_recipe) + + tasks = re.findall(r'task\s+(do_\S+):', ret.output) + + for task in tasks: + self.assertIn('_setscene', task, 'A task different from _setscene ran: %s.\n' + 'Executed tasks were: %s' % (task, str(tasks))) + + @testcase(1425) + def test_bbappend_order(self): + """ Bitbake should bbappend to recipe in a predictable order """ + test_recipe = 'ed' + bb_vars = get_bb_vars(['SUMMARY', 'PV'], test_recipe) + test_recipe_summary_before = bb_vars['SUMMARY'] + test_recipe_pv = bb_vars['PV'] + recipe_append_file = test_recipe + '_' + test_recipe_pv + '.bbappend' + expected_recipe_summary = test_recipe_summary_before + + for i in range(5): + recipe_append_dir = test_recipe + '_test_' + str(i) + recipe_append_path = os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir, recipe_append_file) + os.mkdir(os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir)) + feature = 'SUMMARY += "%s"\n' % i + ftools.write_file(recipe_append_path, feature) + expected_recipe_summary += ' %s' % i + + self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, 'recipes-test', + test_recipe + '_test_*')) + + test_recipe_summary_after = get_bb_var('SUMMARY', test_recipe) + self.assertEqual(expected_recipe_summary, test_recipe_summary_after) diff --git a/meta/lib/oeqa/selftest/buildhistory.py b/meta/lib/oeqa/selftest/buildhistory.py index d8cae4664b..008c39c956 100644 --- a/meta/lib/oeqa/selftest/buildhistory.py +++ b/meta/lib/oeqa/selftest/buildhistory.py @@ -1,18 +1,17 @@ -import unittest import os import re -import shutil import datetime -import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import Command, runCmd, bitbake, get_bb_var, get_test_layer +from oeqa.utils.commands import bitbake, get_bb_vars +from oeqa.utils.decorators import testcase class BuildhistoryBase(oeSelfTest): def config_buildhistory(self, tmp_bh_location=False): - if (not 'buildhistory' in get_bb_var('USER_CLASSES')) and (not 'buildhistory' in get_bb_var('INHERIT')): + bb_vars = get_bb_vars(['USER_CLASSES', 'INHERIT']) + if (not 'buildhistory' in bb_vars['USER_CLASSES']) and (not 'buildhistory' in bb_vars['INHERIT']): add_buildhistory_config = 'INHERIT += "buildhistory"\nBUILDHISTORY_COMMIT = "1"' self.append_config(add_buildhistory_config) @@ -40,6 +39,9 @@ class BuildhistoryBase(oeSelfTest): if expect_error: self.assertEqual(result.status, 1, msg="Error expected for global config '%s' and target config '%s'" % (global_config, target_config)) search_for_error = re.search(error_regex, result.output) - self.assertTrue(search_for_error, msg="Could not find desired error in output: %s" % error_regex) + self.assertTrue(search_for_error, msg="Could not find desired error in output: %s (%s)" % (error_regex, result.output)) else: self.assertEqual(result.status, 0, msg="Command 'bitbake %s' has failed unexpectedly: %s" % (target, result.output)) + + # No tests should be added to the base class. + # Please create a new class that inherit this one, or use one of those already available for adding tests. diff --git a/meta/lib/oeqa/selftest/buildoptions.py b/meta/lib/oeqa/selftest/buildoptions.py index ec541e5be6..a6e0203f5a 100644 --- a/meta/lib/oeqa/selftest/buildoptions.py +++ b/meta/lib/oeqa/selftest/buildoptions.py @@ -1,111 +1,184 @@ -import unittest import os -import logging import re - +import glob as g +import shutil +import tempfile from oeqa.selftest.base import oeSelfTest from oeqa.selftest.buildhistory import BuildhistoryBase -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import oeqa.utils.ftools as ftools +from oeqa.utils.decorators import testcase class ImageOptionsTests(oeSelfTest): + @testcase(761) def test_incremental_image_generation(self): - bitbake("-c cleanall core-image-minimal") + image_pkgtype = get_bb_var("IMAGE_PKGTYPE") + if image_pkgtype != 'rpm': + self.skipTest('Not using RPM as main package format') + bitbake("-c clean core-image-minimal") self.write_config('INC_RPM_IMAGE_GEN = "1"') self.append_config('IMAGE_FEATURES += "ssh-server-openssh"') bitbake("core-image-minimal") - res = runCmd("grep 'Installing openssh-sshd' %s" % (os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs")), ignore_status=True) + log_data_file = os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs") + log_data_created = ftools.read_file(log_data_file) + incremental_created = re.search("Installing : packagegroup-core-ssh-openssh", log_data_created) self.remove_config('IMAGE_FEATURES += "ssh-server-openssh"') - self.assertEqual(0, res.status, msg="No match for openssh-sshd in log.do_rootfs") - bitbake("core-image-minimal") - res = runCmd("grep 'Removing openssh-sshd' %s" %(os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs")),ignore_status=True) - self.assertEqual(0, res.status, msg="openssh-sshd was not removed from image") - - def test_rm_old_image(self): + self.assertTrue(incremental_created, msg = "Match failed in:\n%s" % log_data_created) bitbake("core-image-minimal") - deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") - imagename = get_bb_var("IMAGE_LINK_NAME", target="core-image-minimal") - deploydir_files = os.listdir(deploydir) - track_original_files = [] - for image_file in deploydir_files: - if imagename in image_file and os.path.islink(os.path.join(deploydir, image_file)): - track_original_files.append(os.path.realpath(os.path.join(deploydir, image_file))) - self.append_config("RM_OLD_IMAGE = \"1\"") - bitbake("-C rootfs core-image-minimal") - deploydir_files = os.listdir(deploydir) - remaining_not_expected = [path for path in track_original_files if os.path.basename(path) in deploydir_files] - self.assertFalse(remaining_not_expected, msg="\nThe following image files ware not removed: %s" % ', '.join(map(str, remaining_not_expected))) + log_data_removed = ftools.read_file(log_data_file) + incremental_removed = re.search("Erasing : packagegroup-core-ssh-openssh", log_data_removed) + self.assertTrue(incremental_removed, msg = "Match failed in:\n%s" % log_data_removed) + @testcase(286) def test_ccache_tool(self): bitbake("ccache-native") - self.assertTrue(os.path.isfile(os.path.join(get_bb_var('STAGING_BINDIR_NATIVE', 'ccache-native'), "ccache"))) + bb_vars = get_bb_vars(['SYSROOT_DESTDIR', 'bindir'], 'ccache-native') + p = bb_vars['SYSROOT_DESTDIR'] + bb_vars['bindir'] + "/" + "ccache" + self.assertTrue(os.path.isfile(p), msg = "No ccache found (%s)" % p) self.write_config('INHERIT += "ccache"') - bitbake("m4 -c cleansstate") - bitbake("m4 -c compile") - res = runCmd("grep ccache %s" % (os.path.join(get_bb_var("WORKDIR","m4"),"temp/log.do_compile")), ignore_status=True) - self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile") - bitbake("ccache-native -ccleansstate") - + self.add_command_to_tearDown('bitbake -c clean m4') + bitbake("m4 -f -c compile") + log_compile = os.path.join(get_bb_var("WORKDIR","m4"), "temp/log.do_compile") + res = runCmd("grep ccache %s" % log_compile, ignore_status=True) + self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile) + + @testcase(1435) + def test_read_only_image(self): + distro_features = get_bb_var('DISTRO_FEATURES') + if not ('x11' in distro_features and 'opengl' in distro_features): + self.skipTest('core-image-sato requires x11 and opengl in distro features') + self.write_config('IMAGE_FEATURES += "read-only-rootfs"') + bitbake("core-image-sato") + # do_image will fail if there are any pending postinsts class DiskMonTest(oeSelfTest): + @testcase(277) def test_stoptask_behavior(self): self.write_config('BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},100000G,100K"') res = bitbake("m4", ignore_status = True) - self.assertTrue('ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"!' in res.output) - self.assertEqual(res.status, 1) + self.assertTrue('ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"!' in res.output, msg = "Tasks should have stopped. Disk monitor is set to STOPTASK: %s" % res.output) + self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output)) self.write_config('BB_DISKMON_DIRS = "ABORT,${TMPDIR},100000G,100K"') res = bitbake("m4", ignore_status = True) - self.assertTrue('ERROR: Immediately abort since the disk space monitor action is "ABORT"!' in res.output) - self.assertEqual(res.status, 1) + self.assertTrue('ERROR: Immediately abort since the disk space monitor action is "ABORT"!' in res.output, "Tasks should have been aborted immediatelly. Disk monitor is set to ABORT: %s" % res.output) + self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output)) self.write_config('BB_DISKMON_DIRS = "WARN,${TMPDIR},100000G,100K"') res = bitbake("m4") - self.assertTrue('WARNING: The free space' in res.output) + self.assertTrue('WARNING: The free space' in res.output, msg = "A warning should have been displayed for disk monitor is set to WARN: %s" %res.output) class SanityOptionsTest(oeSelfTest): + def getline(self, res, line): + for l in res.output.split('\n'): + if line in l: + return l + @testcase(927) def test_options_warnqa_errorqa_switch(self): - bitbake("xcursor-transparent-theme -ccleansstate") + self.write_config("INHERIT_remove = \"report-error\"") if "packages-list" not in get_bb_var("ERROR_QA"): - self.write_config("ERROR_QA_append = \" packages-list\"") + self.append_config("ERROR_QA_append = \" packages-list\"") self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') - res = bitbake("xcursor-transparent-theme", ignore_status=True) + self.add_command_to_tearDown('bitbake -c clean xcursor-transparent-theme') + res = bitbake("xcursor-transparent-theme -f -c package", ignore_status=True) self.delete_recipeinc('xcursor-transparent-theme') - self.assertTrue("ERROR: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output, msg=res.output) - self.assertEqual(res.status, 1) + line = self.getline(res, "QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors.") + self.assertTrue(line and line.startswith("ERROR:"), msg=res.output) + self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output)) self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') self.append_config('ERROR_QA_remove = "packages-list"') self.append_config('WARN_QA_append = " packages-list"') - bitbake("xcursor-transparent-theme -ccleansstate") - res = bitbake("xcursor-transparent-theme") + res = bitbake("xcursor-transparent-theme -f -c package") self.delete_recipeinc('xcursor-transparent-theme') - self.assertTrue("WARNING: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output, msg=res.output) + line = self.getline(res, "QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors.") + self.assertTrue(line and line.startswith("WARNING:"), msg=res.output) + + @testcase(278) + def test_sanity_unsafe_script_references(self): + self.write_config('WARN_QA_append = " unsafe-references-in-scripts"') + + self.add_command_to_tearDown('bitbake -c clean gzip') + res = bitbake("gzip -f -c package_qa") + line = self.getline(res, "QA Issue: gzip") + self.assertFalse(line, "WARNING: QA Issue: gzip message is present in bitbake's output and shouldn't be: %s" % res.output) + + self.append_config(""" +do_install_append_pn-gzip () { + echo "\n${bindir}/test" >> ${D}${bindir}/zcat +} +""") + res = bitbake("gzip -f -c package_qa") + line = self.getline(res, "QA Issue: gzip") + self.assertTrue(line and line.startswith("WARNING:"), "WARNING: QA Issue: gzip message is not present in bitbake's output: %s" % res.output) + + @testcase(1421) + def test_layer_without_git_dir(self): + """ + Summary: Test that layer git revisions are displayed and do not fail without git repository + Expected: The build to be successful and without "fatal" errors + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + dirpath = tempfile.mkdtemp() + + dummy_layer_name = 'meta-dummy' + dummy_layer_path = os.path.join(dirpath, dummy_layer_name) + dummy_layer_conf_dir = os.path.join(dummy_layer_path, 'conf') + os.makedirs(dummy_layer_conf_dir) + dummy_layer_conf_path = os.path.join(dummy_layer_conf_dir, 'layer.conf') + + dummy_layer_content = 'BBPATH .= ":${LAYERDIR}"\n' \ + 'BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"\n' \ + 'BBFILE_COLLECTIONS += "%s"\n' \ + 'BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' \ + 'BBFILE_PRIORITY_%s = "6"\n' % (dummy_layer_name, dummy_layer_name, dummy_layer_name) + + ftools.write_file(dummy_layer_conf_path, dummy_layer_content) + + bblayers_conf = 'BBLAYERS += "%s"\n' % dummy_layer_path + self.write_bblayers_config(bblayers_conf) + + test_recipe = 'ed' + + ret = bitbake('-n %s' % test_recipe) + + err = 'fatal: Not a git repository' + + shutil.rmtree(dirpath) + + self.assertNotIn(err, ret.output) - def test_sanity_userspace_dependency(self): - self.append_config('WARN_QA_append = " unsafe-references-in-binaries unsafe-references-in-scripts"') - bitbake("-ccleansstate gzip nfs-utils") - res = bitbake("gzip nfs-utils") - self.assertTrue("WARNING: QA Issue: gzip" in res.output) - self.assertTrue("WARNING: QA Issue: nfs-utils" in res.output) class BuildhistoryTests(BuildhistoryBase): + @testcase(293) def test_buildhistory_basic(self): self.run_buildhistory_operation('xcursor-transparent-theme') - self.assertTrue(os.path.isdir(get_bb_var('BUILDHISTORY_DIR'))) + self.assertTrue(os.path.isdir(get_bb_var('BUILDHISTORY_DIR')), "buildhistory dir was not created.") + @testcase(294) def test_buildhistory_buildtime_pr_backwards(self): - self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' - error = "ERROR: QA Issue: Package version for package %s went backwards which would break package feeds from (.*-r1 to .*-r0)" % target + error = "ERROR:.*QA Issue: Package version for package %s went backwards which would break package feeds from (.*-r1.* to .*-r0.*)" % target self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error) - - - - - +class ArchiverTest(oeSelfTest): + @testcase(926) + def test_arch_work_dir_and_export_source(self): + """ + Test for archiving the work directory and exporting the source files. + """ + self.write_config("INHERIT += \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") + res = bitbake("xcursor-transparent-theme", ignore_status=True) + self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output) + deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC') + pkgs_path = g.glob(str(deploy_dir_src) + "/allarch*/xcurs*") + src_file_glob = str(pkgs_path[0]) + "/xcursor*.src.rpm" + tar_file_glob = str(pkgs_path[0]) + "/xcursor*.tar.gz" + self.assertTrue((g.glob(src_file_glob) and g.glob(tar_file_glob)), "Couldn't find .src.rpm and .tar.gz files under %s/allarch*/xcursor*" % deploy_dir_src) diff --git a/meta/lib/oeqa/selftest/containerimage.py b/meta/lib/oeqa/selftest/containerimage.py new file mode 100644 index 0000000000..def481f144 --- /dev/null +++ b/meta/lib/oeqa/selftest/containerimage.py @@ -0,0 +1,83 @@ +import os + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake, get_bb_vars, runCmd + +# This test builds an image with using the "container" IMAGE_FSTYPE, and +# ensures that then files in the image are only the ones expected. +# +# The only package added to the image is container_image_testpkg, which +# contains one file. However, due to some other things not cleaning up during +# rootfs creation, there is some cruft. Ideally bugs will be filed and the +# cruft removed, but for now we whitelist some known set. +# +# Also for performance reasons we're only checking the cruft when using ipk. +# When using deb, and rpm it is a bit different and we could test all +# of them, but this test is more to catch if other packages get added by +# default other than what is in ROOTFS_BOOTSTRAP_INSTALL. +# +class ContainerImageTests(oeSelfTest): + + # Verify that when specifying a IMAGE_TYPEDEP_ of the form "foo.bar" that + # the conversion type bar gets added as a dep as well + def test_expected_files(self): + + def get_each_path_part(path): + if path: + part = [ '.' + path + '/' ] + result = get_each_path_part(path.rsplit('/', 1)[0]) + if result: + return part + result + else: + return part + else: + return None + + self.write_config("""PREFERRED_PROVIDER_virtual/kernel = "linux-dummy" +IMAGE_FSTYPES = "container" +PACKAGE_CLASSES = "package_ipk" +IMAGE_FEATURES = "" +""") + + bbvars = get_bb_vars(['bindir', 'sysconfdir', 'localstatedir', + 'DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], + target='container-test-image') + expected_files = [ + './', + '.{bindir}/theapp', + '.{sysconfdir}/default/', + '.{sysconfdir}/default/postinst', + '.{sysconfdir}/ld.so.cache', + '.{sysconfdir}/timestamp', + '.{sysconfdir}/version', + './run/', + '.{localstatedir}/cache/', + '.{localstatedir}/cache/ldconfig/', + '.{localstatedir}/cache/ldconfig/aux-cache', + '.{localstatedir}/cache/opkg/', + '.{localstatedir}/lib/', + '.{localstatedir}/lib/opkg/' + ] + + expected_files = [ x.format(bindir=bbvars['bindir'], + sysconfdir=bbvars['sysconfdir'], + localstatedir=bbvars['localstatedir']) + for x in expected_files ] + + # Since tar lists all directories individually, make sure each element + # from bindir, sysconfdir, etc is added + expected_files += get_each_path_part(bbvars['bindir']) + expected_files += get_each_path_part(bbvars['sysconfdir']) + expected_files += get_each_path_part(bbvars['localstatedir']) + + expected_files = sorted(expected_files) + + # Build the image of course + bitbake('container-test-image') + + image = os.path.join(bbvars['DEPLOY_DIR_IMAGE'], + bbvars['IMAGE_LINK_NAME'] + '.tar.bz2') + + # Ensure the files in the image are what we expect + result = runCmd("tar tf {} | sort".format(image), shell=True) + self.assertEqual(result.output.split('\n'), expected_files) diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py new file mode 100644 index 0000000000..8e7642c32b --- /dev/null +++ b/meta/lib/oeqa/selftest/devtool.py @@ -0,0 +1,1696 @@ +import unittest +import os +import logging +import re +import shutil +import tempfile +import glob +import fnmatch + +import oeqa.utils.ftools as ftools +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer +from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer +from oeqa.utils.decorators import testcase + +class DevtoolBase(oeSelfTest): + + def _test_recipe_contents(self, recipefile, checkvars, checkinherits): + with open(recipefile, 'r') as f: + invar = None + invalue = None + for line in f: + var = None + if invar: + value = line.strip().strip('"') + if value.endswith('\\'): + invalue += ' ' + value[:-1].strip() + continue + else: + invalue += ' ' + value.strip() + var = invar + value = invalue + invar = None + elif '=' in line: + splitline = line.split('=', 1) + var = splitline[0].rstrip() + value = splitline[1].strip().strip('"') + if value.endswith('\\'): + invalue = value[:-1].strip() + invar = var + continue + elif line.startswith('inherit '): + inherits = line.split()[1:] + + if var and var in checkvars: + needvalue = checkvars.pop(var) + if needvalue is None: + self.fail('Variable %s should not appear in recipe, but value is being set to "%s"' % (var, value)) + if isinstance(needvalue, set): + if var == 'LICENSE': + value = set(value.split(' & ')) + else: + value = set(value.split()) + self.assertEqual(value, needvalue, 'values for %s do not match' % var) + + + missingvars = {} + for var, value in checkvars.items(): + if value is not None: + missingvars[var] = value + self.assertEqual(missingvars, {}, 'Some expected variables not found in recipe: %s' % checkvars) + + for inherit in checkinherits: + self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit) + + def _check_bbappend(self, testrecipe, recipefile, appenddir): + result = runCmd('bitbake-layers show-appends', cwd=self.builddir) + resultlines = result.output.splitlines() + inrecipe = False + bbappends = [] + bbappendfile = None + for line in resultlines: + if inrecipe: + if line.startswith(' '): + bbappends.append(line.strip()) + else: + break + elif line == '%s:' % os.path.basename(recipefile): + inrecipe = True + self.assertLessEqual(len(bbappends), 2, '%s recipe is being bbappended by another layer - bbappends found:\n %s' % (testrecipe, '\n '.join(bbappends))) + for bbappend in bbappends: + if bbappend.startswith(appenddir): + bbappendfile = bbappend + break + else: + self.fail('bbappend for recipe %s does not seem to be created in test layer' % testrecipe) + return bbappendfile + + def _create_temp_layer(self, templayerdir, addlayer, templayername, priority=999, recipepathspec='recipes-*/*'): + create_temp_layer(templayerdir, templayername, priority, recipepathspec) + if addlayer: + self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir) + result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir) + + def _process_ls_output(self, output): + """ + Convert ls -l output to a format we can reasonably compare from one context + to another (e.g. from host to target) + """ + filelist = [] + for line in output.splitlines(): + splitline = line.split() + if len(splitline) < 8: + self.fail('_process_ls_output: invalid output line: %s' % line) + # Remove trailing . on perms + splitline[0] = splitline[0].rstrip('.') + # Remove leading . on paths + splitline[-1] = splitline[-1].lstrip('.') + # Drop fields we don't want to compare + del splitline[7] + del splitline[6] + del splitline[5] + del splitline[4] + del splitline[1] + filelist.append(' '.join(splitline)) + return filelist + + +class DevtoolTests(DevtoolBase): + + @classmethod + def setUpClass(cls): + bb_vars = get_bb_vars(['TOPDIR', 'SSTATE_DIR']) + cls.original_sstate = bb_vars['SSTATE_DIR'] + cls.devtool_sstate = os.path.join(bb_vars['TOPDIR'], 'sstate_devtool') + cls.sstate_conf = 'SSTATE_DIR = "%s"\n' % cls.devtool_sstate + cls.sstate_conf += ('SSTATE_MIRRORS += "file://.* file:///%s/PATH"\n' + % cls.original_sstate) + + @classmethod + def tearDownClass(cls): + cls.log.debug('Deleting devtool sstate cache on %s' % cls.devtool_sstate) + runCmd('rm -rf %s' % cls.devtool_sstate) + + def setUp(self): + """Test case setup function""" + super(DevtoolTests, self).setUp() + self.workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(self.workspacedir), + 'This test cannot be run with a workspace directory ' + 'under the build directory') + self.append_config(self.sstate_conf) + + def _check_src_repo(self, repo_dir): + """Check srctree git repository""" + self.assertTrue(os.path.isdir(os.path.join(repo_dir, '.git')), + 'git repository for external source tree not found') + result = runCmd('git status --porcelain', cwd=repo_dir) + self.assertEqual(result.output.strip(), "", + 'Created git repo is not clean') + result = runCmd('git symbolic-ref HEAD', cwd=repo_dir) + self.assertEqual(result.output.strip(), "refs/heads/devtool", + 'Wrong branch in git repo') + + def _check_repo_status(self, repo_dir, expected_status): + """Check the worktree status of a repository""" + result = runCmd('git status . --porcelain', + cwd=repo_dir) + for line in result.output.splitlines(): + for ind, (f_status, fn_re) in enumerate(expected_status): + if re.match(fn_re, line[3:]): + if f_status != line[:2]: + self.fail('Unexpected status in line: %s' % line) + expected_status.pop(ind) + break + else: + self.fail('Unexpected modified file in line: %s' % line) + if expected_status: + self.fail('Missing file changes: %s' % expected_status) + + @testcase(1158) + def test_create_workspace(self): + # Check preconditions + result = runCmd('bitbake-layers show-layers') + self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf') + # Try creating a workspace layer with a specific path + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + result = runCmd('devtool create-workspace %s' % tempdir) + self.assertTrue(os.path.isfile(os.path.join(tempdir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output) + result = runCmd('bitbake-layers show-layers') + self.assertIn(tempdir, result.output) + # Try creating a workspace layer with the default path + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool create-workspace') + self.assertTrue(os.path.isfile(os.path.join(self.workspacedir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output) + result = runCmd('bitbake-layers show-layers') + self.assertNotIn(tempdir, result.output) + self.assertIn(self.workspacedir, result.output) + + @testcase(1159) + def test_devtool_add(self): + # Fetch source + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + url = 'http://www.ivarch.com/programs/sources/pv-1.5.3.tar.bz2' + result = runCmd('wget %s' % url, cwd=tempdir) + result = runCmd('tar xfv pv-1.5.3.tar.bz2', cwd=tempdir) + srcdir = os.path.join(tempdir, 'pv-1.5.3') + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory') + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate pv') + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add pv %s' % srcdir) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn('pv', result.output) + self.assertIn(srcdir, result.output) + # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then) + bitbake('pv -c cleansstate') + # Test devtool build + result = runCmd('devtool build pv') + bb_vars = get_bb_vars(['D', 'bindir'], 'pv') + installdir = bb_vars['D'] + self.assertTrue(installdir, 'Could not query installdir variable') + bindir = bb_vars['bindir'] + self.assertTrue(bindir, 'Could not query bindir variable') + if bindir[0] == '/': + bindir = bindir[1:] + self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D') + + @testcase(1423) + def test_devtool_add_git_local(self): + # Fetch source from a remote URL, but do it outside of devtool + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + pn = 'dbus-wait' + srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' + # We choose an https:// git URL here to check rewriting the URL works + url = 'https://git.yoctoproject.org/git/dbus-wait' + # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf + # instead of the directory name + result = runCmd('git clone %s noname' % url, cwd=tempdir) + srcdir = os.path.join(tempdir, 'noname') + result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Don't specify a name since we should be able to auto-detect it + result = runCmd('devtool add %s' % srcdir) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + # Check the recipe name is correct + recipefile = get_bb_var('FILE', pn) + self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named') + self.assertIn(recipefile, result.output) + # Test devtool status + result = runCmd('devtool status') + self.assertIn(pn, result.output) + self.assertIn(srcdir, result.output) + self.assertIn(recipefile, result.output) + checkvars = {} + checkvars['LICENSE'] = 'GPLv2' + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' + checkvars['S'] = '${WORKDIR}/git' + checkvars['PV'] = '0.1+git${SRCPV}' + checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https' + checkvars['SRCREV'] = srcrev + checkvars['DEPENDS'] = set(['dbus']) + self._test_recipe_contents(recipefile, checkvars, []) + + @testcase(1162) + def test_devtool_add_library(self): + # Fetch source + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + version = '1.1' + url = 'https://www.intra2net.com/en/developer/libftdi/download/libftdi1-%s.tar.bz2' % version + result = runCmd('wget %s' % url, cwd=tempdir) + result = runCmd('tar xfv libftdi1-%s.tar.bz2' % version, cwd=tempdir) + srcdir = os.path.join(tempdir, 'libftdi1-%s' % version) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'CMakeLists.txt')), 'Unable to find CMakeLists.txt in source directory') + # Test devtool add (and use -V so we test that too) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add libftdi %s -V %s' % (srcdir, version)) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn('libftdi', result.output) + self.assertIn(srcdir, result.output) + # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then) + bitbake('libftdi -c cleansstate') + # libftdi's python/CMakeLists.txt is a bit broken, so let's just disable it + # There's also the matter of it installing cmake files to a path we don't + # normally cover, which triggers the installed-vs-shipped QA test we have + # within do_package + recipefile = '%s/recipes/libftdi/libftdi_%s.bb' % (self.workspacedir, version) + result = runCmd('recipetool setvar %s EXTRA_OECMAKE -- \'-DPYTHON_BINDINGS=OFF -DLIBFTDI_CMAKE_CONFIG_DIR=${datadir}/cmake/Modules\'' % recipefile) + with open(recipefile, 'a') as f: + f.write('\nFILES_${PN}-dev += "${datadir}/cmake/Modules"\n') + # We don't have the ability to pick up this dependency automatically yet... + f.write('\nDEPENDS += "libusb1"\n') + f.write('\nTESTLIBOUTPUT = "${STAGING_DIR}-components/${TUNE_PKGARCH}/${PN}/${libdir}"\n') + # Test devtool build + result = runCmd('devtool build libftdi') + bb_vars = get_bb_vars(['TESTLIBOUTPUT', 'STAMP'], 'libftdi') + staging_libdir = bb_vars['TESTLIBOUTPUT'] + self.assertTrue(staging_libdir, 'Could not query TESTLIBOUTPUT variable') + self.assertTrue(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), "libftdi binary not found in STAGING_LIBDIR. Output of devtool build libftdi %s" % result.output) + # Test devtool reset + stampprefix = bb_vars['STAMP'] + result = runCmd('devtool reset libftdi') + result = runCmd('devtool status') + self.assertNotIn('libftdi', result.output) + self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe libftdi') + matches = glob.glob(stampprefix + '*') + self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned') + self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning') + + @testcase(1160) + def test_devtool_add_fetch(self): + # Fetch source + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + testver = '0.23' + url = 'https://pypi.python.org/packages/source/M/MarkupSafe/MarkupSafe-%s.tar.gz' % testver + testrecipe = 'python-markupsafe' + srcdir = os.path.join(tempdir, testrecipe) + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') + self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(srcdir, result.output) + # Check recipe + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') + checkvars = {} + checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}' + checkvars['SRC_URI'] = url.replace(testver, '${PV}') + self._test_recipe_contents(recipefile, checkvars, []) + # Try with version specified + result = runCmd('devtool reset -n %s' % testrecipe) + shutil.rmtree(srcdir) + fakever = '1.9' + result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever)) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(srcdir, result.output) + # Check recipe + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('%s_%s.bb' % (testrecipe, fakever), recipefile, 'Recipe file incorrectly named') + checkvars = {} + checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver + checkvars['SRC_URI'] = url + self._test_recipe_contents(recipefile, checkvars, []) + + @testcase(1161) + def test_devtool_add_fetch_git(self): + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + url = 'gitsm://git.yoctoproject.org/mraa' + checkrev = 'ae127b19a50aa54255e4330ccfdd9a5d058e581d' + testrecipe = 'mraa' + srcdir = os.path.join(tempdir, testrecipe) + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add %s %s -a -f %s' % (testrecipe, srcdir, url)) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'imraa', 'imraa.c')), 'Unable to find imraa/imraa.c in source directory') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(srcdir, result.output) + # Check recipe + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') + checkvars = {} + checkvars['S'] = '${WORKDIR}/git' + checkvars['PV'] = '1.0+git${SRCPV}' + checkvars['SRC_URI'] = url + checkvars['SRCREV'] = '${AUTOREV}' + self._test_recipe_contents(recipefile, checkvars, []) + # Try with revision and version specified + result = runCmd('devtool reset -n %s' % testrecipe) + shutil.rmtree(srcdir) + url_rev = '%s;rev=%s' % (url, checkrev) + result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev)) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'imraa', 'imraa.c')), 'Unable to find imraa/imraa.c in source directory') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(srcdir, result.output) + # Check recipe + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') + checkvars = {} + checkvars['S'] = '${WORKDIR}/git' + checkvars['PV'] = '1.5+git${SRCPV}' + checkvars['SRC_URI'] = url + checkvars['SRCREV'] = checkrev + self._test_recipe_contents(recipefile, checkvars, []) + + @testcase(1391) + def test_devtool_add_fetch_simple(self): + # Fetch source from a remote URL, auto-detecting name + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + testver = '1.6.0' + url = 'http://www.ivarch.com/programs/sources/pv-%s.tar.bz2' % testver + testrecipe = 'pv' + srcdir = os.path.join(self.workspacedir, 'sources', testrecipe) + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add %s' % url) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory') + self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(srcdir, result.output) + # Check recipe + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') + checkvars = {} + checkvars['S'] = None + checkvars['SRC_URI'] = url.replace(testver, '${PV}') + self._test_recipe_contents(recipefile, checkvars, []) + + @testcase(1164) + def test_devtool_modify(self): + import oe.path + + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean mdadm') + result = runCmd('devtool modify mdadm -x %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'mdadm_*.bbappend')) + self.assertTrue(matches, 'bbappend not created %s' % result.output) + + # Test devtool status + result = runCmd('devtool status') + self.assertIn('mdadm', result.output) + self.assertIn(tempdir, result.output) + self._check_src_repo(tempdir) + + bitbake('mdadm -C unpack') + + def check_line(checkfile, expected, message, present=True): + # Check for $expected, on a line on its own, in checkfile. + with open(checkfile, 'r') as f: + if present: + self.assertIn(expected + '\n', f, message) + else: + self.assertNotIn(expected + '\n', f, message) + + modfile = os.path.join(tempdir, 'mdadm.8.in') + bb_vars = get_bb_vars(['PKGD', 'mandir'], 'mdadm') + pkgd = bb_vars['PKGD'] + self.assertTrue(pkgd, 'Could not query PKGD variable') + mandir = bb_vars['mandir'] + self.assertTrue(mandir, 'Could not query mandir variable') + manfile = oe.path.join(pkgd, mandir, 'man8', 'mdadm.8') + + check_line(modfile, 'Linux Software RAID', 'Could not find initial string') + check_line(modfile, 'antique pin sardine', 'Unexpectedly found replacement string', present=False) + + result = runCmd("sed -i 's!^Linux Software RAID$!antique pin sardine!' %s" % modfile) + check_line(modfile, 'antique pin sardine', 'mdadm.8.in file not modified (sed failed)') + + bitbake('mdadm -c package') + check_line(manfile, 'antique pin sardine', 'man file not modified. man searched file path: %s' % manfile) + + result = runCmd('git checkout -- %s' % modfile, cwd=tempdir) + check_line(modfile, 'Linux Software RAID', 'man .in file not restored (git failed)') + + bitbake('mdadm -c package') + check_line(manfile, 'Linux Software RAID', 'man file not updated. man searched file path: %s' % manfile) + + result = runCmd('devtool reset mdadm') + result = runCmd('devtool status') + self.assertNotIn('mdadm', result.output) + + def test_devtool_buildclean(self): + def assertFile(path, *paths): + f = os.path.join(path, *paths) + self.assertTrue(os.path.exists(f), "%r does not exist" % f) + def assertNoFile(path, *paths): + f = os.path.join(path, *paths) + self.assertFalse(os.path.exists(os.path.join(f)), "%r exists" % f) + + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('mdadm m4 -c cleansstate') + # Try modifying a recipe + tempdir_mdadm = tempfile.mkdtemp(prefix='devtoolqa') + tempdir_m4 = tempfile.mkdtemp(prefix='devtoolqa') + builddir_m4 = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir_mdadm) + self.track_for_cleanup(tempdir_m4) + self.track_for_cleanup(builddir_m4) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean mdadm m4') + self.write_recipeinc('m4', 'EXTERNALSRC_BUILD = "%s"\ndo_clean() {\n\t:\n}\n' % builddir_m4) + try: + runCmd('devtool modify mdadm -x %s' % tempdir_mdadm) + runCmd('devtool modify m4 -x %s' % tempdir_m4) + assertNoFile(tempdir_mdadm, 'mdadm') + assertNoFile(builddir_m4, 'src/m4') + result = bitbake('m4 -e') + result = bitbake('mdadm m4 -c compile') + self.assertEqual(result.status, 0) + assertFile(tempdir_mdadm, 'mdadm') + assertFile(builddir_m4, 'src/m4') + # Check that buildclean task exists and does call make clean + bitbake('mdadm m4 -c buildclean') + assertNoFile(tempdir_mdadm, 'mdadm') + assertNoFile(builddir_m4, 'src/m4') + bitbake('mdadm m4 -c compile') + assertFile(tempdir_mdadm, 'mdadm') + assertFile(builddir_m4, 'src/m4') + bitbake('mdadm m4 -c clean') + # Check that buildclean task is run before clean for B == S + assertNoFile(tempdir_mdadm, 'mdadm') + # Check that buildclean task is not run before clean for B != S + assertFile(builddir_m4, 'src/m4') + finally: + self.delete_recipeinc('m4') + + @testcase(1166) + def test_devtool_modify_invalid(self): + # Try modifying some recipes + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split() + # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose + result = runCmd('bitbake-layers show-recipes gcc-source*') + for line in result.output.splitlines(): + # just match those lines that contain a real target + m = re.match('(?P<recipe>^[a-zA-Z0-9.-]+)(?P<colon>:$)', line) + if m: + testrecipes.append(m.group('recipe')) + for testrecipe in testrecipes: + # Check it's a valid recipe + bitbake('%s -e' % testrecipe) + # devtool extract should fail + result = runCmd('devtool extract %s %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True) + self.assertNotEqual(result.status, 0, 'devtool extract on %s should have failed. devtool output: %s' % (testrecipe, result.output)) + self.assertNotIn('Fetching ', result.output, 'devtool extract on %s should have errored out before trying to fetch' % testrecipe) + self.assertIn('ERROR: ', result.output, 'devtool extract on %s should have given an ERROR' % testrecipe) + # devtool modify should fail + result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True) + self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed. devtool output: %s' % (testrecipe, result.output)) + self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe) + + @testcase(1365) + def test_devtool_modify_native(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying some recipes + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + bbclassextended = False + inheritnative = False + testrecipes = 'mtools-native apt-native desktop-file-utils-native'.split() + for testrecipe in testrecipes: + checkextend = 'native' in (get_bb_var('BBCLASSEXTEND', testrecipe) or '').split() + if not bbclassextended: + bbclassextended = checkextend + if not inheritnative: + inheritnative = not checkextend + result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe))) + self.assertNotIn('ERROR: ', result.output, 'ERROR in devtool modify output: %s' % result.output) + result = runCmd('devtool build %s' % testrecipe) + self.assertNotIn('ERROR: ', result.output, 'ERROR in devtool build output: %s' % result.output) + result = runCmd('devtool reset %s' % testrecipe) + self.assertNotIn('ERROR: ', result.output, 'ERROR in devtool reset output: %s' % result.output) + + self.assertTrue(bbclassextended, 'None of these recipes are BBCLASSEXTENDed to native - need to adjust testrecipes list: %s' % ', '.join(testrecipes)) + self.assertTrue(inheritnative, 'None of these recipes do "inherit native" - need to adjust testrecipes list: %s' % ', '.join(testrecipes)) + + + @testcase(1165) + def test_devtool_modify_git(self): + # Check preconditions + testrecipe = 'mkelfimage' + src_uri = get_bb_var('SRC_URI', testrecipe) + self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. devtool output: %s' % result.output) + matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'mkelfimage_*.bbappend')) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Check git repo + self._check_src_repo(tempdir) + # Try building + bitbake(testrecipe) + + @testcase(1167) + def test_devtool_modify_localfiles(self): + # Check preconditions + testrecipe = 'lighttpd' + src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split() + foundlocal = False + for item in src_uri: + if item.startswith('file://') and '.patch' not in item: + foundlocal = True + break + self.assertTrue(foundlocal, 'This test expects the %s recipe to fetch local files and it seems that it no longer does' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'configure.ac')), 'Extracted source could not be found') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Try building + bitbake(testrecipe) + + @testcase(1378) + def test_devtool_modify_virtual(self): + # Try modifying a virtual recipe + virtrecipe = 'virtual/make' + realrecipe = 'make' + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (virtrecipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % realrecipe)) + self.assertTrue(matches, 'bbappend not created %s' % result.output) + # Test devtool status + result = runCmd('devtool status') + self.assertNotIn(virtrecipe, result.output) + self.assertIn(realrecipe, result.output) + # Check git repo + self._check_src_repo(tempdir) + # This is probably sufficient + + + @testcase(1169) + def test_devtool_update_recipe(self): + # Check preconditions + testrecipe = 'minicom' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) + # First, modify a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + # We don't use -x here so that we test the behaviour of devtool modify without it + result = runCmd('devtool modify %s %s' % (testrecipe, tempdir)) + # Check git repo + self._check_src_repo(tempdir) + # Add a couple of commits + # FIXME: this only tests adding, need to also test update and remove + result = runCmd('echo "Additional line" >> README', cwd=tempdir) + result = runCmd('git commit -a -m "Change the README"', cwd=tempdir) + result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir) + result = runCmd('git add devtool-new-file', cwd=tempdir) + result = runCmd('git commit -m "Add a new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + ('??', '.*/0001-Change-the-README.patch$'), + ('??', '.*/0002-Add-a-new-file.patch$')] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + @testcase(1172) + def test_devtool_update_recipe_git(self): + # Check preconditions + testrecipe = 'mtd-utils' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) + patches = [] + for entry in src_uri.split(): + if entry.startswith('file://') and entry.endswith('.patch'): + patches.append(entry[7:].split(';')[0]) + self.assertGreater(len(patches), 0, 'The %s recipe does not appear to contain any patches, so this test will not be effective' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) + # First, modify a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Check git repo + self._check_src_repo(tempdir) + # Add a couple of commits + # FIXME: this only tests adding, need to also test update and remove + result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempdir) + result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempdir) + result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir) + result = runCmd('git add devtool-new-file', cwd=tempdir) + result = runCmd('git commit -m "Add a new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; rm -rf %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe -m srcrev %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile))] + \ + [(' D', '.*/%s$' % patch) for patch in patches] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) + addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git"'] + srcurilines = src_uri.split() + srcurilines[0] = 'SRC_URI = "' + srcurilines[0] + srcurilines.append('"') + removelines = ['SRCREV = ".*"'] + srcurilines + for line in result.output.splitlines(): + if line.startswith('+++') or line.startswith('---'): + continue + elif line.startswith('+'): + matched = False + for item in addlines: + if re.match(item, line[1:].strip()): + matched = True + break + self.assertTrue(matched, 'Unexpected diff add line: %s' % line) + elif line.startswith('-'): + matched = False + for item in removelines: + if re.match(item, line[1:].strip()): + matched = True + break + self.assertTrue(matched, 'Unexpected diff remove line: %s' % line) + # Now try with auto mode + runCmd('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + result = runCmd('git rev-parse --show-toplevel', cwd=os.path.dirname(recipefile)) + topleveldir = result.output.strip() + relpatchpath = os.path.join(os.path.relpath(os.path.dirname(recipefile), topleveldir), testrecipe) + expected_status = [(' M', os.path.relpath(recipefile, topleveldir)), + ('??', '%s/0001-Change-the-Makefile.patch' % relpatchpath), + ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + @testcase(1170) + def test_devtool_update_recipe_append(self): + # Check preconditions + testrecipe = 'mdadm' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) + # First, modify a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + tempsrcdir = os.path.join(tempdir, 'source') + templayerdir = os.path.join(tempdir, 'layer') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir)) + # Check git repo + self._check_src_repo(tempsrcdir) + # Add a commit + result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir) + result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir) + self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe)) + # Create a temporary layer and add it to bblayers.conf + self._create_temp_layer(templayerdir, True, 'selftestupdaterecipe') + # Create the bbappend + result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) + self.assertNotIn('WARNING:', result.output) + # Check recipe is still clean + self._check_repo_status(os.path.dirname(recipefile), []) + # Check bbappend was created + splitpath = os.path.dirname(recipefile).split(os.sep) + appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1]) + bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir) + patchfile = os.path.join(appenddir, testrecipe, '0001-Add-our-custom-version.patch') + self.assertTrue(os.path.exists(patchfile), 'Patch file not created') + + # Check bbappend contents + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://0001-Add-our-custom-version.patch"\n', + '\n'] + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, f.readlines()) + + # Check we can run it again and bbappend isn't modified + result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, f.readlines()) + # Drop new commit and check patch gets deleted + result = runCmd('git reset HEAD^', cwd=tempsrcdir) + result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) + self.assertFalse(os.path.exists(patchfile), 'Patch file not deleted') + expectedlines2 = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines2, f.readlines()) + # Put commit back and check we can run it if layer isn't in bblayers.conf + os.remove(bbappendfile) + result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir) + result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir) + result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) + self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output) + self.assertTrue(os.path.exists(patchfile), 'Patch file not created (with disabled layer)') + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, f.readlines()) + # Deleting isn't expected to work under these circumstances + + @testcase(1171) + def test_devtool_update_recipe_append_git(self): + # Check preconditions + testrecipe = 'mtd-utils' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) + for entry in src_uri.split(): + if entry.startswith('git://'): + git_uri = entry + break + self._check_repo_status(os.path.dirname(recipefile), []) + # First, modify a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + tempsrcdir = os.path.join(tempdir, 'source') + templayerdir = os.path.join(tempdir, 'layer') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir)) + # Check git repo + self._check_src_repo(tempsrcdir) + # Add a commit + result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempsrcdir) + result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) + self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe)) + # Create a temporary layer + os.makedirs(os.path.join(templayerdir, 'conf')) + with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f: + f.write('BBPATH .= ":${LAYERDIR}"\n') + f.write('BBFILES += "${LAYERDIR}/recipes-*/*/*.bbappend"\n') + f.write('BBFILE_COLLECTIONS += "oeselftesttemplayer"\n') + f.write('BBFILE_PATTERN_oeselftesttemplayer = "^${LAYERDIR}/"\n') + f.write('BBFILE_PRIORITY_oeselftesttemplayer = "999"\n') + f.write('BBFILE_PATTERN_IGNORE_EMPTY_oeselftesttemplayer = "1"\n') + self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir) + result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir) + # Create the bbappend + result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) + self.assertNotIn('WARNING:', result.output) + # Check recipe is still clean + self._check_repo_status(os.path.dirname(recipefile), []) + # Check bbappend was created + splitpath = os.path.dirname(recipefile).split(os.sep) + appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1]) + bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir) + self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created') + + # Check bbappend contents + result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) + expectedlines = set(['SRCREV = "%s"\n' % result.output, + '\n', + 'SRC_URI = "%s"\n' % git_uri, + '\n']) + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, set(f.readlines())) + + # Check we can run it again and bbappend isn't modified + result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, set(f.readlines())) + # Drop new commit and check SRCREV changes + result = runCmd('git reset HEAD^', cwd=tempsrcdir) + result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) + self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created') + result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) + expectedlines = set(['SRCREV = "%s"\n' % result.output, + '\n', + 'SRC_URI = "%s"\n' % git_uri, + '\n']) + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, set(f.readlines())) + # Put commit back and check we can run it if layer isn't in bblayers.conf + os.remove(bbappendfile) + result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) + result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir) + result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) + self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output) + self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created') + result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) + expectedlines = set(['SRCREV = "%s"\n' % result.output, + '\n', + 'SRC_URI = "%s"\n' % git_uri, + '\n']) + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, set(f.readlines())) + # Deleting isn't expected to work under these circumstances + + @testcase(1370) + def test_devtool_update_recipe_local_files(self): + """Check that local source files are copied over instead of patched""" + testrecipe = 'makedevs' + recipefile = get_bb_var('FILE', testrecipe) + # Setup srctree for modifying the recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be + # building it) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Check git repo + self._check_src_repo(tempdir) + # Try building just to ensure we haven't broken that + bitbake("%s" % testrecipe) + # Edit / commit local source + runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir) + runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) + runCmd('echo "Bar" > new-file', cwd=tempdir) + runCmd('git add new-file', cwd=tempdir) + runCmd('git commit -m "Add new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' % + os.path.dirname(recipefile)) + runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + (' M', '.*/makedevs/makedevs.c$'), + ('??', '.*/makedevs/new-local$'), + ('??', '.*/makedevs/0001-Add-new-file.patch$')] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + @testcase(1371) + def test_devtool_update_recipe_local_files_2(self): + """Check local source files support when oe-local-files is in Git""" + testrecipe = 'lzo' + recipefile = get_bb_var('FILE', testrecipe) + # Setup srctree for modifying the recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Check git repo + self._check_src_repo(tempdir) + # Add oe-local-files to Git + runCmd('rm oe-local-files/.gitignore', cwd=tempdir) + runCmd('git add oe-local-files', cwd=tempdir) + runCmd('git commit -m "Add local sources"', cwd=tempdir) + # Edit / commit local sources + runCmd('echo "# Foobar" >> oe-local-files/acinclude.m4', cwd=tempdir) + runCmd('git commit -am "Edit existing file"', cwd=tempdir) + runCmd('git rm oe-local-files/run-ptest', cwd=tempdir) + runCmd('git commit -m"Remove file"', cwd=tempdir) + runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) + runCmd('git add oe-local-files/new-local', cwd=tempdir) + runCmd('git commit -m "Add new local file"', cwd=tempdir) + runCmd('echo "Gar" > new-file', cwd=tempdir) + runCmd('git add new-file', cwd=tempdir) + runCmd('git commit -m "Add new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' % + os.path.dirname(recipefile)) + # Checkout unmodified file to working copy -> devtool should still pick + # the modified version from HEAD + runCmd('git checkout HEAD^ -- oe-local-files/acinclude.m4', cwd=tempdir) + runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + (' M', '.*/acinclude.m4$'), + (' D', '.*/run-ptest$'), + ('??', '.*/new-local$'), + ('??', '.*/0001-Add-new-file.patch$')] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + def test_devtool_update_recipe_local_files_3(self): + # First, modify the recipe + testrecipe = 'devtool-test-localonly' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + # Modify one file + runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe, 'oe-local-files')) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s/file2$' % testrecipe)] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + def test_devtool_update_recipe_local_patch_gz(self): + # First, modify the recipe + testrecipe = 'devtool-test-patch-gz' + if get_bb_var('DISTRO') == 'poky-tiny': + self.skipTest("The DISTRO 'poky-tiny' does not provide the dependencies needed by %s" % testrecipe) + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + # Modify one file + srctree = os.path.join(self.workspacedir, 'sources', testrecipe) + runCmd('echo "Another line" >> README', cwd=srctree) + runCmd('git commit -a --amend --no-edit', cwd=srctree) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s/readme.patch.gz$' % testrecipe)] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + patch_gz = os.path.join(os.path.dirname(recipefile), testrecipe, 'readme.patch.gz') + result = runCmd('file %s' % patch_gz) + if 'gzip compressed data' not in result.output: + self.fail('New patch file is not gzipped - file reports:\n%s' % result.output) + + def test_devtool_update_recipe_local_files_subdir(self): + # Try devtool extract on a recipe that has a file with subdir= set in + # SRC_URI such that it overwrites a file that was in an archive that + # was also in SRC_URI + # First, modify the recipe + testrecipe = 'devtool-test-subdir' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + testfile = os.path.join(self.workspacedir, 'sources', testrecipe, 'testfile') + self.assertTrue(os.path.exists(testfile), 'Extracted source could not be found') + with open(testfile, 'r') as f: + contents = f.read().rstrip() + self.assertEqual(contents, 'Modified version', 'File has apparently not been overwritten as it should have been') + # Test devtool update-recipe without modifying any files + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + @testcase(1163) + def test_devtool_extract(self): + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + # Try devtool extract + self.track_for_cleanup(tempdir) + self.append_config('PREFERRED_PROVIDER_virtual/make = "remake"') + result = runCmd('devtool extract remake %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + # devtool extract shouldn't create the workspace + self.assertFalse(os.path.exists(self.workspacedir)) + self._check_src_repo(tempdir) + + @testcase(1379) + def test_devtool_extract_virtual(self): + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + # Try devtool extract + self.track_for_cleanup(tempdir) + result = runCmd('devtool extract virtual/make %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + # devtool extract shouldn't create the workspace + self.assertFalse(os.path.exists(self.workspacedir)) + self._check_src_repo(tempdir) + + @testcase(1168) + def test_devtool_reset_all(self): + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + testrecipe1 = 'mdadm' + testrecipe2 = 'cronie' + result = runCmd('devtool modify -x %s %s' % (testrecipe1, os.path.join(tempdir, testrecipe1))) + result = runCmd('devtool modify -x %s %s' % (testrecipe2, os.path.join(tempdir, testrecipe2))) + result = runCmd('devtool build %s' % testrecipe1) + result = runCmd('devtool build %s' % testrecipe2) + stampprefix1 = get_bb_var('STAMP', testrecipe1) + self.assertTrue(stampprefix1, 'Unable to get STAMP value for recipe %s' % testrecipe1) + stampprefix2 = get_bb_var('STAMP', testrecipe2) + self.assertTrue(stampprefix2, 'Unable to get STAMP value for recipe %s' % testrecipe2) + result = runCmd('devtool reset -a') + self.assertIn(testrecipe1, result.output) + self.assertIn(testrecipe2, result.output) + result = runCmd('devtool status') + self.assertNotIn(testrecipe1, result.output) + self.assertNotIn(testrecipe2, result.output) + matches1 = glob.glob(stampprefix1 + '*') + self.assertFalse(matches1, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe1) + matches2 = glob.glob(stampprefix2 + '*') + self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2) + + @testcase(1272) + def test_devtool_deploy_target(self): + # NOTE: Whilst this test would seemingly be better placed as a runtime test, + # unfortunately the runtime tests run under bitbake and you can't run + # devtool within bitbake (since devtool needs to run bitbake itself). + # Additionally we are testing build-time functionality as well, so + # really this has to be done as an oe-selftest test. + # + # Check preconditions + machine = get_bb_var('MACHINE') + if not machine.startswith('qemu'): + self.skipTest('This test only works with qemu machines') + if not os.path.exists('/etc/runqemu-nosudo'): + self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True) + if result.status != 0: + result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True) + if result.status != 0: + self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output) + for line in result.output.splitlines(): + if line.startswith('tap'): + break + else: + self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Definitions + testrecipe = 'mdadm' + testfile = '/sbin/mdadm' + testimage = 'oe-selftest-image' + testcommand = '/sbin/mdadm --help' + # Build an image to run + bitbake("%s qemu-native qemu-helper-native" % testimage) + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + self.add_command_to_tearDown('bitbake -c clean %s' % testimage) + self.add_command_to_tearDown('rm -f %s/%s*' % (deploy_dir_image, testimage)) + # Clean recipe so the first deploy will fail + bitbake("%s -c clean" % testrecipe) + # Try devtool modify + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Test that deploy-target at this point fails (properly) + result = runCmd('devtool deploy-target -n %s root@localhost' % testrecipe, ignore_status=True) + self.assertNotEqual(result.output, 0, 'devtool deploy-target should have failed, output: %s' % result.output) + self.assertNotIn(result.output, 'Traceback', 'devtool deploy-target should have failed with a proper error not a traceback, output: %s' % result.output) + result = runCmd('devtool build %s' % testrecipe) + # First try a dry-run of deploy-target + result = runCmd('devtool deploy-target -n %s root@localhost' % testrecipe) + self.assertIn(' %s' % testfile, result.output) + # Boot the image + with runqemu(testimage) as qemu: + # Now really test deploy-target + result = runCmd('devtool deploy-target -c %s root@%s' % (testrecipe, qemu.ip)) + # Run a test command to see if it was installed properly + sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand)) + # Check if it deployed all of the files with the right ownership/perms + # First look on the host - need to do this under pseudo to get the correct ownership/perms + bb_vars = get_bb_vars(['D', 'FAKEROOTENV', 'FAKEROOTCMD'], testrecipe) + installdir = bb_vars['D'] + fakerootenv = bb_vars['FAKEROOTENV'] + fakerootcmd = bb_vars['FAKEROOTCMD'] + result = runCmd('%s %s find . -type f -exec ls -l {} \;' % (fakerootenv, fakerootcmd), cwd=installdir) + filelist1 = self._process_ls_output(result.output) + + # Now look on the target + tempdir2 = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir2) + tmpfilelist = os.path.join(tempdir2, 'files.txt') + with open(tmpfilelist, 'w') as f: + for line in filelist1: + splitline = line.split() + f.write(splitline[-1] + '\n') + result = runCmd('cat %s | ssh -q %s root@%s \'xargs ls -l\'' % (tmpfilelist, sshargs, qemu.ip)) + filelist2 = self._process_ls_output(result.output) + filelist1.sort(key=lambda item: item.split()[-1]) + filelist2.sort(key=lambda item: item.split()[-1]) + self.assertEqual(filelist1, filelist2) + # Test undeploy-target + result = runCmd('devtool undeploy-target -c %s root@%s' % (testrecipe, qemu.ip)) + result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand), ignore_status=True) + self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have') + + @testcase(1366) + def test_devtool_build_image(self): + """Test devtool build-image plugin""" + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + image = 'core-image-minimal' + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean %s' % image) + bitbake('%s -c clean' % image) + # Add target and native recipes to workspace + recipes = ['mdadm', 'parted-native'] + for recipe in recipes: + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake -c clean %s' % recipe) + runCmd('devtool modify %s -x %s' % (recipe, tempdir)) + # Try to build image + result = runCmd('devtool build-image %s' % image) + self.assertNotEqual(result, 0, 'devtool build-image failed') + # Check if image contains expected packages + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + image_link_name = get_bb_var('IMAGE_LINK_NAME', image) + reqpkgs = [item for item in recipes if not item.endswith('-native')] + with open(os.path.join(deploy_dir_image, image_link_name + '.manifest'), 'r') as f: + for line in f: + splitval = line.split() + if splitval: + pkg = splitval[0] + if pkg in reqpkgs: + reqpkgs.remove(pkg) + if reqpkgs: + self.fail('The following packages were not present in the image as expected: %s' % ', '.join(reqpkgs)) + + @testcase(1367) + def test_devtool_upgrade(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Check parameters + result = runCmd('devtool upgrade -h') + for param in 'recipename srctree --version -V --branch -b --keep-temp --no-patch'.split(): + self.assertIn(param, result.output) + # For the moment, we are using a real recipe. + recipe = 'devtool-upgrade-test1' + version = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that + # we are downgrading instead of upgrading. + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version)) + # Check if srctree at least is populated + self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, version)) + # Check new recipe subdirectory is present + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe, '%s-%s' % (recipe, version))), 'Recipe folder should exist') + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file should exist after upgrade') + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + # Check devtool reset recipe + result = runCmd('devtool reset %s -n' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after resetting') + + @testcase(1433) + def test_devtool_upgrade_git(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + recipe = 'devtool-upgrade-test2' + commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Check upgrade + result = runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit)) + # Check if srctree at least is populated + self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, commit)) + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, os.path.basename(oldrecipefile)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file should exist after upgrade') + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + # Check devtool reset recipe + result = runCmd('devtool reset %s -n' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after resetting') + + @testcase(1352) + def test_devtool_layer_plugins(self): + """Test that devtool can use plugins from other layers. + + This test executes the selftest-reverse command from meta-selftest.""" + + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + s = "Microsoft Made No Profit From Anyone's Zunes Yo" + result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s) + self.assertEqual(result.output, s[::-1]) + + def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): + dstdir = basedstdir + self.assertTrue(os.path.exists(dstdir)) + for p in paths: + dstdir = os.path.join(dstdir, p) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + self.track_for_cleanup(dstdir) + dstfile = os.path.join(dstdir, os.path.basename(srcfile)) + if srcfile != dstfile: + shutil.copy(srcfile, dstfile) + self.track_for_cleanup(dstfile) + + def test_devtool_load_plugin(self): + """Test that devtool loads only the first found plugin in BBPATH.""" + + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + devtool = runCmd("which devtool") + fromname = runCmd("devtool --quiet pluginfile") + srcfile = fromname.output + bbpath = get_bb_var('BBPATH') + searchpath = bbpath.split(':') + [os.path.dirname(devtool.output)] + plugincontent = [] + with open(srcfile) as fh: + plugincontent = fh.readlines() + try: + self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') + for path in searchpath: + self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool') + result = runCmd("devtool --quiet count") + self.assertEqual(result.output, '1') + result = runCmd("devtool --quiet multiloaded") + self.assertEqual(result.output, "no") + for path in searchpath: + result = runCmd("devtool --quiet bbdir") + self.assertEqual(result.output, path) + os.unlink(os.path.join(result.output, 'lib', 'devtool', 'bbpath.py')) + finally: + with open(srcfile, 'w') as fh: + fh.writelines(plugincontent) + + def _setup_test_devtool_finish_upgrade(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Use a "real" recipe from meta-selftest + recipe = 'devtool-upgrade-test1' + oldversion = '1.5.3' + newversion = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Do the upgrade + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, newversion)) + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "pv.h"/a \\/* Here is a new comment *\\/\' src/pv/number.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M src/pv/number.c', result.output) + result = runCmd('git commit src/pv/number.c -m "Add a comment to the code"', cwd=tempdir) + # Check if patch is there + recipedir = os.path.dirname(oldrecipefile) + olddir = os.path.join(recipedir, recipe + '-' + oldversion) + patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch' + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Original patch file does not exist') + return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn + + def test_devtool_finish_upgrade_origlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertFalse(os.path.exists(oldrecipefile), 'Old recipe file should have been deleted but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should have been deleted but wasn\'t') + newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion)) + newdir = os.path.join(recipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def test_devtool_finish_upgrade_otherlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to a different layer - should create a bbappend + # This cleanup isn't strictly necessary but do it anyway just in case it goes wrong and writes to here + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + oe_core_dir = os.path.join(get_bb_var('COREBASE'), 'meta') + newrecipedir = os.path.join(oe_core_dir, 'recipes-test', 'devtool') + newrecipefile = os.path.join(newrecipedir, '%s_%s.bb' % (recipe, newversion)) + self.track_for_cleanup(newrecipedir) + result = runCmd('devtool finish %s oe-core' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertTrue(os.path.exists(oldrecipefile), 'Old recipe file should not have been deleted') + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should not have been deleted') + newdir = os.path.join(newrecipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def _setup_test_devtool_finish_modify(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying a recipe + self.track_for_cleanup(self.workspacedir) + recipe = 'mdadm' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "mdadm.h"/a \\/* Here is a new comment *\\/\' maps.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M maps.c', result.output) + result = runCmd('git commit maps.c -m "Add a comment to the code"', cwd=tempdir) + for entry in os.listdir(recipedir): + filesdir = os.path.join(recipedir, entry) + if os.path.isdir(filesdir): + break + else: + self.fail('Unable to find recipe files directory for %s' % recipe) + return recipe, oldrecipefile, recipedir, filesdir + + def test_devtool_finish_modify_origlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)), + ('??', '.*/.*-Add-a-comment-to-the-code.patch$')] + self._check_repo_status(recipedir, expected_status) + + def test_devtool_finish_modify_otherlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + relpth = os.path.relpath(recipedir, os.path.join(get_bb_var('COREBASE'), 'meta')) + appenddir = os.path.join(get_test_layer(), relpth) + self.track_for_cleanup(appenddir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains the following unexpected changes after finish:\n%s' % (recipe, result.output.strip())) + recipefn = os.path.splitext(os.path.basename(oldrecipefile))[0] + recipefn = recipefn.split('_')[0] + '_%' + appendfile = os.path.join(appenddir, recipefn + '.bbappend') + self.assertTrue(os.path.exists(appendfile), 'bbappend %s should have been created but wasn\'t' % appendfile) + newdir = os.path.join(appenddir, recipe) + files = os.listdir(newdir) + foundpatch = None + for fn in files: + if fnmatch.fnmatch(fn, '*-Add-a-comment-to-the-code.patch'): + foundpatch = fn + if not foundpatch: + self.fail('No patch file created next to bbappend') + files.remove(foundpatch) + if files: + self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files)) + + def test_devtool_rename(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + # First run devtool add + # We already have this recipe in OE-Core, but that doesn't matter + recipename = 'i2c-tools' + recipever = '3.1.2' + recipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, recipever)) + url = 'http://downloads.yoctoproject.org/mirror/sources/i2c-tools-%s.tar.bz2' % recipever + def add_recipe(): + result = runCmd('devtool add %s' % url) + self.assertTrue(os.path.exists(recipefile), 'Expected recipe file not created') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'sources', recipename)), 'Source directory not created') + checkvars = {} + checkvars['S'] = None + checkvars['SRC_URI'] = url.replace(recipever, '${PV}') + self._test_recipe_contents(recipefile, checkvars, []) + add_recipe() + # Now rename it - change both name and version + newrecipename = 'mynewrecipe' + newrecipever = '456' + newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, newrecipever)) + result = runCmd('devtool rename %s %s -V %s' % (recipename, newrecipename, newrecipever)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file not renamed') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipename)), 'Old recipe directory still exists') + newsrctree = os.path.join(self.workspacedir, 'sources', newrecipename) + self.assertTrue(os.path.exists(newsrctree), 'Source directory not renamed') + checkvars = {} + checkvars['S'] = '${WORKDIR}/%s-%s' % (recipename, recipever) + checkvars['SRC_URI'] = url + self._test_recipe_contents(newrecipefile, checkvars, []) + # Try again - change just name this time + result = runCmd('devtool reset -n %s' % newrecipename) + shutil.rmtree(newsrctree) + add_recipe() + newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, recipever)) + result = runCmd('devtool rename %s %s' % (recipename, newrecipename)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file not renamed') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipename)), 'Old recipe directory still exists') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'sources', newrecipename)), 'Source directory not renamed') + checkvars = {} + checkvars['S'] = '${WORKDIR}/%s-${PV}' % recipename + checkvars['SRC_URI'] = url.replace(recipever, '${PV}') + self._test_recipe_contents(newrecipefile, checkvars, []) + # Try again - change just version this time + result = runCmd('devtool reset -n %s' % newrecipename) + shutil.rmtree(newsrctree) + add_recipe() + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, newrecipever)) + result = runCmd('devtool rename %s -V %s' % (recipename, newrecipever)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file not renamed') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'sources', recipename)), 'Source directory no longer exists') + checkvars = {} + checkvars['S'] = '${WORKDIR}/${BPN}-%s' % recipever + checkvars['SRC_URI'] = url + self._test_recipe_contents(newrecipefile, checkvars, []) + + @testcase(1577) + def test_devtool_virtual_kernel_modify(self): + """ + Summary: The purpose of this test case is to verify that + devtool modify works correctly when building + the kernel. + Dependencies: NA + Steps: 1. Build kernel with bitbake. + 2. Save the config file generated. + 3. Clean the environment. + 4. Use `devtool modify virtual/kernel` to validate following: + 4.1 The source is checked out correctly. + 4.2 The resulting configuration is the same as + what was get on step 2. + 4.3 The Kernel can be build correctly. + 4.4 Changes made on the source are reflected on the + subsequent builds. + 4.5 Changes on the configuration are reflected on the + subsequent builds + Expected: devtool modify is able to checkout the source of the kernel + and modification to the source and configurations are reflected + when building the kernel. + """ + #Set machine to qemxu86 to be able to modify the kernel and + #verify the modification. + features = 'MACHINE = "qemux86"\n' + self.write_config(features) + kernel_provider = get_bb_var('PREFERRED_PROVIDER_virtual/kernel') + # Clean up the enviroment + bitbake('%s -c clean' % kernel_provider) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean %s' % kernel_provider) + #Step 1 + #Here is just generated the config file instead of all the kernel to optimize the + #time of executing this test case. + bitbake('%s -c configure' % kernel_provider) + bbconfig = os.path.join(get_bb_var('B', kernel_provider),'.config') + buildir= get_bb_var('TOPDIR') + #Step 2 + runCmd('cp %s %s' % (bbconfig, buildir)) + self.assertTrue(os.path.exists(os.path.join(buildir, '.config')), + 'Could not copy .config file from kernel') + + tmpconfig = os.path.join(buildir, '.config') + #Step 3 + bitbake('%s -c clean' % kernel_provider) + #Step 4.1 + runCmd('devtool modify virtual/kernel -x %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), + 'Extracted source could not be found') + #Step 4.2 + configfile = os.path.join(tempdir,'.config') + diff = runCmd('diff %s %s' % (tmpconfig, configfile)) + self.assertEqual(0,diff.status,'Kernel .config file is not the same using bitbake and devtool') + #Step 4.3 + #NOTE: virtual/kernel is mapped to kernel_provider + result = runCmd('devtool build %s' % kernel_provider) + self.assertEqual(0,result.status,'Cannot build kernel using `devtool build`') + kernelfile = os.path.join(get_bb_var('KBUILD_OUTPUT', kernel_provider), 'vmlinux') + self.assertTrue(os.path.exists(kernelfile),'Kernel was not build correctly') + + #Modify the kernel source, this is specific for qemux86 + modfile = os.path.join(tempdir,'arch/x86/boot/header.S') + modstring = "use a boot loader - Devtool kernel testing" + modapplied = runCmd("sed -i 's/boot loader/%s/' %s" % (modstring, modfile)) + self.assertEqual(0,modapplied.status,'Modification to %s on kernel source failed' % modfile) + #Modify the configuration + codeconfigfile = os.path.join(tempdir,'.config.new') + modconfopt = "CONFIG_SG_POOL=n" + modconf = runCmd("sed -i 's/CONFIG_SG_POOL=y/%s/' %s" % (modconfopt, codeconfigfile)) + self.assertEqual(0,modconf.status,'Modification to %s failed' % codeconfigfile) + #Build again kernel with devtool + rebuild = runCmd('devtool build %s' % kernel_provider) + self.assertEqual(0,rebuild.status,'Fail to build kernel after modification of source and config') + #Step 4.4 + bzimagename = 'bzImage-' + get_bb_var('KERNEL_VERSION_NAME', kernel_provider) + bzimagefile = os.path.join(get_bb_var('D', kernel_provider),'boot', bzimagename) + checkmodcode = runCmd("grep '%s' %s" % (modstring, bzimagefile)) + self.assertEqual(0,checkmodcode.status,'Modification on kernel source failed') + #Step 4.5 + checkmodconfg = runCmd("grep %s %s" % (modconfopt, codeconfigfile)) + self.assertEqual(0,checkmodconfg.status,'Modification to configuration file failed') diff --git a/meta/lib/oeqa/selftest/eSDK.py b/meta/lib/oeqa/selftest/eSDK.py new file mode 100644 index 0000000000..1596c6e9d6 --- /dev/null +++ b/meta/lib/oeqa/selftest/eSDK.py @@ -0,0 +1,115 @@ +import unittest +import tempfile +import shutil +import os +import glob +import logging +import subprocess +import oeqa.utils.ftools as ftools +from oeqa.utils.decorators import testcase +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars + +class oeSDKExtSelfTest(oeSelfTest): + """ + # Bugzilla Test Plan: 6033 + # This code is planned to be part of the automation for eSDK containig + # Install libraries and headers, image generation binary feeds, sdk-update. + """ + + @staticmethod + def get_esdk_environment(env_eSDK, tmpdir_eSDKQA): + # XXX: at this time use the first env need to investigate + # what environment load oe-selftest, i586, x86_64 + pattern = os.path.join(tmpdir_eSDKQA, 'environment-setup-*') + return glob.glob(pattern)[0] + + @staticmethod + def run_esdk_cmd(env_eSDK, tmpdir_eSDKQA, cmd, postconfig=None, **options): + if postconfig: + esdk_conf_file = os.path.join(tmpdir_eSDKQA, 'conf', 'local.conf') + with open(esdk_conf_file, 'a+') as f: + f.write(postconfig) + if not options: + options = {} + if not 'shell' in options: + options['shell'] = True + + runCmd("cd %s; . %s; %s" % (tmpdir_eSDKQA, env_eSDK, cmd), **options) + + @staticmethod + def generate_eSDK(image): + pn_task = '%s -c populate_sdk_ext' % image + bitbake(pn_task) + + @staticmethod + def get_eSDK_toolchain(image): + pn_task = '%s -c populate_sdk_ext' % image + + bb_vars = get_bb_vars(['SDK_DEPLOY', 'TOOLCHAINEXT_OUTPUTNAME'], pn_task) + sdk_deploy = bb_vars['SDK_DEPLOY'] + toolchain_name = bb_vars['TOOLCHAINEXT_OUTPUTNAME'] + return os.path.join(sdk_deploy, toolchain_name + '.sh') + + @staticmethod + def update_configuration(cls, image, tmpdir_eSDKQA, env_eSDK, ext_sdk_path): + sstate_dir = os.path.join(os.environ['BUILDDIR'], 'sstate-cache') + + oeSDKExtSelfTest.generate_eSDK(cls.image) + + cls.ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(cls.image) + runCmd("%s -y -d \"%s\"" % (cls.ext_sdk_path, cls.tmpdir_eSDKQA)) + + cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA) + + sstate_config=""" +SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS" +SSTATE_MIRRORS = "file://.* file://%s/PATH" +CORE_IMAGE_EXTRA_INSTALL = "perl" + """ % sstate_dir + + with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f: + f.write(sstate_config) + + @classmethod + def setUpClass(cls): + cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA') + + sstate_dir = get_bb_var('SSTATE_DIR') + + cls.image = 'core-image-minimal' + oeSDKExtSelfTest.generate_eSDK(cls.image) + + # Install eSDK + cls.ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(cls.image) + runCmd("%s -y -d \"%s\"" % (cls.ext_sdk_path, cls.tmpdir_eSDKQA)) + + cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA) + + # Configure eSDK to use sstate mirror from poky + sstate_config=""" +SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS" +SSTATE_MIRRORS = "file://.* file://%s/PATH" + """ % sstate_dir + with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f: + f.write(sstate_config) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdir_eSDKQA) + + @testcase (1602) + def test_install_libraries_headers(self): + pn_sstate = 'bc' + bitbake(pn_sstate) + cmd = "devtool sdk-install %s " % pn_sstate + oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd) + + @testcase(1603) + def test_image_generation_binary_feeds(self): + image = 'core-image-minimal' + cmd = "devtool build-image %s" % image + oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/selftest/image_typedep.py b/meta/lib/oeqa/selftest/image_typedep.py new file mode 100644 index 0000000000..256142d255 --- /dev/null +++ b/meta/lib/oeqa/selftest/image_typedep.py @@ -0,0 +1,51 @@ +import os + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake + +class ImageTypeDepTests(oeSelfTest): + + # Verify that when specifying a IMAGE_TYPEDEP_ of the form "foo.bar" that + # the conversion type bar gets added as a dep as well + def test_conversion_typedep_added(self): + + self.write_recipeinc('emptytest', """ +# Try to empty out the default dependency list +PACKAGE_INSTALL = "" +DISTRO_EXTRA_RDEPENDS="" + +LICENSE = "MIT" +IMAGE_FSTYPES = "testfstype" + +IMAGE_TYPES_MASKED += "testfstype" +IMAGE_TYPEDEP_testfstype = "tar.bz2" + +inherit image + +""") + # First get the dependency that should exist for bz2, it will look + # like CONVERSION_DEPENDS_bz2="somedep" + result = bitbake('-e emptytest') + + for line in result.output.split('\n'): + if line.startswith('CONVERSION_DEPENDS_bz2'): + dep = line.split('=')[1].strip('"') + break + + # Now get the dependency task list and check for the expected task + # dependency + bitbake('-g emptytest') + + taskdependsfile = os.path.join(self.builddir, 'task-depends.dot') + dep = dep + ".do_populate_sysroot" + depfound = False + expectedline = '"emptytest.do_rootfs" -> "{}"'.format(dep) + + with open(taskdependsfile, "r") as f: + for line in f: + if line.strip() == expectedline: + depfound = True + break + + if not depfound: + raise AssertionError("\"{}\" not found".format(expectedline)) diff --git a/meta/lib/oeqa/selftest/imagefeatures.py b/meta/lib/oeqa/selftest/imagefeatures.py new file mode 100644 index 0000000000..76896c7981 --- /dev/null +++ b/meta/lib/oeqa/selftest/imagefeatures.py @@ -0,0 +1,127 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu +from oeqa.utils.decorators import testcase +from oeqa.utils.sshcontrol import SSHControl +import os +import sys +import logging + +class ImageFeatures(oeSelfTest): + + test_user = 'tester' + root_user = 'root' + + @testcase(1107) + def test_non_root_user_can_connect_via_ssh_without_password(self): + """ + Summary: Check if non root user can connect via ssh without password + Expected: 1. Connection to the image via ssh using root user without providing a password should be allowed. + 2. Connection to the image via ssh using tester user without providing a password should be allowed. + Product: oe-core + Author: Ionut Chisanovici <ionutx.chisanovici@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + features = 'EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password"\n' + features += 'INHERIT += "extrausers"\n' + features += 'EXTRA_USERS_PARAMS = "useradd -p \'\' {}; usermod -s /bin/sh {};"'.format(self.test_user, self.test_user) + self.write_config(features) + + # Build a core-image-minimal + bitbake('core-image-minimal') + + with runqemu("core-image-minimal") as qemu: + # Attempt to ssh with each user into qemu with empty password + for user in [self.root_user, self.test_user]: + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user=user) + status, output = ssh.run("true") + self.assertEqual(status, 0, 'ssh to user %s failed with %s' % (user, output)) + + @testcase(1115) + def test_all_users_can_connect_via_ssh_without_password(self): + """ + Summary: Check if all users can connect via ssh without password + Expected: 1. Connection to the image via ssh using root user without providing a password should NOT be allowed. + 2. Connection to the image via ssh using tester user without providing a password should be allowed. + Product: oe-core + Author: Ionut Chisanovici <ionutx.chisanovici@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + features = 'EXTRA_IMAGE_FEATURES = "ssh-server-openssh allow-empty-password"\n' + features += 'INHERIT += "extrausers"\n' + features += 'EXTRA_USERS_PARAMS = "useradd -p \'\' {}; usermod -s /bin/sh {};"'.format(self.test_user, self.test_user) + self.write_config(features) + + # Build a core-image-minimal + bitbake('core-image-minimal') + + with runqemu("core-image-minimal") as qemu: + # Attempt to ssh with each user into qemu with empty password + for user in [self.root_user, self.test_user]: + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user=user) + status, output = ssh.run("true") + if user == 'root': + self.assertNotEqual(status, 0, 'ssh to user root was allowed when it should not have been') + else: + self.assertEqual(status, 0, 'ssh to user tester failed with %s' % output) + + + @testcase(1116) + def test_clutter_image_can_be_built(self): + """ + Summary: Check if clutter image can be built + Expected: 1. core-image-clutter can be built + Product: oe-core + Author: Ionut Chisanovici <ionutx.chisanovici@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + # Build a core-image-clutter + bitbake('core-image-clutter') + + @testcase(1117) + def test_wayland_support_in_image(self): + """ + Summary: Check Wayland support in image + Expected: 1. Wayland image can be build + 2. Wayland feature can be installed + Product: oe-core + Author: Ionut Chisanovici <ionutx.chisanovici@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + distro_features = get_bb_var('DISTRO_FEATURES') + if not ('opengl' in distro_features and 'wayland' in distro_features): + self.skipTest('neither opengl nor wayland present on DISTRO_FEATURES so core-image-weston cannot be built') + + # Build a core-image-weston + bitbake('core-image-weston') + + def test_bmap(self): + """ + Summary: Check bmap support + Expected: 1. core-image-minimal can be build with bmap support + 2. core-image-minimal is sparse + Product: oe-core + Author: Ed Bartosh <ed.bartosh@linux.intel.com> + """ + + features = 'IMAGE_FSTYPES += " ext4 ext4.bmap"' + self.write_config(features) + + image_name = 'core-image-minimal' + bitbake(image_name) + + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + link_name = get_bb_var('IMAGE_LINK_NAME', image_name) + image_path = os.path.join(deploy_dir_image, "%s.ext4" % link_name) + bmap_path = "%s.bmap" % image_path + + # check if result image and bmap file are in deploy directory + self.assertTrue(os.path.exists(image_path)) + self.assertTrue(os.path.exists(bmap_path)) + + # check if result image is sparse + image_stat = os.stat(image_path) + self.assertTrue(image_stat.st_size > image_stat.st_blocks * 512) diff --git a/meta/lib/oeqa/selftest/layerappend.py b/meta/lib/oeqa/selftest/layerappend.py new file mode 100644 index 0000000000..37bb32cd1d --- /dev/null +++ b/meta/lib/oeqa/selftest/layerappend.py @@ -0,0 +1,100 @@ +import unittest +import os +import logging +import re + +from oeqa.selftest.base import oeSelfTest +from oeqa.selftest.buildhistory import BuildhistoryBase +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +import oeqa.utils.ftools as ftools +from oeqa.utils.decorators import testcase + +class LayerAppendTests(oeSelfTest): + layerconf = """ +# We have a conf and classes directory, append to BBPATH +BBPATH .= ":${LAYERDIR}" + +# We have a recipes directory, add to BBFILES +BBFILES += "${LAYERDIR}/recipes*/*.bb ${LAYERDIR}/recipes*/*.bbappend" + +BBFILE_COLLECTIONS += "meta-layerINT" +BBFILE_PATTERN_meta-layerINT := "^${LAYERDIR}/" +BBFILE_PRIORITY_meta-layerINT = "6" +""" + recipe = """ +LICENSE="CLOSED" +INHIBIT_DEFAULT_DEPS = "1" + +python do_build() { + bb.plain('Building ...') +} +addtask build +""" + append = """ +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" + +SRC_URI_append = " file://appendtest.txt" + +sysroot_stage_all_append() { + install -m 644 ${WORKDIR}/appendtest.txt ${SYSROOT_DESTDIR}/ +} + +""" + + append2 = """ +FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" + +SRC_URI_append += "file://appendtest.txt" +""" + layerappend = '' + + def tearDownLocal(self): + if self.layerappend: + ftools.remove_from_file(self.builddir + "/conf/bblayers.conf", self.layerappend) + + @testcase(1196) + def test_layer_appends(self): + corebase = get_bb_var("COREBASE") + + for l in ["0", "1", "2"]: + layer = os.path.join(corebase, "meta-layertest" + l) + self.assertFalse(os.path.exists(layer)) + os.mkdir(layer) + os.mkdir(layer + "/conf") + with open(layer + "/conf/layer.conf", "w") as f: + f.write(self.layerconf.replace("INT", l)) + os.mkdir(layer + "/recipes-test") + if l == "0": + with open(layer + "/recipes-test/layerappendtest.bb", "w") as f: + f.write(self.recipe) + elif l == "1": + with open(layer + "/recipes-test/layerappendtest.bbappend", "w") as f: + f.write(self.append) + os.mkdir(layer + "/recipes-test/layerappendtest") + with open(layer + "/recipes-test/layerappendtest/appendtest.txt", "w") as f: + f.write("Layer 1 test") + elif l == "2": + with open(layer + "/recipes-test/layerappendtest.bbappend", "w") as f: + f.write(self.append2) + os.mkdir(layer + "/recipes-test/layerappendtest") + with open(layer + "/recipes-test/layerappendtest/appendtest.txt", "w") as f: + f.write("Layer 2 test") + self.track_for_cleanup(layer) + + self.layerappend = "BBLAYERS += \"{0}/meta-layertest0 {0}/meta-layertest1 {0}/meta-layertest2\"".format(corebase) + ftools.append_file(self.builddir + "/conf/bblayers.conf", self.layerappend) + stagingdir = get_bb_var("SYSROOT_DESTDIR", "layerappendtest") + bitbake("layerappendtest") + data = ftools.read_file(stagingdir + "/appendtest.txt") + self.assertEqual(data, "Layer 2 test") + os.remove(corebase + "/meta-layertest2/recipes-test/layerappendtest/appendtest.txt") + bitbake("layerappendtest") + data = ftools.read_file(stagingdir + "/appendtest.txt") + self.assertEqual(data, "Layer 1 test") + with open(corebase + "/meta-layertest2/recipes-test/layerappendtest/appendtest.txt", "w") as f: + f.write("Layer 2 test") + bitbake("layerappendtest") + data = ftools.read_file(stagingdir + "/appendtest.txt") + self.assertEqual(data, "Layer 2 test") + + diff --git a/meta/lib/oeqa/selftest/liboe.py b/meta/lib/oeqa/selftest/liboe.py new file mode 100644 index 0000000000..0b0301def6 --- /dev/null +++ b/meta/lib/oeqa/selftest/liboe.py @@ -0,0 +1,99 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import get_bb_var, get_bb_vars, bitbake, runCmd +import oe.path +import glob +import os +import os.path + +class LibOE(oeSelfTest): + + @classmethod + def setUpClass(cls): + cls.tmp_dir = get_bb_var('TMPDIR') + + def test_copy_tree_special(self): + """ + Summary: oe.path.copytree() should copy files with special character + Expected: 'test file with sp£c!al @nd spaces' should exist in + copy destination + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + testloc = oe.path.join(self.tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfilename = 'test file with sp£c!al @nd spaces' + + # create the test file and copy it + open(oe.path.join(src, testfilename), 'w+b').close() + oe.path.copytree(src, dst) + + # ensure path exists in dest + fileindst = os.path.isfile(oe.path.join(dst, testfilename)) + self.assertTrue(fileindst, "File with spaces doesn't exist in dst") + + oe.path.remove(testloc) + + def test_copy_tree_xattr(self): + """ + Summary: oe.path.copytree() should preserve xattr on copied files + Expected: testxattr file in destination should have user.oetest + extended attribute + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + testloc = oe.path.join(self.tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfilename = 'testxattr' + + # ensure we have setfattr available + bitbake("attr-native") + + bb_vars = get_bb_vars(['SYSROOT_DESTDIR', 'bindir'], 'attr-native') + destdir = bb_vars['SYSROOT_DESTDIR'] + bindir = bb_vars['bindir'] + bindir = destdir + bindir + + # create a file with xattr and copy it + open(oe.path.join(src, testfilename), 'w+b').close() + runCmd('%s/setfattr -n user.oetest -v "testing liboe" %s' % (bindir, oe.path.join(src, testfilename))) + oe.path.copytree(src, dst) + + # ensure file in dest has user.oetest xattr + result = runCmd('%s/getfattr -n user.oetest %s' % (bindir, oe.path.join(dst, testfilename))) + self.assertIn('user.oetest="testing liboe"', result.output, 'Extended attribute not sert in dst') + + oe.path.remove(testloc) + + def test_copy_hardlink_tree_count(self): + """ + Summary: oe.path.copyhardlinktree() shouldn't miss out files + Expected: src and dst should have the same number of files + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + testloc = oe.path.join(self.tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfiles = ['foo', 'bar', '.baz', 'quux'] + + def touchfile(tf): + open(oe.path.join(src, tf), 'w+b').close() + + for f in testfiles: + touchfile(f) + + oe.path.copyhardlinktree(src, dst) + + dstcnt = len(os.listdir(dst)) + srccnt = len(os.listdir(src)) + self.assertEquals(dstcnt, len(testfiles), "Number of files in dst (%s) differs from number of files in src(%s)." % (dstcnt, srccnt)) + + oe.path.remove(testloc) diff --git a/meta/lib/oeqa/selftest/lic-checksum.py b/meta/lib/oeqa/selftest/lic-checksum.py new file mode 100644 index 0000000000..2e81373ae4 --- /dev/null +++ b/meta/lib/oeqa/selftest/lic-checksum.py @@ -0,0 +1,35 @@ +import os +import tempfile + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake +from oeqa.utils import CommandError +from oeqa.utils.decorators import testcase + +class LicenseTests(oeSelfTest): + + # Verify that changing a license file that has an absolute path causes + # the license qa to fail due to a mismatched md5sum. + @testcase(1197) + def test_nonmatching_checksum(self): + bitbake_cmd = '-c populate_lic emptytest' + error_msg = 'emptytest: The new md5 checksum is 8d777f385d3dfec8815d20f7496026dc' + + lic_file, lic_path = tempfile.mkstemp() + os.close(lic_file) + self.track_for_cleanup(lic_path) + + self.write_recipeinc('emptytest', """ +INHIBIT_DEFAULT_DEPS = "1" +LIC_FILES_CHKSUM = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" +SRC_URI = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" +""" % (lic_path, lic_path)) + result = bitbake(bitbake_cmd) + + with open(lic_path, "w") as f: + f.write("data") + + self.write_config("INHERIT_remove = \"report-error\"") + result = bitbake(bitbake_cmd, ignore_status=True) + if error_msg not in result.output: + raise AssertionError(result.output) diff --git a/meta/lib/oeqa/selftest/manifest.py b/meta/lib/oeqa/selftest/manifest.py new file mode 100644 index 0000000000..fe6f949644 --- /dev/null +++ b/meta/lib/oeqa/selftest/manifest.py @@ -0,0 +1,166 @@ +import unittest +import os + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import get_bb_var, get_bb_vars, bitbake +from oeqa.utils.decorators import testcase + +class ManifestEntry: + '''A manifest item of a collection able to list missing packages''' + def __init__(self, entry): + self.file = entry + self.missing = [] + +class VerifyManifest(oeSelfTest): + '''Tests for the manifest files and contents of an image''' + + @classmethod + def check_manifest_entries(self, manifest, path): + manifest_errors = [] + try: + with open(manifest, "r") as mfile: + for line in mfile: + manifest_entry = os.path.join(path, line.split()[0]) + self.log.debug("{}: looking for {}"\ + .format(self.classname, manifest_entry)) + if not os.path.isfile(manifest_entry): + manifest_errors.append(manifest_entry) + self.log.debug("{}: {} not found"\ + .format(self.classname, manifest_entry)) + except OSError as e: + self.log.debug("{}: checking of {} failed"\ + .format(self.classname, manifest)) + raise e + + return manifest_errors + + #this will possibly move from here + @classmethod + def get_dir_from_bb_var(self, bb_var, target = None): + target == self.buildtarget if target == None else target + directory = get_bb_var(bb_var, target); + if not directory or not os.path.isdir(directory): + self.log.debug("{}: {} points to {} when target = {}"\ + .format(self.classname, bb_var, directory, target)) + raise OSError + return directory + + @classmethod + def setUpClass(self): + + self.buildtarget = 'core-image-minimal' + self.classname = 'VerifyManifest' + + self.log.info("{}: doing bitbake {} as a prerequisite of the test"\ + .format(self.classname, self.buildtarget)) + if bitbake(self.buildtarget).status: + self.log.debug("{} Failed to setup {}"\ + .format(self.classname, self.buildtarget)) + unittest.SkipTest("{}: Cannot setup testing scenario"\ + .format(self.classname)) + + @testcase(1380) + def test_SDK_manifest_entries(self): + '''Verifying the SDK manifest entries exist, this may take a build''' + + # the setup should bitbake core-image-minimal and here it is required + # to do an additional setup for the sdk + sdktask = '-c populate_sdk' + bbargs = sdktask + ' ' + self.buildtarget + self.log.debug("{}: doing bitbake {} as a prerequisite of the test"\ + .format(self.classname, bbargs)) + if bitbake(bbargs).status: + self.log.debug("{} Failed to bitbake {}"\ + .format(self.classname, bbargs)) + unittest.SkipTest("{}: Cannot setup testing scenario"\ + .format(self.classname)) + + + pkgdata_dir = reverse_dir = {} + mfilename = mpath = m_entry = {} + # get manifest location based on target to query about + d_target= dict(target = self.buildtarget, + host = 'nativesdk-packagegroup-sdk-host') + try: + mdir = self.get_dir_from_bb_var('SDK_DEPLOY', self.buildtarget) + for k in d_target.keys(): + bb_vars = get_bb_vars(['SDK_NAME', 'SDK_VERSION'], self.buildtarget) + mfilename[k] = "{}-toolchain-{}.{}.manifest".format( + bb_vars['SDK_NAME'], + bb_vars['SDK_VERSION'], + k) + mpath[k] = os.path.join(mdir, mfilename[k]) + if not os.path.isfile(mpath[k]): + self.log.debug("{}: {} does not exist".format( + self.classname, mpath[k])) + raise IOError + m_entry[k] = ManifestEntry(mpath[k]) + + pkgdata_dir[k] = self.get_dir_from_bb_var('PKGDATA_DIR', + d_target[k]) + reverse_dir[k] = os.path.join(pkgdata_dir[k], + 'runtime-reverse') + if not os.path.exists(reverse_dir[k]): + self.log.debug("{}: {} does not exist".format( + self.classname, reverse_dir[k])) + raise IOError + except OSError: + raise unittest.SkipTest("{}: Error in obtaining manifest dirs"\ + .format(self.classname)) + except IOError: + msg = "{}: Error cannot find manifests in the specified dir:\n{}"\ + .format(self.classname, mdir) + self.fail(msg) + + for k in d_target.keys(): + self.log.debug("{}: Check manifest {}".format( + self.classname, m_entry[k].file)) + + m_entry[k].missing = self.check_manifest_entries(\ + m_entry[k].file,reverse_dir[k]) + if m_entry[k].missing: + msg = '{}: {} Error has the following missing entries'\ + .format(self.classname, m_entry[k].file) + logmsg = msg+':\n'+'\n'.join(m_entry[k].missing) + self.log.debug(logmsg) + self.log.info(msg) + self.fail(logmsg) + + @testcase(1381) + def test_image_manifest_entries(self): + '''Verifying the image manifest entries exist''' + + # get manifest location based on target to query about + try: + mdir = self.get_dir_from_bb_var('DEPLOY_DIR_IMAGE', + self.buildtarget) + mfilename = get_bb_var("IMAGE_LINK_NAME", self.buildtarget)\ + + ".manifest" + mpath = os.path.join(mdir, mfilename) + if not os.path.isfile(mpath): raise IOError + m_entry = ManifestEntry(mpath) + + pkgdata_dir = {} + pkgdata_dir = self.get_dir_from_bb_var('PKGDATA_DIR', + self.buildtarget) + revdir = os.path.join(pkgdata_dir, 'runtime-reverse') + if not os.path.exists(revdir): raise IOError + except OSError: + raise unittest.SkipTest("{}: Error in obtaining manifest dirs"\ + .format(self.classname)) + except IOError: + msg = "{}: Error cannot find manifests in dir:\n{}"\ + .format(self.classname, mdir) + self.fail(msg) + + self.log.debug("{}: Check manifest {}"\ + .format(self.classname, m_entry.file)) + m_entry.missing = self.check_manifest_entries(\ + m_entry.file, revdir) + if m_entry.missing: + msg = '{}: {} Error has the following missing entries'\ + .format(self.classname, m_entry.file) + logmsg = msg+':\n'+'\n'.join(m_entry.missing) + self.log.debug(logmsg) + self.log.info(msg) + self.fail(logmsg) diff --git a/meta/lib/oeqa/selftest/oelib/__init__.py b/meta/lib/oeqa/selftest/oelib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/selftest/oelib/__init__.py diff --git a/meta/lib/oeqa/selftest/oelib/buildhistory.py b/meta/lib/oeqa/selftest/oelib/buildhistory.py new file mode 100644 index 0000000000..5ed4b026fe --- /dev/null +++ b/meta/lib/oeqa/selftest/oelib/buildhistory.py @@ -0,0 +1,88 @@ +import os +import unittest +import tempfile +from git import Repo +from oeqa.utils.commands import get_bb_var +from oe.buildhistory_analysis import blob_to_dict, compare_dict_blobs + +class TestBlobParsing(unittest.TestCase): + + def setUp(self): + import time + self.repo_path = tempfile.mkdtemp(prefix='selftest-buildhistory', + dir=get_bb_var('TOPDIR')) + + self.repo = Repo.init(self.repo_path) + self.test_file = "test" + self.var_map = {} + + def tearDown(self): + import shutil + shutil.rmtree(self.repo_path) + + def commit_vars(self, to_add={}, to_remove = [], msg="A commit message"): + if len(to_add) == 0 and len(to_remove) == 0: + return + + for k in to_remove: + self.var_map.pop(x,None) + for k in to_add: + self.var_map[k] = to_add[k] + + with open(os.path.join(self.repo_path, self.test_file), 'w') as repo_file: + for k in self.var_map: + repo_file.write("%s = %s\n" % (k, self.var_map[k])) + + self.repo.git.add("--all") + self.repo.git.commit(message=msg) + + def test_blob_to_dict(self): + """ + Test convertion of git blobs to dictionary + """ + valuesmap = { "foo" : "1", "bar" : "2" } + self.commit_vars(to_add = valuesmap) + + blob = self.repo.head.commit.tree.blobs[0] + self.assertEqual(valuesmap, blob_to_dict(blob), + "commit was not translated correctly to dictionary") + + def test_compare_dict_blobs(self): + """ + Test comparisson of dictionaries extracted from git blobs + """ + changesmap = { "foo-2" : ("2", "8"), "bar" : ("","4"), "bar-2" : ("","5")} + + self.commit_vars(to_add = { "foo" : "1", "foo-2" : "2", "foo-3" : "3" }) + blob1 = self.repo.heads.master.commit.tree.blobs[0] + + self.commit_vars(to_add = { "foo-2" : "8", "bar" : "4", "bar-2" : "5" }) + blob2 = self.repo.heads.master.commit.tree.blobs[0] + + change_records = compare_dict_blobs(os.path.join(self.repo_path, self.test_file), + blob1, blob2, False, False) + + var_changes = { x.fieldname : (x.oldvalue, x.newvalue) for x in change_records} + self.assertEqual(changesmap, var_changes, "Changes not reported correctly") + + def test_compare_dict_blobs_default(self): + """ + Test default values for comparisson of git blob dictionaries + """ + defaultmap = { x : ("default", "1") for x in ["PKG", "PKGE", "PKGV", "PKGR"]} + + self.commit_vars(to_add = { "foo" : "1" }) + blob1 = self.repo.heads.master.commit.tree.blobs[0] + + self.commit_vars(to_add = { "PKG" : "1", "PKGE" : "1", "PKGV" : "1", "PKGR" : "1" }) + blob2 = self.repo.heads.master.commit.tree.blobs[0] + + change_records = compare_dict_blobs(os.path.join(self.repo_path, self.test_file), + blob1, blob2, False, False) + + var_changes = {} + for x in change_records: + oldvalue = "default" if ("default" in x.oldvalue) else x.oldvalue + var_changes[x.fieldname] = (oldvalue, x.newvalue) + + self.assertEqual(defaultmap, var_changes, "Defaults not set properly") diff --git a/meta/lib/oeqa/selftest/oelib/elf.py b/meta/lib/oeqa/selftest/oelib/elf.py new file mode 100644 index 0000000000..1f59037ed9 --- /dev/null +++ b/meta/lib/oeqa/selftest/oelib/elf.py @@ -0,0 +1,21 @@ +import unittest +import oe.qa + +class TestElf(unittest.TestCase): + def test_machine_name(self): + """ + Test elf_machine_to_string() + """ + self.assertEqual(oe.qa.elf_machine_to_string(0x02), "SPARC") + self.assertEqual(oe.qa.elf_machine_to_string(0x03), "x86") + self.assertEqual(oe.qa.elf_machine_to_string(0x08), "MIPS") + self.assertEqual(oe.qa.elf_machine_to_string(0x14), "PowerPC") + self.assertEqual(oe.qa.elf_machine_to_string(0x28), "ARM") + self.assertEqual(oe.qa.elf_machine_to_string(0x2A), "SuperH") + self.assertEqual(oe.qa.elf_machine_to_string(0x32), "IA-64") + self.assertEqual(oe.qa.elf_machine_to_string(0x3E), "x86-64") + self.assertEqual(oe.qa.elf_machine_to_string(0xB7), "AArch64") + + self.assertEqual(oe.qa.elf_machine_to_string(0x00), "Unknown (0)") + self.assertEqual(oe.qa.elf_machine_to_string(0xDEADBEEF), "Unknown (3735928559)") + self.assertEqual(oe.qa.elf_machine_to_string("foobar"), "Unknown ('foobar')") diff --git a/meta/lib/oe/tests/test_license.py b/meta/lib/oeqa/selftest/oelib/license.py index c388886184..c388886184 100644 --- a/meta/lib/oe/tests/test_license.py +++ b/meta/lib/oeqa/selftest/oelib/license.py diff --git a/meta/lib/oe/tests/test_path.py b/meta/lib/oeqa/selftest/oelib/path.py index 3d41ce157a..44d068143e 100644 --- a/meta/lib/oe/tests/test_path.py +++ b/meta/lib/oeqa/selftest/oelib/path.py @@ -55,7 +55,7 @@ class TestRealPath(unittest.TestCase): for d in self.DIRS: os.mkdir(os.path.join(self.root, d)) for f in self.FILES: - file(os.path.join(self.root, f), "w") + open(os.path.join(self.root, f), "w") for l in self.LINKS: os.symlink(l[1], os.path.join(self.root, l[0])) @@ -85,5 +85,5 @@ class TestRealPath(unittest.TestCase): def test_loop(self): for e in self.EXCEPTIONS: - self.assertRaisesRegexp(OSError, r'\[Errno %u\]' % e[1], + self.assertRaisesRegex(OSError, r'\[Errno %u\]' % e[1], self.__realpath, e[0], False, False) diff --git a/meta/lib/oe/tests/test_types.py b/meta/lib/oeqa/selftest/oelib/types.py index 367cc30e45..4fe2746a3b 100644 --- a/meta/lib/oe/tests/test_types.py +++ b/meta/lib/oeqa/selftest/oelib/types.py @@ -1,19 +1,7 @@ import unittest -from oe.maketype import create, factory +from oe.maketype import create -class TestTypes(unittest.TestCase): - def assertIsInstance(self, obj, cls): - return self.assertTrue(isinstance(obj, cls)) - - def assertIsNot(self, obj, other): - return self.assertFalse(obj is other) - - def assertFactoryCreated(self, value, type, **flags): - cls = factory(type) - self.assertIsNot(cls, None) - self.assertIsInstance(create(value, type, **flags), cls) - -class TestBooleanType(TestTypes): +class TestBooleanType(unittest.TestCase): def test_invalid(self): self.assertRaises(ValueError, create, '', 'boolean') self.assertRaises(ValueError, create, 'foo', 'boolean') @@ -43,7 +31,7 @@ class TestBooleanType(TestTypes): self.assertEqual(create('y', 'boolean'), True) self.assertNotEqual(create('y', 'boolean'), False) -class TestList(TestTypes): +class TestList(unittest.TestCase): def assertListEqual(self, value, valid, sep=None): obj = create(value, 'list', separator=sep) self.assertEqual(obj, valid) diff --git a/meta/lib/oe/tests/test_utils.py b/meta/lib/oeqa/selftest/oelib/utils.py index 5d9ac52e7d..7deb10f3c8 100644 --- a/meta/lib/oe/tests/test_utils.py +++ b/meta/lib/oeqa/selftest/oelib/utils.py @@ -1,5 +1,5 @@ import unittest -from oe.utils import packages_filter_out_system +from oe.utils import packages_filter_out_system, trim_version class TestPackagesFilterOutSystem(unittest.TestCase): def test_filter(self): diff --git a/meta/lib/oeqa/selftest/oescripts.py b/meta/lib/oeqa/selftest/oescripts.py index 31cd50809c..29547f56a9 100644 --- a/meta/lib/oeqa/selftest/oescripts.py +++ b/meta/lib/oeqa/selftest/oescripts.py @@ -10,42 +10,10 @@ from oeqa.selftest.buildhistory import BuildhistoryBase from oeqa.utils.commands import Command, runCmd, bitbake, get_bb_var, get_test_layer from oeqa.utils.decorators import testcase -class TestScripts(oeSelfTest): - - @testcase(300) - def test_cleanup_workdir(self): - path = os.path.dirname(get_bb_var('WORKDIR', 'gzip')) - old_version_recipe = os.path.join(get_bb_var('COREBASE'), 'meta/recipes-extended/gzip/gzip_1.3.12.bb') - old_version = '1.3.12' - bitbake("-ccleansstate gzip") - bitbake("-ccleansstate -b %s" % old_version_recipe) - if os.path.exists(get_bb_var('WORKDIR', "-b %s" % old_version_recipe)): - shutil.rmtree(get_bb_var('WORKDIR', "-b %s" % old_version_recipe)) - if os.path.exists(get_bb_var('WORKDIR', 'gzip')): - shutil.rmtree(get_bb_var('WORKDIR', 'gzip')) - - if os.path.exists(path): - initial_contents = os.listdir(path) - else: - initial_contents = [] - - bitbake('gzip') - intermediary_contents = os.listdir(path) - bitbake("-b %s" % old_version_recipe) - runCmd('cleanup-workdir') - remaining_contents = os.listdir(path) - - expected_contents = [x for x in intermediary_contents if x not in initial_contents] - remaining_not_expected = [x for x in remaining_contents if x not in expected_contents] - self.assertFalse(remaining_not_expected, msg="Not all necessary content has been deleted from %s: %s" % (path, ', '.join(map(str, remaining_not_expected)))) - expected_not_remaining = [x for x in expected_contents if x not in remaining_contents] - self.assertFalse(expected_not_remaining, msg="The script removed extra contents from %s: %s" % (path, ', '.join(map(str, expected_not_remaining)))) - class BuildhistoryDiffTests(BuildhistoryBase): @testcase(295) def test_buildhistory_diff(self): - self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True) diff --git a/meta/lib/oeqa/selftest/pkgdata.py b/meta/lib/oeqa/selftest/pkgdata.py new file mode 100644 index 0000000000..d69c3c800a --- /dev/null +++ b/meta/lib/oeqa/selftest/pkgdata.py @@ -0,0 +1,227 @@ +import unittest +import os +import tempfile +import logging +import fnmatch + +import oeqa.utils.ftools as ftools +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars +from oeqa.utils.decorators import testcase + +class OePkgdataUtilTests(oeSelfTest): + + @classmethod + def setUpClass(cls): + # Ensure we have the right data in pkgdata + logger = logging.getLogger("selftest") + logger.info('Running bitbake to generate pkgdata') + bitbake('busybox zlib m4') + + @testcase(1203) + def test_lookup_pkg(self): + # Forward tests + result = runCmd('oe-pkgdata-util lookup-pkg "zlib busybox"') + self.assertEqual(result.output, 'libz1\nbusybox') + result = runCmd('oe-pkgdata-util lookup-pkg zlib-dev') + self.assertEqual(result.output, 'libz-dev') + result = runCmd('oe-pkgdata-util lookup-pkg nonexistentpkg', ignore_status=True) + self.assertEqual(result.status, 1, "Status different than 1. output: %s" % result.output) + self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') + # Reverse tests + result = runCmd('oe-pkgdata-util lookup-pkg -r "libz1 busybox"') + self.assertEqual(result.output, 'zlib\nbusybox') + result = runCmd('oe-pkgdata-util lookup-pkg -r libz-dev') + self.assertEqual(result.output, 'zlib-dev') + result = runCmd('oe-pkgdata-util lookup-pkg -r nonexistentpkg', ignore_status=True) + self.assertEqual(result.status, 1, "Status different than 1. output: %s" % result.output) + self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') + + @testcase(1205) + def test_read_value(self): + result = runCmd('oe-pkgdata-util read-value PN libz1') + self.assertEqual(result.output, 'zlib') + result = runCmd('oe-pkgdata-util read-value PKG libz1') + self.assertEqual(result.output, 'libz1') + result = runCmd('oe-pkgdata-util read-value PKGSIZE m4') + pkgsize = int(result.output.strip()) + self.assertGreater(pkgsize, 1, "Size should be greater than 1. %s" % result.output) + + @testcase(1198) + def test_find_path(self): + result = runCmd('oe-pkgdata-util find-path /lib/libz.so.1') + self.assertEqual(result.output, 'zlib: /lib/libz.so.1') + result = runCmd('oe-pkgdata-util find-path /usr/bin/m4') + self.assertEqual(result.output, 'm4: /usr/bin/m4') + result = runCmd('oe-pkgdata-util find-path /not/exist', ignore_status=True) + self.assertEqual(result.status, 1, "Status different than 1. output: %s" % result.output) + self.assertEqual(result.output, 'ERROR: Unable to find any package producing path /not/exist') + + @testcase(1204) + def test_lookup_recipe(self): + result = runCmd('oe-pkgdata-util lookup-recipe "libz-staticdev busybox"') + self.assertEqual(result.output, 'zlib\nbusybox') + result = runCmd('oe-pkgdata-util lookup-recipe libz-dbg') + self.assertEqual(result.output, 'zlib') + result = runCmd('oe-pkgdata-util lookup-recipe nonexistentpkg', ignore_status=True) + self.assertEqual(result.status, 1, "Status different than 1. output: %s" % result.output) + self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') + + @testcase(1202) + def test_list_pkgs(self): + # No arguments + result = runCmd('oe-pkgdata-util list-pkgs') + pkglist = result.output.split() + self.assertIn('zlib', pkglist, "Listed packages: %s" % result.output) + self.assertIn('zlib-dev', pkglist, "Listed packages: %s" % result.output) + # No pkgspec, runtime + result = runCmd('oe-pkgdata-util list-pkgs -r') + pkglist = result.output.split() + self.assertIn('libz-dev', pkglist, "Listed packages: %s" % result.output) + # With recipe specified + result = runCmd('oe-pkgdata-util list-pkgs -p zlib') + pkglist = sorted(result.output.split()) + try: + pkglist.remove('zlib-ptest') # in case ptest is disabled + except ValueError: + pass + self.assertEqual(pkglist, ['zlib', 'zlib-dbg', 'zlib-dev', 'zlib-doc', 'zlib-staticdev'], "Packages listed after remove: %s" % result.output) + # With recipe specified, runtime + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r') + pkglist = sorted(result.output.split()) + try: + pkglist.remove('libz-ptest') # in case ptest is disabled + except ValueError: + pass + self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc', 'libz-staticdev', 'libz1'], "Packages listed after remove: %s" % result.output) + # With recipe specified and unpackaged + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u') + pkglist = sorted(result.output.split()) + self.assertIn('zlib-locale', pkglist, "Listed packages: %s" % result.output) + # With recipe specified and unpackaged, runtime + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u -r') + pkglist = sorted(result.output.split()) + self.assertIn('libz-locale', pkglist, "Listed packages: %s" % result.output) + # With recipe specified and pkgspec + result = runCmd('oe-pkgdata-util list-pkgs -p zlib "*-d*"') + pkglist = sorted(result.output.split()) + self.assertEqual(pkglist, ['zlib-dbg', 'zlib-dev', 'zlib-doc'], "Packages listed: %s" % result.output) + # With recipe specified and pkgspec, runtime + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r "*-d*"') + pkglist = sorted(result.output.split()) + self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc'], "Packages listed: %s" % result.output) + + @testcase(1201) + def test_list_pkg_files(self): + def splitoutput(output): + files = {} + curpkg = None + for line in output.splitlines(): + if line.startswith('\t'): + self.assertTrue(curpkg, 'Unexpected non-package line:\n%s' % line) + files[curpkg].append(line.strip()) + else: + self.assertTrue(line.rstrip().endswith(':'), 'Invalid package line in output:\n%s' % line) + curpkg = line.split(':')[0] + files[curpkg] = [] + return files + bb_vars = get_bb_vars(['base_libdir', 'libdir', 'includedir', 'mandir']) + base_libdir = bb_vars['base_libdir'] + libdir = bb_vars['libdir'] + includedir = bb_vars['includedir'] + mandir = bb_vars['mandir'] + # Test recipe-space package name + result = runCmd('oe-pkgdata-util list-pkg-files zlib-dev zlib-doc') + files = splitoutput(result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) + self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) + # Test runtime package name + result = runCmd('oe-pkgdata-util list-pkg-files -r libz1 libz-dev') + files = splitoutput(result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertGreater(len(files['libz1']), 1) + libspec = os.path.join(base_libdir, 'libz.so.1.*') + found = False + for fileitem in files['libz1']: + if fnmatch.fnmatchcase(fileitem, libspec): + found = True + break + self.assertTrue(found, 'Could not find zlib library file %s in libz1 package file list: %s' % (libspec, files['libz1'])) + self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) + # Test recipe + result = runCmd('oe-pkgdata-util list-pkg-files -p zlib') + files = splitoutput(result.output) + self.assertIn('zlib-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertNotIn('zlib-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) + # (ignore ptest, might not be there depending on config) + self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) + self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) + self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev']) + # Test recipe, runtime + result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r') + files = splitoutput(result.output) + self.assertIn('libz-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertNotIn('libz-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) + self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc']) + self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev']) + # Test recipe, unpackaged + result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -u') + files = splitoutput(result.output) + self.assertIn('zlib-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # this is the key one + self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) + self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) + self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev']) + # Test recipe, runtime, unpackaged + result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r -u') + files = splitoutput(result.output) + self.assertIn('libz-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # this is the key one + self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) + self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc']) + self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev']) + + @testcase(1200) + def test_glob(self): + tempdir = tempfile.mkdtemp(prefix='pkgdataqa') + self.track_for_cleanup(tempdir) + pkglistfile = os.path.join(tempdir, 'pkglist') + with open(pkglistfile, 'w') as f: + f.write('libz1\n') + f.write('busybox\n') + result = runCmd('oe-pkgdata-util glob %s "*-dev"' % pkglistfile) + desiredresult = ['libz-dev', 'busybox-dev'] + self.assertEqual(sorted(result.output.split()), sorted(desiredresult)) + # The following should not error (because when we use this during rootfs construction, sometimes the complementary package won't exist) + result = runCmd('oe-pkgdata-util glob %s "*-nonexistent"' % pkglistfile) + self.assertEqual(result.output, '') + # Test exclude option + result = runCmd('oe-pkgdata-util glob %s "*-dev *-dbg" -x "^libz"' % pkglistfile) + resultlist = result.output.split() + self.assertNotIn('libz-dev', resultlist) + self.assertNotIn('libz-dbg', resultlist) + + @testcase(1206) + def test_specify_pkgdatadir(self): + result = runCmd('oe-pkgdata-util -p %s lookup-pkg zlib' % get_bb_var('PKGDATA_DIR')) + self.assertEqual(result.output, 'libz1') diff --git a/meta/lib/oeqa/selftest/prservice.py b/meta/lib/oeqa/selftest/prservice.py index fb6d68d3bf..34d419762c 100644 --- a/meta/lib/oeqa/selftest/prservice.py +++ b/meta/lib/oeqa/selftest/prservice.py @@ -9,15 +9,19 @@ import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import runCmd, bitbake, get_bb_var from oeqa.utils.decorators import testcase +from oeqa.utils.network import get_free_port class BitbakePrTests(oeSelfTest): + @classmethod + def setUpClass(cls): + cls.pkgdata_dir = get_bb_var('PKGDATA_DIR') + def get_pr_version(self, package_name): - pkgdata_dir = get_bb_var('PKGDATA_DIR') - package_data_file = os.path.join(pkgdata_dir, 'runtime', package_name) + package_data_file = os.path.join(self.pkgdata_dir, 'runtime', package_name) package_data = ftools.read_file(package_data_file) find_pr = re.search("PKGR: r[0-9]+\.([0-9]+)", package_data) - self.assertTrue(find_pr) + self.assertTrue(find_pr, "No PKG revision found in %s" % package_data_file) return int(find_pr.group(1)) def get_task_stamp(self, package_name, recipe_task): @@ -26,7 +30,7 @@ class BitbakePrTests(oeSelfTest): package_stamps_path = "/".join(stampdata[:-1]) stamps = [] for stamp in os.listdir(package_stamps_path): - find_stamp = re.match("%s\.%s\.([a-z0-9]{32})" % (prefix, recipe_task), stamp) + find_stamp = re.match("%s\.%s\.([a-z0-9]{32})" % (re.escape(prefix), recipe_task), stamp) if find_stamp: stamps.append(find_stamp.group(1)) self.assertFalse(len(stamps) == 0, msg="Cound not find stamp for task %s for recipe %s" % (recipe_task, package_name)) @@ -34,9 +38,8 @@ class BitbakePrTests(oeSelfTest): return str(stamps[0]) def increment_package_pr(self, package_name): - inc_data = "do_package_append() {\nbb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\"\n}" % datetime.datetime.now() + inc_data = "do_package_append() {\n bb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\"\n}" % datetime.datetime.now() self.write_recipeinc(package_name, inc_data) - bitbake("-ccleansstate %s" % package_name) res = bitbake(package_name, ignore_status=True) self.delete_recipeinc(package_name) self.assertEqual(res.status, 0, msg=res.output) @@ -59,9 +62,8 @@ class BitbakePrTests(oeSelfTest): pr_2 = self.get_pr_version(package_name) stamp_2 = self.get_task_stamp(package_name, track_task) - bitbake("-ccleansstate %s" % package_name) - self.assertTrue(pr_2 - pr_1 == 1) - self.assertTrue(stamp_1 != stamp_2) + self.assertTrue(pr_2 - pr_1 == 1, "Step between same pkg. revision is greater than 1") + self.assertTrue(stamp_1 != stamp_2, "Different pkg rev. but same stamp: %s" % stamp_1) def run_test_pr_export_import(self, package_name, replace_current_db=True): self.config_pr_tests(package_name) @@ -85,8 +87,7 @@ class BitbakePrTests(oeSelfTest): self.increment_package_pr(package_name) pr_2 = self.get_pr_version(package_name) - bitbake("-ccleansstate %s" % package_name) - self.assertTrue(pr_2 - pr_1 == 1) + self.assertTrue(pr_2 - pr_1 == 1, "Step between same pkg. revision is greater than 1") @testcase(930) def test_import_export_replace_db(self): @@ -119,3 +120,13 @@ class BitbakePrTests(oeSelfTest): @testcase(936) def test_pr_service_ipk_arch_indep(self): self.run_test_pr_service('xcursor-transparent-theme', 'ipk', 'do_package') + + @testcase(1419) + def test_stopping_prservice_message(self): + port = get_free_port() + + runCmd('bitbake-prserv --host localhost --port %s --loglevel=DEBUG --start' % port) + ret = runCmd('bitbake-prserv --host localhost --port %s --loglevel=DEBUG --stop' % port) + + self.assertEqual(ret.status, 0) + diff --git a/meta/lib/oeqa/selftest/recipetool.py b/meta/lib/oeqa/selftest/recipetool.py new file mode 100644 index 0000000000..7bfb02f161 --- /dev/null +++ b/meta/lib/oeqa/selftest/recipetool.py @@ -0,0 +1,695 @@ +import os +import logging +import shutil +import tempfile +import urllib.parse + +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import get_bb_vars, create_temp_layer +from oeqa.utils.decorators import testcase +from oeqa.selftest import devtool + + +templayerdir = None + + +def setUpModule(): + global templayerdir + templayerdir = tempfile.mkdtemp(prefix='recipetoolqa') + create_temp_layer(templayerdir, 'selftestrecipetool') + runCmd('bitbake-layers add-layer %s' % templayerdir) + + +def tearDownModule(): + runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True) + runCmd('rm -rf %s' % templayerdir) + + +class RecipetoolBase(devtool.DevtoolBase): + + def setUpLocal(self): + self.templayerdir = templayerdir + self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa') + self.track_for_cleanup(self.tempdir) + self.testfile = os.path.join(self.tempdir, 'testfile') + with open(self.testfile, 'w') as f: + f.write('Test file\n') + + def tearDownLocal(self): + runCmd('rm -rf %s/recipes-*' % self.templayerdir) + + def _try_recipetool_appendcmd(self, cmd, testrecipe, expectedfiles, expectedlines=None): + result = runCmd(cmd) + self.assertNotIn('Traceback', result.output) + + # Check the bbappend was created and applies properly + recipefile = get_bb_var('FILE', testrecipe) + bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) + + # Check the bbappend contents + if expectedlines is not None: + with open(bbappendfile, 'r') as f: + self.assertEqual(expectedlines, f.readlines(), "Expected lines are not present in %s" % bbappendfile) + + # Check file was copied + filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) + for expectedfile in expectedfiles: + self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile) + + # Check no other files created + createdfiles = [] + for root, _, files in os.walk(filesdir): + for f in files: + createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir)) + self.assertTrue(sorted(createdfiles), sorted(expectedfiles)) + + return bbappendfile, result.output + + +class RecipetoolTests(RecipetoolBase): + + @classmethod + def setUpClass(cls): + # Ensure we have the right data in shlibs/pkgdata + logger = logging.getLogger("selftest") + logger.info('Running bitbake to generate pkgdata') + bitbake('-c packagedata base-files coreutils busybox selftest-recipetool-appendfile') + bb_vars = get_bb_vars(['COREBASE', 'BBPATH']) + cls.corebase = bb_vars['COREBASE'] + cls.bbpath = bb_vars['BBPATH'] + + def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles): + cmd = 'recipetool appendfile %s %s %s %s' % (self.templayerdir, destfile, newfile, options) + return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) + + def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror): + cmd = 'recipetool appendfile %s %s %s' % (self.templayerdir, destfile, newfile) + result = runCmd(cmd, ignore_status=True) + self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) + self.assertNotIn('Traceback', result.output) + for errorstr in checkerror: + self.assertIn(errorstr, result.output) + + @testcase(1177) + def test_recipetool_appendfile_basic(self): + # Basic test + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd']) + self.assertNotIn('WARNING: ', output) + + @testcase(1183) + def test_recipetool_appendfile_invalid(self): + # Test some commands that should error + self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers']) + self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool']) + self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool']) + + @testcase(1176) + def test_recipetool_appendfile_alternatives(self): + # Now try with a file we know should be an alternative + # (this is very much a fake example, but one we know is reliably an alternative) + self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox']) + # Need a test file - should be executable + testfile2 = os.path.join(self.corebase, 'oe-init-build-env') + testfile2name = os.path.basename(testfile2) + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://%s"\n' % testfile2name, + '\n', + 'do_install_append() {\n', + ' install -d ${D}${base_bindir}\n', + ' install -m 0755 ${WORKDIR}/%s ${D}${base_bindir}/ls\n' % testfile2name, + '}\n'] + self._try_recipetool_appendfile('coreutils', '/bin/ls', testfile2, '-r coreutils', expectedlines, [testfile2name]) + # Now try bbappending the same file again, contents should not change + bbappendfile, _ = self._try_recipetool_appendfile('coreutils', '/bin/ls', self.testfile, '-r coreutils', expectedlines, [testfile2name]) + # But file should have + copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name) + result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True) + self.assertNotEqual(result.status, 0, 'New file should have been copied but was not %s' % result.output) + + @testcase(1178) + def test_recipetool_appendfile_binary(self): + # Try appending a binary file + # /bin/ls can be a symlink to /usr/bin/ls + ls = os.path.realpath("/bin/ls") + result = runCmd('recipetool appendfile %s /bin/ls %s -r coreutils' % (self.templayerdir, ls)) + self.assertIn('WARNING: ', result.output) + self.assertIn('is a binary', result.output) + + @testcase(1173) + def test_recipetool_appendfile_add(self): + # Try arbitrary file add to a recipe + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', + '}\n'] + self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile']) + # Try adding another file, this time where the source file is executable + # (so we're testing that, plus modifying an existing bbappend) + testfile2 = os.path.join(self.corebase, 'oe-init-build-env') + testfile2name = os.path.basename(testfile2) + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile \\\n', + ' file://%s \\\n' % testfile2name, + ' "\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', + ' install -m 0755 ${WORKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name, + '}\n'] + self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name]) + + @testcase(1174) + def test_recipetool_appendfile_add_bindir(self): + # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${bindir}\n', + ' install -m 0755 ${WORKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n', + '}\n'] + _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1175) + def test_recipetool_appendfile_add_machine(self): + # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'PACKAGE_ARCH = "${MACHINE_ARCH}"\n', + '\n', + 'SRC_URI_append_mymachine = " file://testfile"\n', + '\n', + 'do_install_append_mymachine() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', + '}\n'] + _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1184) + def test_recipetool_appendfile_orig(self): + # A file that's in SRC_URI and in do_install with the same name + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig']) + self.assertNotIn('WARNING: ', output) + + @testcase(1191) + def test_recipetool_appendfile_todir(self): + # A file that's in SRC_URI and in do_install with destination directory rather than file + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir']) + self.assertNotIn('WARNING: ', output) + + @testcase(1187) + def test_recipetool_appendfile_renamed(self): + # A file that's in SRC_URI with a different name to the destination file + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1']) + self.assertNotIn('WARNING: ', output) + + @testcase(1190) + def test_recipetool_appendfile_subdir(self): + # A file that's in SRC_URI in a subdir + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n', + '}\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1189) + def test_recipetool_appendfile_src_glob(self): + # A file that's in SRC_URI as a glob + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-src-globfile\n', + '}\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-src-globfile', self.testfile, '', expectedlines, ['testfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1181) + def test_recipetool_appendfile_inst_glob(self): + # A file that's in do_install as a glob + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1182) + def test_recipetool_appendfile_inst_todir_glob(self): + # A file that's in do_install as a glob with destination as a directory + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1185) + def test_recipetool_appendfile_patch(self): + # A file that's added by a patch in SRC_URI + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${sysconfdir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n', + '}\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile']) + for line in output.splitlines(): + if 'WARNING: ' in line: + self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line) + break + else: + self.fail('Patch warning not found in output:\n%s' % output) + + @testcase(1188) + def test_recipetool_appendfile_script(self): + # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install) + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n', + '}\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile']) + self.assertNotIn('WARNING: ', output) + + @testcase(1180) + def test_recipetool_appendfile_inst_func(self): + # A file that's installed from a function called by do_install + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func']) + self.assertNotIn('WARNING: ', output) + + @testcase(1186) + def test_recipetool_appendfile_postinstall(self): + # A file that's created by a postinstall script (and explicitly mentioned in it) + # First try without specifying recipe + self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile']) + # Now specify recipe + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://testfile"\n', + '\n', + 'do_install_append() {\n', + ' install -d ${D}${datadir}\n', + ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n', + '}\n'] + _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile']) + + @testcase(1179) + def test_recipetool_appendfile_extlayer(self): + # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure + exttemplayerdir = os.path.join(self.tempdir, 'extlayer') + self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*') + result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile)) + self.assertNotIn('Traceback', result.output) + createdfiles = [] + for root, _, files in os.walk(exttemplayerdir): + for f in files: + createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir)) + createdfiles.remove('conf/layer.conf') + expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend', + 'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig'] + self.assertEqual(sorted(createdfiles), sorted(expectedfiles)) + + @testcase(1192) + def test_recipetool_appendfile_wildcard(self): + + def try_appendfile_wc(options): + result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (self.templayerdir, self.testfile, options)) + self.assertNotIn('Traceback', result.output) + bbappendfile = None + for root, _, files in os.walk(self.templayerdir): + for f in files: + if f.endswith('.bbappend'): + bbappendfile = f + break + if not bbappendfile: + self.fail('No bbappend file created') + runCmd('rm -rf %s/recipes-*' % self.templayerdir) + return bbappendfile + + # Check without wildcard option + recipefn = os.path.basename(get_bb_var('FILE', 'base-files')) + filename = try_appendfile_wc('') + self.assertEqual(filename, recipefn.replace('.bb', '.bbappend')) + # Now check with wildcard option + filename = try_appendfile_wc('-w') + self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend') + + @testcase(1193) + def test_recipetool_create(self): + # Try adding a recipe + tempsrc = os.path.join(self.tempdir, 'srctree') + os.makedirs(tempsrc) + recipefile = os.path.join(self.tempdir, 'logrotate_3.8.7.bb') + srcuri = 'https://github.com/logrotate/logrotate/archive/r3-8-7.tar.gz' + result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = 'GPLv2' + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=18810669f13b87348459e611d31ab760' + checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/archive/r3-8-7.tar.gz' + checkvars['SRC_URI[md5sum]'] = '6b1aa0e0d07eda3c9a2526520850397a' + checkvars['SRC_URI[sha256sum]'] = 'dece4bfeb9d8374a0ecafa34be139b5a697db5c926dcc69a9b8715431a22e733' + self._test_recipe_contents(recipefile, checkvars, []) + + @testcase(1194) + def test_recipetool_create_git(self): + if 'x11' not in get_bb_var('DISTRO_FEATURES'): + self.skipTest('Test requires x11 as distro feature') + # Ensure we have the right data in shlibs/pkgdata + bitbake('libpng pango libx11 libxext jpeg libcheck') + # Try adding a recipe + tempsrc = os.path.join(self.tempdir, 'srctree') + os.makedirs(tempsrc) + recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') + srcuri = 'git://git.yoctoproject.org/libmatchbox' + result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) + self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) + checkvars = {} + checkvars['LICENSE'] = 'LGPLv2.1' + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' + checkvars['S'] = '${WORKDIR}/git' + checkvars['PV'] = '1.11+git${SRCPV}' + checkvars['SRC_URI'] = srcuri + checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) + inherits = ['autotools', 'pkgconfig'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + @testcase(1392) + def test_recipetool_create_simple(self): + # Try adding a recipe + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pv = '1.7.3.0' + srcuri = 'http://www.dest-unreach.org/socat/download/socat-%s.tar.bz2' % pv + result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe)) + dirlist = os.listdir(temprecipe) + if len(dirlist) > 1: + self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) + if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): + self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) + self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named') + checkvars = {} + checkvars['LICENSE'] = set(['Unknown', 'GPLv2']) + checkvars['LIC_FILES_CHKSUM'] = set(['file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4', 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263']) + # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot + checkvars['S'] = None + checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}') + inherits = ['autotools'] + self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits) + + @testcase(1418) + def test_recipetool_create_cmake(self): + # Try adding a recipe + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'navit_0.5.0.bb') + srcuri = 'http://downloads.sourceforge.net/project/navit/v0.5.0/navit-0.5.0.tar.gz' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Unknown', 'GPLv2', 'LGPLv2']) + checkvars['SRC_URI'] = 'http://downloads.sourceforge.net/project/navit/v${PV}/navit-${PV}.tar.gz' + checkvars['SRC_URI[md5sum]'] = '242f398e979a6b8c0f3c802b63435b68' + checkvars['SRC_URI[sha256sum]'] = '13353481d7fc01a4f64e385dda460b51496366bba0fd2cc85a89a0747910e94d' + checkvars['DEPENDS'] = set(['freetype', 'zlib', 'openssl', 'glib-2.0', 'virtual/libgl', 'virtual/egl', 'gtk+', 'libpng', 'libsdl', 'freeglut', 'dbus-glib']) + inherits = ['cmake', 'python-dir', 'gettext', 'pkgconfig'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_github(self): + # Basic test to see if github URL mangling works + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'meson_git.bb') + srcuri = 'https://github.com/mesonbuild/meson' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Apache-2.0']) + checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https' + inherits = ['setuptools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_github_tarball(self): + # Basic test to ensure github URL mangling doesn't apply to release tarballs + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pv = '0.32.0' + recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv) + srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Apache-2.0']) + checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz' + inherits = ['setuptools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_git_http(self): + # Basic test to check http git URL mangling works + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'matchbox-terminal_git.bb') + srcuri = 'http://git.yoctoproject.org/git/matchbox-terminal' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['GPLv2']) + checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/matchbox-terminal;protocol=http' + inherits = ['pkgconfig', 'autotools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): + dstdir = basedstdir + self.assertTrue(os.path.exists(dstdir)) + for p in paths: + dstdir = os.path.join(dstdir, p) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + self.track_for_cleanup(dstdir) + dstfile = os.path.join(dstdir, os.path.basename(srcfile)) + if srcfile != dstfile: + shutil.copy(srcfile, dstfile) + self.track_for_cleanup(dstfile) + + def test_recipetool_load_plugin(self): + """Test that recipetool loads only the first found plugin in BBPATH.""" + + recipetool = runCmd("which recipetool") + fromname = runCmd("recipetool --quiet pluginfile") + srcfile = fromname.output + searchpath = self.bbpath.split(':') + [os.path.dirname(recipetool.output)] + plugincontent = [] + with open(srcfile) as fh: + plugincontent = fh.readlines() + try: + self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') + for path in searchpath: + self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') + result = runCmd("recipetool --quiet count") + self.assertEqual(result.output, '1') + result = runCmd("recipetool --quiet multiloaded") + self.assertEqual(result.output, "no") + for path in searchpath: + result = runCmd("recipetool --quiet bbdir") + self.assertEqual(result.output, path) + os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py')) + finally: + with open(srcfile, 'w') as fh: + fh.writelines(plugincontent) + + +class RecipetoolAppendsrcBase(RecipetoolBase): + def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): + cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile) + return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) + + def _try_recipetool_appendsrcfiles(self, testrecipe, newfiles, expectedlines=None, expectedfiles=None, destdir=None, options=''): + + if destdir: + options += ' -D %s' % destdir + + if expectedfiles is None: + expectedfiles = [os.path.basename(f) for f in newfiles] + + cmd = 'recipetool appendsrcfiles %s %s %s %s' % (options, self.templayerdir, testrecipe, ' '.join(newfiles)) + return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) + + def _try_recipetool_appendsrcfile_fail(self, testrecipe, newfile, destfile, checkerror): + cmd = 'recipetool appendsrcfile %s %s %s %s' % (self.templayerdir, testrecipe, newfile, destfile or '') + result = runCmd(cmd, ignore_status=True) + self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) + self.assertNotIn('Traceback', result.output) + for errorstr in checkerror: + self.assertIn(errorstr, result.output) + + @staticmethod + def _get_first_file_uri(recipe): + '''Return the first file:// in SRC_URI for the specified recipe.''' + src_uri = get_bb_var('SRC_URI', recipe).split() + for uri in src_uri: + p = urllib.parse.urlparse(uri) + if p.scheme == 'file': + return p.netloc + p.path + + def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, options=''): + if newfile is None: + newfile = self.testfile + + if srcdir: + if destdir: + expected_subdir = os.path.join(srcdir, destdir) + else: + expected_subdir = srcdir + else: + options += " -W" + expected_subdir = destdir + + if filename: + if destdir: + destpath = os.path.join(destdir, filename) + else: + destpath = filename + else: + filename = os.path.basename(newfile) + if destdir: + destpath = destdir + os.sep + else: + destpath = '.' + os.sep + + expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + '\n'] + if has_src_uri: + uri = 'file://%s' % filename + if expected_subdir: + uri += ';subdir=%s' % expected_subdir + expectedlines[0:0] = ['SRC_URI += "%s"\n' % uri, + '\n'] + + return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename]) + + def _test_appendsrcfiles(self, testrecipe, newfiles, expectedfiles=None, destdir=None, options=''): + if expectedfiles is None: + expectedfiles = [os.path.basename(n) for n in newfiles] + + self._try_recipetool_appendsrcfiles(testrecipe, newfiles, expectedfiles=expectedfiles, destdir=destdir, options=options) + + bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'FILESEXTRAPATHS'], testrecipe) + src_uri = bb_vars['SRC_URI'].split() + for f in expectedfiles: + if destdir: + self.assertIn('file://%s;subdir=%s' % (f, destdir), src_uri) + else: + self.assertIn('file://%s' % f, src_uri) + + recipefile = bb_vars['FILE'] + bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) + filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) + filesextrapaths = bb_vars['FILESEXTRAPATHS'].split(':') + self.assertIn(filesdir, filesextrapaths) + + + + +class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): + + @testcase(1273) + def test_recipetool_appendsrcfile_basic(self): + self._test_appendsrcfile('base-files', 'a-file') + + @testcase(1274) + def test_recipetool_appendsrcfile_basic_wildcard(self): + testrecipe = 'base-files' + self._test_appendsrcfile(testrecipe, 'a-file', options='-w') + recipefile = get_bb_var('FILE', testrecipe) + bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) + self.assertEqual(os.path.basename(bbappendfile), '%s_%%.bbappend' % testrecipe) + + @testcase(1281) + def test_recipetool_appendsrcfile_subdir_basic(self): + self._test_appendsrcfile('base-files', 'a-file', 'tmp') + + @testcase(1282) + def test_recipetool_appendsrcfile_subdir_basic_dirdest(self): + self._test_appendsrcfile('base-files', destdir='tmp') + + @testcase(1280) + def test_recipetool_appendsrcfile_srcdir_basic(self): + testrecipe = 'bash' + bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) + srcdir = bb_vars['S'] + workdir = bb_vars['WORKDIR'] + subdir = os.path.relpath(srcdir, workdir) + self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir) + + @testcase(1275) + def test_recipetool_appendsrcfile_existing_in_src_uri(self): + testrecipe = 'base-files' + filepath = self._get_first_file_uri(testrecipe) + self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) + self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) + + @testcase(1276) + def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): + testrecipe = 'base-files' + subdir = 'tmp' + filepath = self._get_first_file_uri(testrecipe) + self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) + + output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False) + self.assertTrue(any('with different parameters' in l for l in output)) + + @testcase(1277) + def test_recipetool_appendsrcfile_replace_file_srcdir(self): + testrecipe = 'bash' + filepath = 'Makefile.in' + bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) + srcdir = bb_vars['S'] + workdir = bb_vars['WORKDIR'] + subdir = os.path.relpath(srcdir, workdir) + + self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir) + bitbake('%s:do_unpack' % testrecipe) + self.assertEqual(open(self.testfile, 'r').read(), open(os.path.join(srcdir, filepath), 'r').read()) + + @testcase(1278) + def test_recipetool_appendsrcfiles_basic(self, destdir=None): + newfiles = [self.testfile] + for i in range(1, 5): + testfile = os.path.join(self.tempdir, 'testfile%d' % i) + with open(testfile, 'w') as f: + f.write('Test file %d\n' % i) + newfiles.append(testfile) + self._test_appendsrcfiles('gcc', newfiles, destdir=destdir, options='-W') + + @testcase(1279) + def test_recipetool_appendsrcfiles_basic_subdir(self): + self.test_recipetool_appendsrcfiles_basic(destdir='testdir') diff --git a/meta/lib/oeqa/selftest/runqemu.py b/meta/lib/oeqa/selftest/runqemu.py new file mode 100644 index 0000000000..58c6f96f98 --- /dev/null +++ b/meta/lib/oeqa/selftest/runqemu.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2017 Wind River Systems, Inc. +# + +import re +import logging + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake, runqemu, get_bb_var +from oeqa.utils.decorators import testcase + +class RunqemuTests(oeSelfTest): + """Runqemu test class""" + + image_is_ready = False + deploy_dir_image = '' + + def setUpLocal(self): + self.recipe = 'core-image-minimal' + self.machine = 'qemux86-64' + self.fstypes = "ext4 iso hddimg vmdk qcow2 vdi" + self.cmd_common = "runqemu nographic" + + # Avoid emit the same record multiple times. + mainlogger = logging.getLogger("BitBake.Main") + mainlogger.propagate = False + + self.write_config( +""" +MACHINE = "%s" +IMAGE_FSTYPES = "%s" +# 10 means 1 second +SYSLINUX_TIMEOUT = "10" +""" +% (self.machine, self.fstypes) + ) + + if not RunqemuTests.image_is_ready: + RunqemuTests.deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + bitbake(self.recipe) + RunqemuTests.image_is_ready = True + + @testcase(2001) + def test_boot_machine(self): + """Test runqemu machine""" + cmd = "%s %s" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2002) + def test_boot_machine_ext4(self): + """Test runqemu machine ext4""" + cmd = "%s %s ext4" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('rootfs.ext4' in f.read(), "Failed: %s" % cmd) + + @testcase(2003) + def test_boot_machine_iso(self): + """Test runqemu machine iso""" + cmd = "%s %s iso" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue(' -cdrom ' in f.read(), "Failed: %s" % cmd) + + @testcase(2004) + def test_boot_recipe_image(self): + """Test runqemu recipe-image""" + cmd = "%s %s" % (self.cmd_common, self.recipe) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2005) + def test_boot_recipe_image_vmdk(self): + """Test runqemu recipe-image vmdk""" + cmd = "%s %s vmdk" % (self.cmd_common, self.recipe) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('format=vmdk' in f.read(), "Failed: %s" % cmd) + + @testcase(2006) + def test_boot_recipe_image_vdi(self): + """Test runqemu recipe-image vdi""" + cmd = "%s %s vdi" % (self.cmd_common, self.recipe) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('format=vdi' in f.read(), "Failed: %s" % cmd) + + @testcase(2007) + def test_boot_deploy(self): + """Test runqemu deploy_dir_image""" + cmd = "%s %s" % (self.cmd_common, self.deploy_dir_image) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2008) + def test_boot_deploy_hddimg(self): + """Test runqemu deploy_dir_image hddimg""" + cmd = "%s %s hddimg" % (self.cmd_common, self.deploy_dir_image) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s" % cmd) + + @testcase(2009) + def test_boot_machine_slirp(self): + """Test runqemu machine slirp""" + cmd = "%s slirp %s" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue(' -netdev user' in f.read(), "Failed: %s" % cmd) + + @testcase(2009) + def test_boot_machine_slirp_qcow2(self): + """Test runqemu machine slirp qcow2""" + cmd = "%s slirp qcow2 %s" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('format=qcow2' in f.read(), "Failed: %s" % cmd) + + @testcase(2010) + def test_boot_qemu_boot(self): + """Test runqemu /path/to/image.qemuboot.conf""" + qemuboot_conf = "%s-%s.qemuboot.conf" % (self.recipe, self.machine) + qemuboot_conf = os.path.join(self.deploy_dir_image, qemuboot_conf) + if not os.path.exists(qemuboot_conf): + self.skipTest("%s not found" % qemuboot_conf) + cmd = "%s %s" % (self.cmd_common, qemuboot_conf) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2011) + def test_boot_rootfs(self): + """Test runqemu /path/to/rootfs.ext4""" + rootfs = "%s-%s.ext4" % (self.recipe, self.machine) + rootfs = os.path.join(self.deploy_dir_image, rootfs) + if not os.path.exists(rootfs): + self.skipTest("%s not found" % rootfs) + cmd = "%s %s" % (self.cmd_common, rootfs) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) diff --git a/meta/lib/oeqa/selftest/runtime-test.py b/meta/lib/oeqa/selftest/runtime-test.py new file mode 100644 index 0000000000..e498d046cf --- /dev/null +++ b/meta/lib/oeqa/selftest/runtime-test.py @@ -0,0 +1,237 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu +from oeqa.utils.decorators import testcase +import os +import re + +class TestExport(oeSelfTest): + + @classmethod + def tearDownClass(cls): + runCmd("rm -rf /tmp/sdk") + + def test_testexport_basic(self): + """ + Summary: Check basic testexport functionality with only ping test enabled. + Expected: 1. testexport directory must be created. + 2. runexported.py must run without any error/exception. + 3. ping test must succeed. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testexport"\n' + # These aren't the actual IP addresses but testexport class needs something defined + features += 'TEST_SERVER_IP = "192.168.7.1"\n' + features += 'TEST_TARGET_IP = "192.168.7.1"\n' + features += 'TEST_SUITES = "ping"\n' + self.write_config(features) + + # Build tesexport for core-image-minimal + bitbake('core-image-minimal') + bitbake('-c testexport core-image-minimal') + + testexport_dir = get_bb_var('TEST_EXPORT_DIR', 'core-image-minimal') + + # Verify if TEST_EXPORT_DIR was created + isdir = os.path.isdir(testexport_dir) + self.assertEqual(True, isdir, 'Failed to create testexport dir: %s' % testexport_dir) + + with runqemu('core-image-minimal') as qemu: + # Attempt to run runexported.py to perform ping test + test_path = os.path.join(testexport_dir, "oe-test") + data_file = os.path.join(testexport_dir, 'data', 'testdata.json') + manifest = os.path.join(testexport_dir, 'data', 'manifest') + cmd = ("%s runtime --test-data-file %s --packages-manifest %s " + "--target-ip %s --server-ip %s --quiet" + % (test_path, data_file, manifest, qemu.ip, qemu.server_ip)) + result = runCmd(cmd) + # Verify ping test was succesful + self.assertEqual(0, result.status, 'oe-test runtime returned a non 0 status') + + def test_testexport_sdk(self): + """ + Summary: Check sdk functionality for testexport. + Expected: 1. testexport directory must be created. + 2. SDK tarball must exists. + 3. Uncompressing of tarball must succeed. + 4. Check if the SDK directory is added to PATH. + 5. Run tar from the SDK directory. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testexport"\n' + # These aren't the actual IP addresses but testexport class needs something defined + features += 'TEST_SERVER_IP = "192.168.7.1"\n' + features += 'TEST_TARGET_IP = "192.168.7.1"\n' + features += 'TEST_SUITES = "ping"\n' + features += 'TEST_EXPORT_SDK_ENABLED = "1"\n' + features += 'TEST_EXPORT_SDK_PACKAGES = "nativesdk-tar"\n' + self.write_config(features) + + # Build tesexport for core-image-minimal + bitbake('core-image-minimal') + bitbake('-c testexport core-image-minimal') + + needed_vars = ['TEST_EXPORT_DIR', 'TEST_EXPORT_SDK_DIR', 'TEST_EXPORT_SDK_NAME'] + bb_vars = get_bb_vars(needed_vars, 'core-image-minimal') + testexport_dir = bb_vars['TEST_EXPORT_DIR'] + sdk_dir = bb_vars['TEST_EXPORT_SDK_DIR'] + sdk_name = bb_vars['TEST_EXPORT_SDK_NAME'] + + # Check for SDK + tarball_name = "%s.sh" % sdk_name + tarball_path = os.path.join(testexport_dir, sdk_dir, tarball_name) + msg = "Couldn't find SDK tarball: %s" % tarball_path + self.assertEqual(os.path.isfile(tarball_path), True, msg) + + # Extract SDK and run tar from SDK + result = runCmd("%s -y -d /tmp/sdk" % tarball_path) + self.assertEqual(0, result.status, "Couldn't extract SDK") + + env_script = result.output.split()[-1] + result = runCmd(". %s; which tar" % env_script, shell=True) + self.assertEqual(0, result.status, "Couldn't setup SDK environment") + is_sdk_tar = True if "/tmp/sdk" in result.output else False + self.assertTrue(is_sdk_tar, "Couldn't setup SDK environment") + + tar_sdk = result.output + result = runCmd("%s --version" % tar_sdk) + self.assertEqual(0, result.status, "Couldn't run tar from SDK") + + +class TestImage(oeSelfTest): + + def test_testimage_install(self): + """ + Summary: Check install packages functionality for testimage/testexport. + Expected: 1. Import tests from a directory other than meta. + 2. Check install/uninstall of socat. + 3. Check that remote package feeds can be accessed + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + Author: Alexander Kanavin <alexander.kanavin@intel.com> + """ + if get_bb_var('DISTRO') == 'poky-tiny': + self.skipTest('core-image-full-cmdline not buildable for poky-tiny') + + features = 'INHERIT += "testimage"\n' + features += 'TEST_SUITES = "ping ssh selftest"\n' + # We don't yet know what the server ip and port will be - they will be patched + # in at the start of the on-image test + features += 'PACKAGE_FEED_URIS = "http://bogus_ip:bogus_port"\n' + features += 'EXTRA_IMAGE_FEATURES += "package-management"\n' + features += 'PACKAGE_CLASSES = "package_rpm"' + self.write_config(features) + + # Build core-image-sato and testimage + bitbake('core-image-full-cmdline socat') + bitbake('-c testimage core-image-full-cmdline') + +class Postinst(oeSelfTest): + @testcase(1540) + def test_verify_postinst(self): + """ + Summary: The purpose of this test is to verify the execution order of postinst Bugzilla ID: [5319] + Expected : + 1. Compile a minimal image. + 2. The compiled image will add the created layer with the recipes postinst[ abdpt] + 3. Run qemux86 + 4. Validate the task execution order + Author: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com> + """ + features = 'INHERIT += "testimage"\n' + features += 'CORE_IMAGE_EXTRA_INSTALL += "postinst-at-rootfs \ +postinst-delayed-a \ +postinst-delayed-b \ +postinst-delayed-d \ +postinst-delayed-p \ +postinst-delayed-t \ +"\n' + self.write_config(features) + + bitbake('core-image-minimal -f ') + + postinst_list = ['100-postinst-at-rootfs', + '101-postinst-delayed-a', + '102-postinst-delayed-b', + '103-postinst-delayed-d', + '104-postinst-delayed-p', + '105-postinst-delayed-t'] + path_workdir = get_bb_var('WORKDIR','core-image-minimal') + workspacedir = 'testimage/qemu_boot_log' + workspacedir = os.path.join(path_workdir, workspacedir) + rexp = re.compile("^Running postinst .*/(?P<postinst>.*)\.\.\.$") + with runqemu('core-image-minimal') as qemu: + with open(workspacedir) as f: + found = False + idx = 0 + for line in f.readlines(): + line = line.strip().replace("^M","") + if not line: # To avoid empty lines + continue + m = rexp.search(line) + if m: + self.assertEqual(postinst_list[idx], m.group('postinst'), "Fail") + idx = idx+1 + found = True + elif found: + self.assertEqual(idx, len(postinst_list), "Not found all postinsts") + break + + @testcase(1545) + def test_postinst_rootfs_and_boot(self): + """ + Summary: The purpose of this test case is to verify Post-installation + scripts are called when rootfs is created and also test + that script can be delayed to run at first boot. + Dependencies: NA + Steps: 1. Add proper configuration to local.conf file + 2. Build a "core-image-minimal" image + 3. Verify that file created by postinst_rootfs recipe is + present on rootfs dir. + 4. Boot the image created on qemu and verify that the file + created by postinst_boot recipe is present on image. + Expected: The files are successfully created during rootfs and boot + time for 3 different package managers: rpm,ipk,deb and + for initialization managers: sysvinit and systemd. + + """ + file_rootfs_name = "this-was-created-at-rootfstime" + fileboot_name = "this-was-created-at-first-boot" + rootfs_pkg = 'postinst-at-rootfs' + boot_pkg = 'postinst-delayed-a' + #Step 1 + features = 'MACHINE = "qemux86"\n' + features += 'CORE_IMAGE_EXTRA_INSTALL += "%s %s "\n'% (rootfs_pkg, boot_pkg) + features += 'IMAGE_FEATURES += "ssh-server-openssh"\n' + for init_manager in ("sysvinit", "systemd"): + #for sysvinit no extra configuration is needed, + if (init_manager is "systemd"): + features += 'DISTRO_FEATURES_append = " systemd"\n' + features += 'VIRTUAL-RUNTIME_init_manager = "systemd"\n' + features += 'DISTRO_FEATURES_BACKFILL_CONSIDERED = "sysvinit"\n' + features += 'VIRTUAL-RUNTIME_initscripts = ""\n' + for classes in ("package_rpm package_deb package_ipk", + "package_deb package_rpm package_ipk", + "package_ipk package_deb package_rpm"): + features += 'PACKAGE_CLASSES = "%s"\n' % classes + self.write_config(features) + + #Step 2 + bitbake('core-image-minimal') + + #Step 3 + file_rootfs_created = os.path.join(get_bb_var('IMAGE_ROOTFS',"core-image-minimal"), + file_rootfs_name) + found = os.path.isfile(file_rootfs_created) + self.assertTrue(found, "File %s was not created at rootfs time by %s" % \ + (file_rootfs_name, rootfs_pkg)) + + #Step 4 + testcommand = 'ls /etc/'+fileboot_name + with runqemu('core-image-minimal') as qemu: + sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand)) + self.assertEqual(result.status, 0, 'File %s was not created at firts boot'% fileboot_name) diff --git a/meta/lib/oeqa/selftest/signing.py b/meta/lib/oeqa/selftest/signing.py new file mode 100644 index 0000000000..0ac3d1fac9 --- /dev/null +++ b/meta/lib/oeqa/selftest/signing.py @@ -0,0 +1,183 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars +import os +import glob +import re +import shutil +import tempfile +from oeqa.utils.decorators import testcase +from oeqa.utils.ftools import write_file + + +class Signing(oeSelfTest): + + gpg_dir = "" + pub_key_path = "" + secret_key_path = "" + + @classmethod + def setUpClass(cls): + # Check that we can find the gpg binary and fail early if we can't + if not shutil.which("gpg"): + raise AssertionError("This test needs GnuPG") + + cls.gpg_home_dir = tempfile.TemporaryDirectory(prefix="oeqa-signing-") + cls.gpg_dir = cls.gpg_home_dir.name + + cls.pub_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.pub") + cls.secret_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.secret") + + runCmd('gpg --batch --homedir %s --import %s %s' % (cls.gpg_dir, cls.pub_key_path, cls.secret_key_path)) + + @testcase(1362) + def test_signing_packages(self): + """ + Summary: Test that packages can be signed in the package feed + Expected: Package should be signed with the correct key + Expected: Images can be created from signed packages + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + Author: Alexander Kanavin <alexander.kanavin@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + import oe.packagedata + + package_classes = get_bb_var('PACKAGE_CLASSES') + if 'package_rpm' not in package_classes: + self.skipTest('This test requires RPM Packaging.') + + test_recipe = 'ed' + + feature = 'INHERIT += "sign_rpm"\n' + feature += 'RPM_GPG_PASSPHRASE = "test123"\n' + feature += 'RPM_GPG_NAME = "testuser"\n' + feature += 'GPG_PATH = "%s"\n' % self.gpg_dir + + self.write_config(feature) + + bitbake('-c clean %s' % test_recipe) + bitbake('-f -c package_write_rpm %s' % test_recipe) + + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + + needed_vars = ['PKGDATA_DIR', 'DEPLOY_DIR_RPM', 'PACKAGE_ARCH', 'STAGING_BINDIR_NATIVE'] + bb_vars = get_bb_vars(needed_vars, test_recipe) + pkgdatadir = bb_vars['PKGDATA_DIR'] + pkgdata = oe.packagedata.read_pkgdatafile(pkgdatadir + "/runtime/ed") + if 'PKGE' in pkgdata: + pf = pkgdata['PN'] + "-" + pkgdata['PKGE'] + pkgdata['PKGV'] + '-' + pkgdata['PKGR'] + else: + pf = pkgdata['PN'] + "-" + pkgdata['PKGV'] + '-' + pkgdata['PKGR'] + deploy_dir_rpm = bb_vars['DEPLOY_DIR_RPM'] + package_arch = bb_vars['PACKAGE_ARCH'].replace('-', '_') + staging_bindir_native = bb_vars['STAGING_BINDIR_NATIVE'] + + pkg_deploy = os.path.join(deploy_dir_rpm, package_arch, '.'.join((pf, package_arch, 'rpm'))) + + # Use a temporary rpmdb + rpmdb = tempfile.mkdtemp(prefix='oeqa-rpmdb') + + runCmd('%s/rpmkeys --define "_dbpath %s" --import %s' % + (staging_bindir_native, rpmdb, self.pub_key_path)) + + ret = runCmd('%s/rpmkeys --define "_dbpath %s" --checksig %s' % + (staging_bindir_native, rpmdb, pkg_deploy)) + # tmp/deploy/rpm/i586/ed-1.9-r0.i586.rpm: rsa sha1 md5 OK + self.assertIn('rsa sha1 (md5) pgp md5 OK', ret.output, 'Package signed incorrectly.') + shutil.rmtree(rpmdb) + + #Check that an image can be built from signed packages + self.add_command_to_tearDown('bitbake -c clean core-image-minimal') + bitbake('-c clean core-image-minimal') + bitbake('core-image-minimal') + + + @testcase(1382) + def test_signing_sstate_archive(self): + """ + Summary: Test that sstate archives can be signed + Expected: Package should be signed with the correct key + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + test_recipe = 'ed' + + builddir = os.environ.get('BUILDDIR') + sstatedir = os.path.join(builddir, 'test-sstate') + + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + self.add_command_to_tearDown('rm -rf %s' % sstatedir) + + feature = 'SSTATE_SIG_KEY ?= "testuser"\n' + feature += 'SSTATE_SIG_PASSPHRASE ?= "test123"\n' + feature += 'SSTATE_VERIFY_SIG ?= "1"\n' + feature += 'GPG_PATH = "%s"\n' % self.gpg_dir + feature += 'SSTATE_DIR = "%s"\n' % sstatedir + # Any mirror might have partial sstate without .sig files, triggering failures + feature += 'SSTATE_MIRRORS_forcevariable = ""\n' + + self.write_config(feature) + + bitbake('-c clean %s' % test_recipe) + bitbake(test_recipe) + + recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_package.tgz.sig') + recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_package.tgz') + + self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.') + self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.') + + ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0])) + # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30 + # gpg: Good signature from "testuser (nocomment) <testuser@email.com>" + self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.') + + +class LockedSignatures(oeSelfTest): + + @testcase(1420) + def test_locked_signatures(self): + """ + Summary: Test locked signature mechanism + Expected: Locked signatures will prevent task to run + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + test_recipe = 'ed' + locked_sigs_file = 'locked-sigs.inc' + + self.add_command_to_tearDown('rm -f %s' % os.path.join(self.builddir, locked_sigs_file)) + + bitbake(test_recipe) + # Generate locked sigs include file + bitbake('-S none %s' % test_recipe) + + feature = 'require %s\n' % locked_sigs_file + feature += 'SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n' + self.write_config(feature) + + # Build a locked recipe + bitbake(test_recipe) + + # Make a change that should cause the locked task signature to change + recipe_append_file = test_recipe + '_' + get_bb_var('PV', test_recipe) + '.bbappend' + recipe_append_path = os.path.join(self.testlayer_path, 'recipes-test', test_recipe, recipe_append_file) + feature = 'SUMMARY += "test locked signature"\n' + + os.mkdir(os.path.join(self.testlayer_path, 'recipes-test', test_recipe)) + write_file(recipe_append_path, feature) + + self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, 'recipes-test', test_recipe)) + + # Build the recipe again + ret = bitbake(test_recipe) + + # Verify you get the warning and that the real task *isn't* run (i.e. the locked signature has worked) + patt = r'WARNING: The %s:do_package sig is computed to be \S+, but the sig is locked to \S+ in SIGGEN_LOCKEDSIGS\S+' % test_recipe + found_warn = re.search(patt, ret.output) + + self.assertIsNotNone(found_warn, "Didn't find the expected warning message. Output: %s" % ret.output) diff --git a/meta/lib/oeqa/selftest/sstate.py b/meta/lib/oeqa/selftest/sstate.py index 5989724432..f54bc41465 100644 --- a/meta/lib/oeqa/selftest/sstate.py +++ b/meta/lib/oeqa/selftest/sstate.py @@ -6,16 +6,24 @@ import shutil import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer +from oeqa.utils.commands import runCmd, bitbake, get_bb_vars, get_test_layer class SStateBase(oeSelfTest): def setUpLocal(self): self.temp_sstate_location = None - self.sstate_path = get_bb_var('SSTATE_DIR') - self.distro = get_bb_var('NATIVELSBSTRING') - self.distro_specific_sstate = os.path.join(self.sstate_path, self.distro) + needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', + 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] + bb_vars = get_bb_vars(needed_vars) + self.sstate_path = bb_vars['SSTATE_DIR'] + self.hostdistro = bb_vars['NATIVELSBSTRING'] + self.tclibc = bb_vars['TCLIBC'] + self.tune_arch = bb_vars['TUNE_ARCH'] + self.topdir = bb_vars['TOPDIR'] + self.target_vendor = bb_vars['TARGET_VENDOR'] + self.target_os = bb_vars['TARGET_OS'] + self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) # Creates a special sstate configuration with the option to add sstate mirrors def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): @@ -26,9 +34,10 @@ class SStateBase(oeSelfTest): config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path self.append_config(config_temp_sstate) self.track_for_cleanup(temp_sstate_path) - self.sstate_path = get_bb_var('SSTATE_DIR') - self.distro = get_bb_var('NATIVELSBSTRING') - self.distro_specific_sstate = os.path.join(self.sstate_path, self.distro) + bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) + self.sstate_path = bb_vars['SSTATE_DIR'] + self.hostdistro = bb_vars['NATIVELSBSTRING'] + self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) if add_local_mirrors: config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' @@ -42,7 +51,7 @@ class SStateBase(oeSelfTest): def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): result = [] for root, dirs, files in os.walk(self.sstate_path): - if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.distro, root): + if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.hostdistro, root): for f in files: if re.search(filename_regex, f): result.append(f) diff --git a/meta/lib/oeqa/selftest/sstatetests.py b/meta/lib/oeqa/selftest/sstatetests.py index 3789426797..e35ddfff5f 100644 --- a/meta/lib/oeqa/selftest/sstatetests.py +++ b/meta/lib/oeqa/selftest/sstatetests.py @@ -3,6 +3,8 @@ import unittest import os import re import shutil +import glob +import subprocess import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest @@ -14,7 +16,7 @@ class SStateTests(SStateBase): # Test sstate files creation and their location def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): - self.config_sstate(temp_sstate_location) + self.config_sstate(temp_sstate_location, [self.sstate_path]) if self.temp_sstate_location: bitbake(['-cclean'] + targets) @@ -22,71 +24,85 @@ class SStateTests(SStateBase): bitbake(['-ccleansstate'] + targets) bitbake(targets) - file_tracker = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) + file_tracker = [] + results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) + if distro_nonspecific: + for r in results: + if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo", "_fetch.tgz.siginfo", "_unpack.tgz.siginfo", "_patch.tgz.siginfo")): + continue + file_tracker.append(r) + else: + file_tracker = results + if should_pass: self.assertTrue(file_tracker , msg="Could not find sstate files for: %s" % ', '.join(map(str, targets))) else: - self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s" % ', '.join(map(str, targets))) + self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) @testcase(975) def test_sstate_creation_distro_specific_pass(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) - @testcase(975) + @testcase(1374) def test_sstate_creation_distro_specific_fail(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) + self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) @testcase(976) def test_sstate_creation_distro_nonspecific_pass(self): - self.run_test_sstate_creation(['eglibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) - @testcase(976) + @testcase(1375) def test_sstate_creation_distro_nonspecific_fail(self): - self.run_test_sstate_creation(['eglibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) - + self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) # Test the sstate files deletion part of the do_cleansstate task def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): - self.config_sstate(temp_sstate_location) + self.config_sstate(temp_sstate_location, [self.sstate_path]) bitbake(['-ccleansstate'] + targets) bitbake(targets) tgz_created = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) - self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s" % ', '.join(map(str, targets))) + self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_created))) siginfo_created = self.search_sstate('|'.join(map(str, [s + '.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) - self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s" % ', '.join(map(str, targets))) + self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) bitbake(['-ccleansstate'] + targets) tgz_removed = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) - self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s" % ', '.join(map(str, targets))) + self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_removed))) @testcase(977) def test_cleansstate_task_distro_specific_nonspecific(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_cleansstate_task(['binutils-cross-' + targetarch, 'binutils-native', 'eglibc-initial'], distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) + targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] + targets.append('linux-libc-headers') + self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) - @testcase(977) + @testcase(1376) def test_cleansstate_task_distro_nonspecific(self): - self.run_test_cleansstate_task(['eglibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) - @testcase(977) + @testcase(1377) def test_cleansstate_task_distro_specific(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_cleansstate_task(['binutils-cross-'+ targetarch, 'binutils-native', 'eglibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] + targets.append('linux-libc-headers') + self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) # Test rebuilding of distro-specific sstate files def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): - self.config_sstate(temp_sstate_location) + self.config_sstate(temp_sstate_location, [self.sstate_path]) bitbake(['-ccleansstate'] + targets) bitbake(targets) - self.assertTrue(self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) == [], msg="Found distro non-specific sstate for: %s" % ', '.join(map(str, targets))) + results = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) + filtered_results = [] + for r in results: + if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo")): + continue + filtered_results.append(r) + self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) file_tracker_1 = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files ware created for: %s" % ', '.join(map(str, targets))) @@ -107,15 +123,13 @@ class SStateTests(SStateBase): @testcase(175) def test_rebuild_distro_specific_sstate_cross_native_targets(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch, 'binutils-native'], temp_sstate_location=True) + self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) - @testcase(175) + @testcase(1372) def test_rebuild_distro_specific_sstate_cross_target(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch], temp_sstate_location=True) + self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch], temp_sstate_location=True) - @testcase(175) + @testcase(1373) def test_rebuild_distro_specific_sstate_native_target(self): self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) @@ -128,10 +142,9 @@ class SStateTests(SStateBase): self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) - # If buildhistory is enabled, we need to disable version-going-backwards QA checks for this test. It may report errors otherwise. - if ('buildhistory' in get_bb_var('USER_CLASSES')) or ('buildhistory' in get_bb_var('INHERIT')): - remove_errors_config = 'ERROR_QA_remove = "version-going-backwards"' - self.append_config(remove_errors_config) + # If buildhistory is enabled, we need to disable version-going-backwards + # QA checks for this test. It may report errors otherwise. + self.append_config('ERROR_QA_remove = "version-going-backwards"') # For not this only checks if random sstate tasks are handled correctly as a group. # In the future we should add control over what tasks we check for. @@ -153,7 +166,7 @@ class SStateTests(SStateBase): expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] self.remove_config(global_config[idx]) self.remove_recipeinc(target, target_config[idx]) - self.assertEqual(result.status, 0) + self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list)))) actual_remaining_sstate = [x for x in self.search_sstate(target + '.*?\.tgz$') if not any(pattern in x for pattern in ignore_patterns)] @@ -202,3 +215,269 @@ class SStateTests(SStateBase): global_config.append('MACHINE = "qemux86"') target_config.append('') self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) + + @testcase(1270) + def test_sstate_32_64_same_hash(self): + """ + The sstate checksums for both native and target should not vary whether + they're built on a 32 or 64 bit system. Rather than requiring two different + build machines and running a builds, override the variables calling uname() + manually and check using bitbake -S. + """ + + self.write_config(""" +MACHINE = "qemux86" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash" +BUILD_ARCH = "x86_64" +BUILD_OS = "linux" +SDKMACHINE = "x86_64" +PACKAGE_CLASSES = "package_rpm package_ipk package_deb" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("core-image-sato -S none") + self.write_config(""" +MACHINE = "qemux86" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" +BUILD_ARCH = "i686" +BUILD_OS = "linux" +SDKMACHINE = "i686" +PACKAGE_CLASSES = "package_rpm package_ipk package_deb" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("core-image-sato -S none") + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + if "core-image-sato" in root: + # SDKMACHINE changing will change + # do_rootfs/do_testimage/do_build stamps of images which + # is safe to ignore. + continue + f.extend(os.path.join(root, name) for name in files) + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + self.target_vendor + "-linux", "x86_64" + self.target_vendor + "-linux", ) for x in files2] + self.maxDiff = None + self.assertCountEqual(files1, files2) + + + @testcase(1271) + def test_sstate_nativelsbstring_same_hash(self): + """ + The sstate checksums should be independent of whichever NATIVELSBSTRING is + detected. Rather than requiring two different build machines and running + builds, override the variables manually and check using bitbake -S. + """ + + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +NATIVELSBSTRING = \"DistroA\" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("core-image-sato -S none") + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +NATIVELSBSTRING = \"DistroB\" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("core-image-sato -S none") + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + f.extend(os.path.join(root, name) for name in files) + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None + self.assertCountEqual(files1, files2) + + @testcase(1368) + def test_sstate_allarch_samesigs(self): + """ + The sstate checksums of allarch packages should be independent of whichever + MACHINE is set. Check this using bitbake -S. + Also, rather than duplicate the test, check nativesdk stamps are the same between + the two MACHINE values. + """ + + configA = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86-64\" +""" + configB = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemuarm\" +""" + self.sstate_allarch_samesigs(configA, configB) + + def test_sstate_allarch_samesigs_multilib(self): + """ + The sstate checksums of allarch multilib packages should be independent of whichever + MACHINE is set. Check this using bitbake -S. + Also, rather than duplicate the test, check nativesdk stamps are the same between + the two MACHINE values. + """ + + configA = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86-64\" +require conf/multilib.conf +MULTILIBS = \"multilib:lib32\" +DEFAULTTUNE_virtclass-multilib-lib32 = \"x86\" +""" + configB = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemuarm\" +require conf/multilib.conf +MULTILIBS = \"\" +""" + self.sstate_allarch_samesigs(configA, configB) + + def sstate_allarch_samesigs(self, configA, configB): + + self.write_config(configA) + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("world meta-toolchain -S none") + self.write_config(configB) + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("world meta-toolchain -S none") + + def get_files(d): + f = {} + for root, dirs, files in os.walk(d): + for name in files: + if "meta-environment" in root or "cross-canadian" in root: + continue + if "do_build" not in name: + # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 + (_, task, _, shash) = name.rsplit(".", 3) + f[os.path.join(os.path.basename(root), task)] = shash + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/all" + self.target_vendor + "-" + self.target_os) + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/all" + self.target_vendor + "-" + self.target_os) + self.maxDiff = None + self.assertEqual(files1, files2) + + nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) + + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) + self.maxDiff = None + self.assertEqual(files1, files2) + + @testcase(1369) + def test_sstate_sametune_samesigs(self): + """ + The sstate checksums of two identical machines (using the same tune) should be the + same, apart from changes within the machine specific stamps directory. We use the + qemux86copy machine to test this. Also include multilibs in the test. + """ + + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE_virtclass-multilib-lib32 = "x86" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("world meta-toolchain -S none") + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemux86copy\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE_virtclass-multilib-lib32 = "x86" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("world meta-toolchain -S none") + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + for name in files: + if "meta-environment" in root or "cross-canadian" in root: + continue + if "qemux86copy-" in root or "qemux86-" in root: + continue + if "do_build" not in name and "do_populate_sdk" not in name: + f.append(os.path.join(root, name)) + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None + self.assertCountEqual(files1, files2) + + + def test_sstate_noop_samesigs(self): + """ + The sstate checksums of two builds with these variables changed or + classes inherits should be the same. + """ + + self.write_config(""" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash" +BB_NUMBER_THREADS = "1" +PARALLEL_MAKE = "-j 1" +DL_DIR = "${TOPDIR}/download1" +TIME = "111111" +DATE = "20161111" +INHERIT_remove = "buildstats-summary buildhistory uninative" +http_proxy = "" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + self.track_for_cleanup(self.topdir + "/download1") + bitbake("world meta-toolchain -S none") + self.write_config(""" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" +BB_NUMBER_THREADS = "2" +PARALLEL_MAKE = "-j 2" +DL_DIR = "${TOPDIR}/download2" +TIME = "222222" +DATE = "20161212" +# Always remove uninative as we're changing proxies +INHERIT_remove = "uninative" +INHERIT += "buildstats-summary buildhistory" +http_proxy = "http://example.com/" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + self.track_for_cleanup(self.topdir + "/download2") + bitbake("world meta-toolchain -S none") + + def get_files(d): + f = {} + for root, dirs, files in os.walk(d): + for name in files: + name, shash = name.rsplit('.', 1) + # Extract just the machine and recipe name + base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name]) + f[base] = shash + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") + # Remove items that are identical in both sets + for k,v in files1.items() & files2.items(): + del files1[k] + del files2[k] + if not files1 and not files2: + # No changes, so we're done + return + + for k in files1.keys() | files2.keys(): + if k in files1 and k in files2: + print("%s differs:" % k) + print(subprocess.check_output(("bitbake-diffsigs", + self.topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k], + self.topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k]))) + elif k in files1 and k not in files2: + print("%s in files1" % k) + elif k not in files1 and k in files2: + print("%s in files2" % k) + else: + assert "shouldn't reach here" + self.fail("sstate hashes not identical.") diff --git a/meta/lib/oeqa/selftest/tinfoil.py b/meta/lib/oeqa/selftest/tinfoil.py new file mode 100644 index 0000000000..73a0c3bac0 --- /dev/null +++ b/meta/lib/oeqa/selftest/tinfoil.py @@ -0,0 +1,190 @@ +import unittest +import os +import re +import bb.tinfoil + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd +from oeqa.utils.decorators import testcase + +class TinfoilTests(oeSelfTest): + """ Basic tests for the tinfoil API """ + + @testcase(1568) + def test_getvar(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(True) + machine = tinfoil.config_data.getVar('MACHINE') + if not machine: + self.fail('Unable to get MACHINE value - returned %s' % machine) + + @testcase(1569) + def test_expand(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(True) + expr = '${@os.getpid()}' + pid = tinfoil.config_data.expand(expr) + if not pid: + self.fail('Unable to expand "%s" - returned %s' % (expr, pid)) + + @testcase(1570) + def test_getvar_bb_origenv(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(True) + origenv = tinfoil.config_data.getVar('BB_ORIGENV', False) + if not origenv: + self.fail('Unable to get BB_ORIGENV value - returned %s' % origenv) + self.assertEqual(origenv.getVar('HOME', False), os.environ['HOME']) + + @testcase(1571) + def test_parse_recipe(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3]) + self.assertEqual(testrecipe, rd.getVar('PN')) + + @testcase(1572) + def test_parse_recipe_copy_expand(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3]) + # Check we can get variable values + self.assertEqual(testrecipe, rd.getVar('PN')) + # Check that expanding a value that includes a variable reference works + self.assertEqual(testrecipe, rd.getVar('BPN')) + # Now check that changing the referenced variable's value in a copy gives that + # value when expanding + localdata = bb.data.createCopy(rd) + localdata.setVar('PN', 'hello') + self.assertEqual('hello', localdata.getVar('BPN')) + + @testcase(1573) + def test_parse_recipe_initial_datastore(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + dcopy = bb.data.createCopy(tinfoil.config_data) + dcopy.setVar('MYVARIABLE', 'somevalue') + rd = tinfoil.parse_recipe_file(best[3], config_data=dcopy) + # Check we can get variable values + self.assertEqual('somevalue', rd.getVar('MYVARIABLE')) + + @testcase(1574) + def test_list_recipes(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + # Check pkg_pn + checkpns = ['tar', 'automake', 'coreutils', 'm4-native', 'nativesdk-gcc'] + pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn + for pn in checkpns: + self.assertIn(pn, pkg_pn) + # Check pkg_fn + checkfns = {'nativesdk-gcc': '^virtual:nativesdk:.*', 'coreutils': '.*/coreutils_.*.bb'} + for fn, pn in tinfoil.cooker.recipecaches[''].pkg_fn.items(): + if pn in checkpns: + if pn in checkfns: + self.assertTrue(re.match(checkfns[pn], fn), 'Entry for %s: %s did not match %s' % (pn, fn, checkfns[pn])) + checkpns.remove(pn) + if checkpns: + self.fail('Unable to find pkg_fn entries for: %s' % ', '.join(checkpns)) + + @testcase(1575) + def test_wait_event(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + # Need to drain events otherwise events that will be masked will still be in the queue + while tinfoil.wait_event(0.25): + pass + tinfoil.set_event_mask(['bb.event.FilesMatchingFound', 'bb.command.CommandCompleted']) + pattern = 'conf' + res = tinfoil.run_command('findFilesMatchingInDir', pattern, 'conf/machine') + self.assertTrue(res) + + eventreceived = False + waitcount = 5 + while waitcount > 0: + event = tinfoil.wait_event(1) + if event: + if isinstance(event, bb.command.CommandCompleted): + break + elif isinstance(event, bb.event.FilesMatchingFound): + self.assertEqual(pattern, event._pattern) + self.assertIn('qemuarm.conf', event._matches) + eventreceived = True + else: + self.fail('Unexpected event: %s' % event) + + waitcount = waitcount - 1 + + self.assertNotEqual(waitcount, 0, 'Timed out waiting for CommandCompleted event from bitbake server') + self.assertTrue(eventreceived, 'Did not receive FilesMatchingFound event from bitbake server') + + @testcase(1576) + def test_setvariable_clean(self): + # First check that setVariable affects the datastore + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + tinfoil.run_command('setVariable', 'TESTVAR', 'specialvalue') + self.assertEqual(tinfoil.config_data.getVar('TESTVAR'), 'specialvalue', 'Value set using setVariable is not reflected in client-side getVar()') + + # Now check that the setVariable's effects are no longer present + # (this may legitimately break in future if we stop reinitialising + # the datastore, in which case we'll have to reconsider use of + # setVariable entirely) + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + self.assertNotEqual(tinfoil.config_data.getVar('TESTVAR'), 'specialvalue', 'Value set using setVariable is still present!') + + # Now check that setVar on the main datastore works (uses setVariable internally) + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + tinfoil.config_data.setVar('TESTVAR', 'specialvalue') + value = tinfoil.run_command('getVariable', 'TESTVAR') + self.assertEqual(value, 'specialvalue', 'Value set using config_data.setVar() is not reflected in config_data.getVar()') + + def test_datastore_operations(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + # Test setVarFlag() / getVarFlag() + tinfoil.config_data.setVarFlag('TESTVAR', 'flagname', 'flagval') + value = tinfoil.config_data.getVarFlag('TESTVAR', 'flagname') + self.assertEqual(value, 'flagval', 'Value set using config_data.setVarFlag() is not reflected in config_data.getVarFlag()') + # Test delVarFlag() + tinfoil.config_data.setVarFlag('TESTVAR', 'otherflag', 'othervalue') + tinfoil.config_data.delVarFlag('TESTVAR', 'flagname') + value = tinfoil.config_data.getVarFlag('TESTVAR', 'flagname') + self.assertEqual(value, None, 'Varflag deleted using config_data.delVarFlag() is not reflected in config_data.getVarFlag()') + value = tinfoil.config_data.getVarFlag('TESTVAR', 'otherflag') + self.assertEqual(value, 'othervalue', 'Varflag deleted using config_data.delVarFlag() caused unrelated flag to be removed') + # Test delVar() + tinfoil.config_data.setVar('TESTVAR', 'varvalue') + value = tinfoil.config_data.getVar('TESTVAR') + self.assertEqual(value, 'varvalue', 'Value set using config_data.setVar() is not reflected in config_data.getVar()') + tinfoil.config_data.delVar('TESTVAR') + value = tinfoil.config_data.getVar('TESTVAR') + self.assertEqual(value, None, 'Variable deleted using config_data.delVar() appears to still have a value') + # Test renameVar() + tinfoil.config_data.setVar('TESTVAROLD', 'origvalue') + tinfoil.config_data.renameVar('TESTVAROLD', 'TESTVARNEW') + value = tinfoil.config_data.getVar('TESTVAROLD') + self.assertEqual(value, None, 'Variable renamed using config_data.renameVar() still seems to exist') + value = tinfoil.config_data.getVar('TESTVARNEW') + self.assertEqual(value, 'origvalue', 'Variable renamed using config_data.renameVar() does not appear with new name') + # Test overrides + tinfoil.config_data.setVar('TESTVAR', 'original') + tinfoil.config_data.setVar('TESTVAR_overrideone', 'one') + tinfoil.config_data.setVar('TESTVAR_overridetwo', 'two') + tinfoil.config_data.appendVar('OVERRIDES', ':overrideone') + value = tinfoil.config_data.getVar('TESTVAR') + self.assertEqual(value, 'one', 'Variable overrides not functioning correctly') diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py new file mode 100644 index 0000000000..726af19e9d --- /dev/null +++ b/meta/lib/oeqa/selftest/wic.py @@ -0,0 +1,792 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2015, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# AUTHORS +# Ed Bartosh <ed.bartosh@linux.intel.com> + +"""Test cases for wic.""" + +import os +import sys +import unittest + +from glob import glob +from shutil import rmtree +from functools import wraps, lru_cache +from tempfile import NamedTemporaryFile + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu +from oeqa.utils.decorators import testcase + + +@lru_cache(maxsize=32) +def get_host_arch(recipe): + """A cached call to get_bb_var('HOST_ARCH', <recipe>)""" + return get_bb_var('HOST_ARCH', recipe) + + +def only_for_arch(archs, image='core-image-minimal'): + """Decorator for wrapping test cases that can be run only for specific target + architectures. A list of compatible architectures is passed in `archs`. + Current architecture will be determined by parsing bitbake output for + `image` recipe. + """ + def wrapper(func): + @wraps(func) + def wrapped_f(*args, **kwargs): + arch = get_host_arch(image) + if archs and arch not in archs: + raise unittest.SkipTest("Testcase arch dependency not met: %s" % arch) + return func(*args, **kwargs) + wrapped_f.__name__ = func.__name__ + return wrapped_f + return wrapper + + +class Wic(oeSelfTest): + """Wic test class.""" + + resultdir = "/var/tmp/wic.oe-selftest/" + image_is_ready = False + native_sysroot = None + wicenv_cache = {} + + def setUpLocal(self): + """This code is executed before each test method.""" + if not self.native_sysroot: + Wic.native_sysroot = get_bb_var('STAGING_DIR_NATIVE', 'wic-tools') + + # Do this here instead of in setUpClass as the base setUp does some + # clean up which can result in the native tools built earlier in + # setUpClass being unavailable. + if not Wic.image_is_ready: + if get_bb_var('USE_NLS') == 'yes': + bitbake('wic-tools') + else: + self.skipTest('wic-tools cannot be built due its (intltool|gettext)-native dependency and NLS disable') + + bitbake('core-image-minimal') + Wic.image_is_ready = True + + rmtree(self.resultdir, ignore_errors=True) + + def tearDownLocal(self): + """Remove resultdir as it may contain images.""" + rmtree(self.resultdir, ignore_errors=True) + + @testcase(1552) + def test_version(self): + """Test wic --version""" + self.assertEqual(0, runCmd('wic --version').status) + + @testcase(1208) + def test_help(self): + """Test wic --help and wic -h""" + self.assertEqual(0, runCmd('wic --help').status) + self.assertEqual(0, runCmd('wic -h').status) + + @testcase(1209) + def test_createhelp(self): + """Test wic create --help""" + self.assertEqual(0, runCmd('wic create --help').status) + + @testcase(1210) + def test_listhelp(self): + """Test wic list --help""" + self.assertEqual(0, runCmd('wic list --help').status) + + @testcase(1553) + def test_help_create(self): + """Test wic help create""" + self.assertEqual(0, runCmd('wic help create').status) + + @testcase(1554) + def test_help_list(self): + """Test wic help list""" + self.assertEqual(0, runCmd('wic help list').status) + + @testcase(1215) + def test_help_overview(self): + """Test wic help overview""" + self.assertEqual(0, runCmd('wic help overview').status) + + @testcase(1216) + def test_help_plugins(self): + """Test wic help plugins""" + self.assertEqual(0, runCmd('wic help plugins').status) + + @testcase(1217) + def test_help_kickstart(self): + """Test wic help kickstart""" + self.assertEqual(0, runCmd('wic help kickstart').status) + + @testcase(1555) + def test_list_images(self): + """Test wic list images""" + self.assertEqual(0, runCmd('wic list images').status) + + @testcase(1556) + def test_list_source_plugins(self): + """Test wic list source-plugins""" + self.assertEqual(0, runCmd('wic list source-plugins').status) + + @testcase(1557) + def test_listed_images_help(self): + """Test wic listed images help""" + output = runCmd('wic list images').output + imagelist = [line.split()[0] for line in output.splitlines()] + for image in imagelist: + self.assertEqual(0, runCmd('wic list %s help' % image).status) + + @testcase(1213) + def test_unsupported_subcommand(self): + """Test unsupported subcommand""" + self.assertEqual(1, runCmd('wic unsupported', + ignore_status=True).status) + + @testcase(1214) + def test_no_command(self): + """Test wic without command""" + self.assertEqual(1, runCmd('wic', ignore_status=True).status) + + @testcase(1211) + def test_build_image_name(self): + """Test wic create wictestdisk --image-name=core-image-minimal""" + cmd = "wic create wictestdisk --image-name=core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1157) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_gpt_image(self): + """Test creation of core-image-minimal with gpt table and UUID boot""" + cmd = "wic create directdisk-gpt --image-name core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) + + @testcase(1346) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_iso_image(self): + """Test creation of hybrid iso image with legacy and EFI boot""" + config = 'INITRAMFS_IMAGE = "core-image-minimal-initramfs"\n'\ + 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) + cmd = "wic create mkhybridiso --image-name core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.direct"))) + self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.iso"))) + + @testcase(1348) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_qemux86_directdisk(self): + """Test creation of qemux-86-directdisk image""" + cmd = "wic create qemux86-directdisk -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "qemux86-directdisk-*direct"))) + + @testcase(1350) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_mkefidisk(self): + """Test creation of mkefidisk image""" + cmd = "wic create mkefidisk -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "mkefidisk-*direct"))) + + @testcase(1385) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_bootloader_config(self): + """Test creation of directdisk-bootloader-config image""" + cmd = "wic create directdisk-bootloader-config -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-bootloader-config-*direct"))) + + @testcase(1560) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_systemd_bootdisk(self): + """Test creation of systemd-bootdisk image""" + config = 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) + cmd = "wic create systemd-bootdisk -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "systemd-bootdisk-*direct"))) + + @testcase(1561) + def test_sdimage_bootpart(self): + """Test creation of sdimage-bootpart image""" + cmd = "wic create sdimage-bootpart -e core-image-minimal -o %s" % self.resultdir + kimgtype = get_bb_var('KERNEL_IMAGETYPE', 'core-image-minimal') + self.write_config('IMAGE_BOOT_FILES = "%s"\n' % kimgtype) + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "sdimage-bootpart-*direct"))) + + @testcase(1562) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_default_output_dir(self): + """Test default output location""" + for fname in glob("directdisk-*.direct"): + os.remove(fname) + cmd = "wic create directdisk -e core-image-minimal" + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob("directdisk-*.direct"))) + + @testcase(1212) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_build_artifacts(self): + """Test wic create directdisk providing all artifacts.""" + bb_vars = get_bb_vars(['STAGING_DATADIR', 'RECIPE_SYSROOT_NATIVE'], + 'wic-tools') + bb_vars.update(get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_ROOTFS'], + 'core-image-minimal')) + bbvars = {key.lower(): value for key, value in bb_vars.items()} + bbvars['resultdir'] = self.resultdir + status = runCmd("wic create directdisk " + "-b %(staging_datadir)s " + "-k %(deploy_dir_image)s " + "-n %(recipe_sysroot_native)s " + "-r %(image_rootfs)s " + "-o %(resultdir)s" % bbvars).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) + + @testcase(1264) + def test_compress_gzip(self): + """Test compressing an image with gzip""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name core-image-minimal " + "-c gzip -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct.gz"))) + + @testcase(1265) + def test_compress_bzip2(self): + """Test compressing an image with bzip2""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-c bzip2 -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct.bz2"))) + + @testcase(1266) + def test_compress_xz(self): + """Test compressing an image with xz""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--compress-with=xz -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct.xz"))) + + @testcase(1267) + def test_wrong_compressor(self): + """Test how wic breaks if wrong compressor is provided""" + self.assertEqual(2, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-c wrong -o %s" % self.resultdir, + ignore_status=True).status) + + @testcase(1558) + def test_debug_short(self): + """Test -D option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-D -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + def test_debug_long(self): + """Test --debug option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--debug -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1563) + def test_skip_build_check_short(self): + """Test -s option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-s -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + def test_skip_build_check_long(self): + """Test --skip-build-check option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--skip-build-check " + "--outdir %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1564) + def test_build_rootfs_short(self): + """Test -f option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-f -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + def test_build_rootfs_long(self): + """Test --build-rootfs option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--build-rootfs " + "--outdir %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1268) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_rootfs_indirect_recipes(self): + """Test usage of rootfs plugin with rootfs recipes""" + status = runCmd("wic create directdisk-multi-rootfs " + "--image-name=core-image-minimal " + "--rootfs rootfs1=core-image-minimal " + "--rootfs rootfs2=core-image-minimal " + "--outdir %s" % self.resultdir).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-multi-rootfs*.direct"))) + + @testcase(1269) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_rootfs_artifacts(self): + """Test usage of rootfs plugin with rootfs paths""" + bb_vars = get_bb_vars(['STAGING_DATADIR', 'RECIPE_SYSROOT_NATIVE'], + 'wic-tools') + bb_vars.update(get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_ROOTFS'], + 'core-image-minimal')) + bbvars = {key.lower(): value for key, value in bb_vars.items()} + bbvars['wks'] = "directdisk-multi-rootfs" + bbvars['resultdir'] = self.resultdir + status = runCmd("wic create %(wks)s " + "--bootimg-dir=%(staging_datadir)s " + "--kernel-dir=%(deploy_dir_image)s " + "--native-sysroot=%(recipe_sysroot_native)s " + "--rootfs-dir rootfs1=%(image_rootfs)s " + "--rootfs-dir rootfs2=%(image_rootfs)s " + "--outdir %(resultdir)s" % bbvars).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "%(wks)s-*.direct" % bbvars))) + + def test_exclude_path(self): + """Test --exclude-path wks option.""" + + oldpath = os.environ['PATH'] + os.environ['PATH'] = get_bb_var("PATH", "wic-tools") + + try: + wks_file = 'temp.wks' + with open(wks_file, 'w') as wks: + rootfs_dir = get_bb_var('IMAGE_ROOTFS', 'core-image-minimal') + wks.write(""" +part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path usr +part /usr --source rootfs --ondisk mmcblk0 --fstype=ext4 --rootfs-dir %s/usr +part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --rootfs-dir %s/usr""" + % (rootfs_dir, rootfs_dir)) + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir)).status) + + os.remove(wks_file) + wicout = glob(self.resultdir + "%s-*direct" % 'temp') + self.assertEqual(1, len(wicout)) + + wicimg = wicout[0] + + # verify partition size with wic + res = runCmd("parted -m %s unit b p 2>/dev/null" % wicimg) + self.assertEqual(0, res.status) + + # parse parted output which looks like this: + # BYT;\n + # /var/tmp/wic/build/tmpfwvjjkf_-201611101222-hda.direct:200MiB:file:512:512:msdos::;\n + # 1:0.00MiB:200MiB:200MiB:ext4::;\n + partlns = res.output.splitlines()[2:] + + self.assertEqual(3, len(partlns)) + + for part in [1, 2, 3]: + part_file = os.path.join(self.resultdir, "selftest_img.part%d" % part) + partln = partlns[part-1].split(":") + self.assertEqual(7, len(partln)) + start = int(partln[1].rstrip("B")) / 512 + length = int(partln[3].rstrip("B")) / 512 + self.assertEqual(0, runCmd("dd if=%s of=%s skip=%d count=%d" % + (wicimg, part_file, start, length)).status) + + def extract_files(debugfs_output): + """ + extract file names from the output of debugfs -R 'ls -p', + which looks like this: + + /2/040755/0/0/.//\n + /2/040755/0/0/..//\n + /11/040700/0/0/lost+found^M//\n + /12/040755/1002/1002/run//\n + /13/040755/1002/1002/sys//\n + /14/040755/1002/1002/bin//\n + /80/040755/1002/1002/var//\n + /92/040755/1002/1002/tmp//\n + """ + # NOTE the occasional ^M in file names + return [line.split('/')[5].strip() for line in \ + debugfs_output.strip().split('/\n')] + + # Test partition 1, should contain the normal root directories, except + # /usr. + res = runCmd("debugfs -R 'ls -p' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part1")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertIn("etc", files) + self.assertNotIn("usr", files) + + # Partition 2, should contain common directories for /usr, not root + # directories. + res = runCmd("debugfs -R 'ls -p' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part2")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertNotIn("etc", files) + self.assertNotIn("usr", files) + self.assertIn("share", files) + + # Partition 3, should contain the same as partition 2, including the bin + # directory, but not the files inside it. + res = runCmd("debugfs -R 'ls -p' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part3")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertNotIn("etc", files) + self.assertNotIn("usr", files) + self.assertIn("share", files) + self.assertIn("bin", files) + res = runCmd("debugfs -R 'ls -p bin' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part3")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertIn(".", files) + self.assertIn("..", files) + self.assertEqual(2, len(files)) + + for part in [1, 2, 3]: + part_file = os.path.join(self.resultdir, "selftest_img.part%d" % part) + os.remove(part_file) + + finally: + os.environ['PATH'] = oldpath + + def test_exclude_path_errors(self): + """Test --exclude-path wks option error handling.""" + wks_file = 'temp.wks' + + # Absolute argument. + with open(wks_file, 'w') as wks: + wks.write("part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path /usr") + self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir), ignore_status=True).status) + os.remove(wks_file) + + # Argument pointing to parent directory. + with open(wks_file, 'w') as wks: + wks.write("part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path ././..") + self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir), ignore_status=True).status) + os.remove(wks_file) + + @testcase(1496) + def test_bmap_short(self): + """Test generation of .bmap file -m option""" + cmd = "wic create wictestdisk -e core-image-minimal -m -o %s" % self.resultdir + status = runCmd(cmd).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct.bmap"))) + + def test_bmap_long(self): + """Test generation of .bmap file --bmap option""" + cmd = "wic create wictestdisk -e core-image-minimal --bmap -o %s" % self.resultdir + status = runCmd(cmd).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct.bmap"))) + + def _get_image_env_path(self, image): + """Generate and obtain the path to <image>.env""" + if image not in self.wicenv_cache: + self.assertEqual(0, bitbake('%s -c do_rootfs_wicenv' % image).status) + bb_vars = get_bb_vars(['STAGING_DIR', 'MACHINE'], image) + stdir = bb_vars['STAGING_DIR'] + machine = bb_vars['MACHINE'] + self.wicenv_cache[image] = os.path.join(stdir, machine, 'imgdata') + return self.wicenv_cache[image] + + @testcase(1347) + def test_image_env(self): + """Test generation of <image>.env files.""" + image = 'core-image-minimal' + imgdatadir = self._get_image_env_path(image) + + bb_vars = get_bb_vars(['IMAGE_BASENAME', 'WICVARS'], image) + basename = bb_vars['IMAGE_BASENAME'] + self.assertEqual(basename, image) + path = os.path.join(imgdatadir, basename) + '.env' + self.assertTrue(os.path.isfile(path)) + + wicvars = set(bb_vars['WICVARS'].split()) + # filter out optional variables + wicvars = wicvars.difference(('DEPLOY_DIR_IMAGE', 'IMAGE_BOOT_FILES', + 'INITRD', 'INITRD_LIVE', 'ISODIR')) + with open(path) as envfile: + content = dict(line.split("=", 1) for line in envfile) + # test if variables used by wic present in the .env file + for var in wicvars: + self.assertTrue(var in content, "%s is not in .env file" % var) + self.assertTrue(content[var]) + + @testcase(1559) + def test_image_vars_dir_short(self): + """Test image vars directory selection -v option""" + image = 'core-image-minimal' + imgenvdir = self._get_image_env_path(image) + + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=%s -v %s -o %s" + % (image, imgenvdir, self.resultdir)).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + + def test_image_vars_dir_long(self): + """Test image vars directory selection --vars option""" + image = 'core-image-minimal' + imgenvdir = self._get_image_env_path(image) + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=%s " + "--vars %s " + "--outdir %s" + % (image, imgenvdir, self.resultdir)).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + + @testcase(1351) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_wic_image_type(self): + """Test building wic images by bitbake""" + config = 'IMAGE_FSTYPES += "wic"\nWKS_FILE = "wic-image-minimal"\n'\ + 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) + self.assertEqual(0, bitbake('wic-image-minimal').status) + self.remove_config(config) + + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'MACHINE']) + deploy_dir = bb_vars['DEPLOY_DIR_IMAGE'] + machine = bb_vars['MACHINE'] + prefix = os.path.join(deploy_dir, 'wic-image-minimal-%s.' % machine) + # check if we have result image and manifests symlinks + # pointing to existing files + for suffix in ('wic', 'manifest'): + path = prefix + suffix + self.assertTrue(os.path.islink(path)) + self.assertTrue(os.path.isfile(os.path.realpath(path))) + + @testcase(1422) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_qemu(self): + """Test wic-image-minimal under qemu""" + config = 'IMAGE_FSTYPES += "wic"\nWKS_FILE = "wic-image-minimal"\n'\ + 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) + self.assertEqual(0, bitbake('wic-image-minimal').status) + self.remove_config(config) + + with runqemu('wic-image-minimal', ssh=False) as qemu: + cmd = "mount |grep '^/dev/' | cut -f1,3 -d ' '" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '/dev/root /\r\n/dev/sda3 /mnt') + + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_qemu_efi(self): + """Test core-image-minimal efi image under qemu""" + config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "mkefidisk.wks"\n' + self.append_config(config) + self.assertEqual(0, bitbake('core-image-minimal ovmf').status) + self.remove_config(config) + + with runqemu('core-image-minimal', ssh=False, + runqemuparams='ovmf', image_fstype='wic') as qemu: + cmd = "grep sda. /proc/partitions |wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '3') + + @staticmethod + def _make_fixed_size_wks(size): + """ + Create a wks of an image with a single partition. Size of the partition is set + using --fixed-size flag. Returns a tuple: (path to wks file, wks image name) + """ + with NamedTemporaryFile("w", suffix=".wks", delete=False) as tempf: + wkspath = tempf.name + tempf.write("part " \ + "--source rootfs --ondisk hda --align 4 --fixed-size %d " + "--fstype=ext4\n" % size) + wksname = os.path.splitext(os.path.basename(wkspath))[0] + + return wkspath, wksname + + def test_fixed_size(self): + """ + Test creation of a simple image with partition size controlled through + --fixed-size flag + """ + wkspath, wksname = Wic._make_fixed_size_wks(200) + + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wkspath, self.resultdir)).status) + os.remove(wkspath) + wicout = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(wicout)) + + wicimg = wicout[0] + + # verify partition size with wic + res = runCmd("parted -m %s unit mib p 2>/dev/null" % wicimg, + ignore_status=True, + native_sysroot=self.native_sysroot) + self.assertEqual(0, res.status) + + # parse parted output which looks like this: + # BYT;\n + # /var/tmp/wic/build/tmpfwvjjkf_-201611101222-hda.direct:200MiB:file:512:512:msdos::;\n + # 1:0.00MiB:200MiB:200MiB:ext4::;\n + partlns = res.output.splitlines()[2:] + + self.assertEqual(1, len(partlns)) + self.assertEqual("1:0.00MiB:200MiB:200MiB:ext4::;", partlns[0]) + + def test_fixed_size_error(self): + """ + Test creation of a simple image with partition size controlled through + --fixed-size flag. The size of partition is intentionally set to 1MiB + in order to trigger an error in wic. + """ + wkspath, wksname = Wic._make_fixed_size_wks(1) + + self.assertEqual(1, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wkspath, self.resultdir), ignore_status=True).status) + os.remove(wkspath) + wicout = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(0, len(wicout)) + + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_rawcopy_plugin_qemu(self): + """Test rawcopy plugin in qemu""" + # build ext4 and wic images + for fstype in ("ext4", "wic"): + config = 'IMAGE_FSTYPES = "%s"\nWKS_FILE = "test_rawcopy_plugin.wks.in"\n' % fstype + self.append_config(config) + self.assertEqual(0, bitbake('core-image-minimal').status) + self.remove_config(config) + + with runqemu('core-image-minimal', ssh=False, image_fstype='wic') as qemu: + cmd = "grep sda. /proc/partitions |wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '2') + + def test_rawcopy_plugin(self): + """Test rawcopy plugin""" + img = 'core-image-minimal' + machine = get_bb_var('MACHINE', img) + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part /boot --active --source bootimg-pcbios\n', + 'part / --source rawcopy --sourceparams="file=%s-%s.ext4" --use-uuid\n'\ + % (img, machine), + 'bootloader --timeout=0 --append="console=ttyS0,115200n8"\n']) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + self.assertEqual(0, runCmd(cmd).status) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(out)) + + def test_fs_types(self): + """Test filesystem types for empty and not empty partitions""" + img = 'core-image-minimal' + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part ext2 --fstype ext2 --source rootfs\n', + 'part btrfs --fstype btrfs --source rootfs --size 40M\n', + 'part squash --fstype squashfs --source rootfs\n', + 'part swap --fstype swap --size 1M\n', + 'part emptyvfat --fstype vfat --size 1M\n', + 'part emptymsdos --fstype msdos --size 1M\n', + 'part emptyext2 --fstype ext2 --size 1M\n', + 'part emptybtrfs --fstype btrfs --size 100M\n']) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + self.assertEqual(0, runCmd(cmd).status) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(out)) + + def test_kickstart_parser(self): + """Test wks parser options""" + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part / --fstype ext3 --source rootfs --system-id 0xFF '\ + '--overhead-factor 1.2 --size 100k\n']) + wks.flush() + cmd = "wic create %s -e core-image-minimal -o %s" % (wks.name, self.resultdir) + self.assertEqual(0, runCmd(cmd).status) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(out)) + + def test_image_bootpart_globbed(self): + """Test globbed sources with image-bootpart plugin""" + img = "core-image-minimal" + cmd = "wic create sdimage-bootpart -e %s -o %s" % (img, self.resultdir) + config = 'IMAGE_BOOT_FILES = "%s*"' % get_bb_var('KERNEL_IMAGETYPE', img) + self.append_config(config) + self.assertEqual(0, runCmd(cmd).status) + self.remove_config(config) + self.assertEqual(1, len(glob(self.resultdir + "sdimage-bootpart-*direct"))) + + def test_sparse_copy(self): + """Test sparse_copy with FIEMAP and SEEK_HOLE filemap APIs""" + libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'wic') + sys.path.insert(0, libpath) + from filemap import FilemapFiemap, FilemapSeek, sparse_copy, ErrorNotSupp + with NamedTemporaryFile("w", suffix=".wic-sparse") as sparse: + src_name = sparse.name + src_size = 1024 * 10 + sparse.truncate(src_size) + # write one byte to the file + with open(src_name, 'r+b') as sfile: + sfile.seek(1024 * 4) + sfile.write(b'\x00') + dest = sparse.name + '.out' + # copy src file to dest using different filemap APIs + for api in (FilemapFiemap, FilemapSeek, None): + if os.path.exists(dest): + os.unlink(dest) + try: + sparse_copy(sparse.name, dest, api=api) + except ErrorNotSupp: + continue # skip unsupported API + dest_stat = os.stat(dest) + self.assertEqual(dest_stat.st_size, src_size) + # 8 blocks is 4K (physical sector size) + self.assertEqual(dest_stat.st_blocks, 8) + os.unlink(dest) diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py index cc582dd1ad..3255e3a5c6 100644 --- a/meta/lib/oeqa/targetcontrol.py +++ b/meta/lib/oeqa/targetcontrol.py @@ -10,13 +10,18 @@ import subprocess import bb import traceback import sys +import logging from oeqa.utils.sshcontrol import SSHControl from oeqa.utils.qemurunner import QemuRunner +from oeqa.utils.qemutinyrunner import QemuTinyRunner +from oeqa.utils.dump import TargetDumper from oeqa.controllers.testtargetloader import TestTargetLoader from abc import ABCMeta, abstractmethod +logger = logging.getLogger('BitBake.QemuRunner') + def get_target_controller(d): - testtarget = d.getVar("TEST_TARGET", True) + testtarget = d.getVar("TEST_TARGET") # old, simple names if testtarget == "qemu": return QemuTarget(d) @@ -30,7 +35,7 @@ def get_target_controller(d): except AttributeError: # nope, perhaps a layer defined one try: - bbpath = d.getVar("BBPATH", True).split(':') + bbpath = d.getVar("BBPATH").split(':') testtargetloader = TestTargetLoader() controller = testtargetloader.get_controller_module(testtarget, bbpath) except ImportError as e: @@ -40,9 +45,7 @@ def get_target_controller(d): return controller(d) -class BaseTarget(object): - - __metaclass__ = ABCMeta +class BaseTarget(object, metaclass=ABCMeta): supported_image_fstypes = [] @@ -50,9 +53,9 @@ class BaseTarget(object): self.connection = None self.ip = None self.server_ip = None - self.datetime = d.getVar('DATETIME', True) - self.testdir = d.getVar("TEST_LOG_DIR", True) - self.pn = d.getVar("PN", True) + self.datetime = d.getVar('DATETIME') + self.testdir = d.getVar("TEST_LOG_DIR") + self.pn = d.getVar("PN") @abstractmethod def deploy(self): @@ -62,10 +65,10 @@ class BaseTarget(object): if os.path.islink(sshloglink): os.unlink(sshloglink) os.symlink(self.sshlog, sshloglink) - bb.note("SSH log file: %s" % self.sshlog) + logger.info("SSH log file: %s" % self.sshlog) @abstractmethod - def start(self, params=None): + def start(self, params=None, ssh=True, extra_bootparams=None): pass @abstractmethod @@ -79,7 +82,7 @@ class BaseTarget(object): @classmethod def match_image_fstype(self, d, image_fstypes=None): if not image_fstypes: - image_fstypes = d.getVar('IMAGE_FSTYPES', True).split(' ') + image_fstypes = d.getVar('IMAGE_FSTYPES').split(' ') possible_image_fstypes = [fstype for fstype in self.supported_image_fstypes if fstype in image_fstypes] if possible_image_fstypes: return possible_image_fstypes[0] @@ -110,49 +113,97 @@ class BaseTarget(object): class QemuTarget(BaseTarget): - supported_image_fstypes = ['ext3'] + supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] - def __init__(self, d): + def __init__(self, d, image_fstype=None): super(QemuTarget, self).__init__(d) - self.image_fstype = self.get_image_fstype(d) - self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) - self.origrootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) - self.rootfs = os.path.join(self.testdir, d.getVar("IMAGE_LINK_NAME", True) + '-testimage.' + self.image_fstype) + self.rootfs = '' + self.kernel = '' + self.image_fstype = '' - self.runner = QemuRunner(machine=d.getVar("MACHINE", True), - rootfs=self.rootfs, - tmpdir = d.getVar("TMPDIR", True), - deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True), - display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True), - logfile = self.qemulog, - boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True))) + if d.getVar('FIND_ROOTFS') == '1': + self.image_fstype = image_fstype or self.get_image_fstype(d) + self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("IMAGE_LINK_NAME") + '.' + self.image_fstype) + self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') + self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) + dump_target_cmds = d.getVar("testimage_dump_target") + dump_host_cmds = d.getVar("testimage_dump_host") + dump_dir = d.getVar("TESTIMAGE_DUMP_DIR") + qemu_use_kvm = d.getVar("QEMU_USE_KVM") + if qemu_use_kvm and \ + (qemu_use_kvm == "True" and "x86" in d.getVar("MACHINE") or \ + d.getVar("MACHINE") in qemu_use_kvm.split()): + use_kvm = True + else: + use_kvm = False + + # Log QemuRunner log output to a file + import oe.path + bb.utils.mkdirhier(self.testdir) + self.qemurunnerlog = os.path.join(self.testdir, 'qemurunner_log.%s' % self.datetime) + loggerhandler = logging.FileHandler(self.qemurunnerlog) + loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(loggerhandler) + oe.path.symlink(os.path.basename(self.qemurunnerlog), os.path.join(self.testdir, 'qemurunner_log'), force=True) + + if d.getVar("DISTRO") == "poky-tiny": + self.runner = QemuTinyRunner(machine=d.getVar("MACHINE"), + rootfs=self.rootfs, + tmpdir = d.getVar("TMPDIR"), + deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE"), + display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY"), + logfile = self.qemulog, + kernel = self.kernel, + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT"))) + else: + self.runner = QemuRunner(machine=d.getVar("MACHINE"), + rootfs=self.rootfs, + tmpdir = d.getVar("TMPDIR"), + deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE"), + display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY"), + logfile = self.qemulog, + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")), + use_kvm = use_kvm, + dump_dir = dump_dir, + dump_host_cmds = d.getVar("testimage_dump_host")) + + self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) def deploy(self): - try: - shutil.copyfile(self.origrootfs, self.rootfs) - except Exception as e: - bb.fatal("Error copying rootfs: %s" % e) + bb.utils.mkdirhier(self.testdir) qemuloglink = os.path.join(self.testdir, "qemu_boot_log") if os.path.islink(qemuloglink): os.unlink(qemuloglink) os.symlink(self.qemulog, qemuloglink) - bb.note("rootfs file: %s" % self.rootfs) - bb.note("Qemu log file: %s" % self.qemulog) + logger.info("rootfs file: %s" % self.rootfs) + logger.info("Qemu log file: %s" % self.qemulog) super(QemuTarget, self).deploy() - def start(self, params=None): - if self.runner.start(params): - self.ip = self.runner.ip - self.server_ip = self.runner.server_ip - self.connection = SSHControl(ip=self.ip, logfile=self.sshlog) + def start(self, params=None, ssh=True, extra_bootparams='', runqemuparams='', launch_cmd='', discard_writes=True): + if launch_cmd: + start = self.runner.launch(get_ip=ssh, launch_cmd=launch_cmd) + else: + start = self.runner.start(params, get_ip=ssh, extra_bootparams=extra_bootparams, runqemuparams=runqemuparams, discard_writes=discard_writes) + + if start: + if ssh: + self.ip = self.runner.ip + self.server_ip = self.runner.server_ip + self.connection = SSHControl(ip=self.ip, logfile=self.sshlog) else: self.stop() + if os.path.exists(self.qemulog): + with open(self.qemulog, 'r') as f: + bb.error("Qemu log output from %s:\n%s" % (self.qemulog, f.read())) raise bb.build.FuncFailed("%s - FAILED to start qemu - check the task log and the boot log" % self.pn) + def check(self): + return self.runner.is_alive() + def stop(self): self.runner.stop() self.connection = None @@ -167,31 +218,35 @@ class QemuTarget(BaseTarget): else: raise bb.build.FuncFailed("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn) + def run_serial(self, command, timeout=5): + return self.runner.run_serial(command, timeout=timeout) + class SimpleRemoteTarget(BaseTarget): def __init__(self, d): super(SimpleRemoteTarget, self).__init__(d) - addr = d.getVar("TEST_TARGET_IP", True) or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') + addr = d.getVar("TEST_TARGET_IP") or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') self.ip = addr.split(":")[0] try: self.port = addr.split(":")[1] except IndexError: self.port = None - bb.note("Target IP: %s" % self.ip) - self.server_ip = d.getVar("TEST_SERVER_IP", True) + logger.info("Target IP: %s" % self.ip) + self.server_ip = d.getVar("TEST_SERVER_IP") if not self.server_ip: try: self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split("\n")[0].split()[-1] except Exception as e: bb.fatal("Failed to determine the host IP address (alternatively you can set TEST_SERVER_IP with the IP address of this machine): %s" % e) - bb.note("Server IP: %s" % self.server_ip) + logger.info("Server IP: %s" % self.server_ip) def deploy(self): super(SimpleRemoteTarget, self).deploy() - def start(self, params=None): - self.connection = SSHControl(self.ip, logfile=self.sshlog, port=self.port) + def start(self, params=None, ssh=True, extra_bootparams=None): + if ssh: + self.connection = SSHControl(self.ip, logfile=self.sshlog, port=self.port) def stop(self): self.connection = None diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py index 2260046026..485de031a9 100644 --- a/meta/lib/oeqa/utils/__init__.py +++ b/meta/lib/oeqa/utils/__init__.py @@ -13,3 +13,56 @@ class CommandError(Exception): def __str__(self): return "Command '%s' returned non-zero exit status %d with output: %s" % (self.cmd, self.retcode, self.output) +def avoid_paths_in_environ(paths): + """ + Searches for every path in os.environ['PATH'] + if found remove it. + + Returns new PATH without avoided PATHs. + """ + import os + + new_path = '' + for p in os.environ['PATH'].split(':'): + avoid = False + for pa in paths: + if pa in p: + avoid = True + break + if avoid: + continue + + new_path = new_path + p + ':' + + new_path = new_path[:-1] + return new_path + +def make_logger_bitbake_compatible(logger): + import logging + + """ + Bitbake logger redifines debug() in order to + set a level within debug, this breaks compatibility + with vainilla logging, so we neeed to redifine debug() + method again also add info() method with INFO + 1 level. + """ + def _bitbake_log_debug(*args, **kwargs): + lvl = logging.DEBUG + + if isinstance(args[0], int): + lvl = args[0] + msg = args[1] + args = args[2:] + else: + msg = args[0] + args = args[1:] + + logger.log(lvl, msg, *args, **kwargs) + + def _bitbake_log_info(msg, *args, **kwargs): + logger.log(logging.INFO + 1, msg, *args, **kwargs) + + logger.debug = _bitbake_log_debug + logger.info = _bitbake_log_info + + return logger diff --git a/meta/lib/oeqa/utils/buildproject.py b/meta/lib/oeqa/utils/buildproject.py new file mode 100644 index 0000000000..487f08be49 --- /dev/null +++ b/meta/lib/oeqa/utils/buildproject.py @@ -0,0 +1,55 @@ +# Copyright (C) 2013-2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Provides a class for automating build tests for projects + +import os +import re +import subprocess +import shutil +import tempfile + +from abc import ABCMeta, abstractmethod + +class BuildProject(metaclass=ABCMeta): + def __init__(self, uri, foldername=None, tmpdir=None, dl_dir=None): + self.uri = uri + self.archive = os.path.basename(uri) + if not tmpdir: + tmpdir = tempfile.mkdtemp(prefix='buildproject') + self.localarchive = os.path.join(tmpdir, self.archive) + self.dl_dir = dl_dir + if foldername: + self.fname = foldername + else: + self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive) + + # Download self.archive to self.localarchive + def _download_archive(self): + if self.dl_dir and os.path.exists(os.path.join(self.dl_dir, self.archive)): + shutil.copyfile(os.path.join(self.dl_dir, self.archive), self.localarchive) + return + + cmd = "wget -O %s %s" % (self.localarchive, self.uri) + subprocess.check_output(cmd, shell=True) + + # This method should provide a way to run a command in the desired environment. + @abstractmethod + def _run(self, cmd): + pass + + # The timeout parameter of target.run is set to 0 to make the ssh command + # run with no timeout. + def run_configure(self, configure_args='', extra_cmds=''): + return self._run('cd %s; gnu-configize; %s ./configure %s' % (self.targetdir, extra_cmds, configure_args)) + + def run_make(self, make_args=''): + return self._run('cd %s; make %s' % (self.targetdir, make_args)) + + def run_install(self, install_args=''): + return self._run('cd %s; make install %s' % (self.targetdir, install_args)) + + def clean(self): + self._run('rm -rf %s' % self.targetdir) + subprocess.call('rm -f %s' % self.localarchive, shell=True) diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 802bc2f208..57286fcb10 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -16,6 +16,13 @@ import threading import logging from oeqa.utils import CommandError from oeqa.utils import ftools +import re +import contextlib +# Export test doesn't require bb +try: + import bb +except ImportError: + pass class Command(object): def __init__(self, command, bg=False, timeout=None, data=None, **options): @@ -34,7 +41,7 @@ class Command(object): self.data = data self.options = dict(self.defaultopts) - if isinstance(self.cmd, basestring): + if isinstance(self.cmd, str): self.options["shell"] = True if self.data: self.options['stdin'] = subprocess.PIPE @@ -71,7 +78,10 @@ class Command(object): self.process.kill() self.thread.join() - self.output = self.output.rstrip() + if not self.output: + self.output = "" + else: + self.output = self.output.decode("utf-8", errors='replace').rstrip() self.status = self.process.poll() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) @@ -87,22 +97,37 @@ class Result(object): pass -def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **options): +def runCmd(command, ignore_status=False, timeout=None, assert_error=True, + native_sysroot=None, limit_exc_output=0, **options): result = Result() + if native_sysroot: + extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ + (native_sysroot, native_sysroot, native_sysroot) + nenv = dict(options.get('env', os.environ)) + nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') + options['env'] = nenv + cmd = Command(command, timeout=timeout, **options) cmd.run() result.command = command result.status = cmd.status result.output = cmd.output + result.error = cmd.error result.pid = cmd.process.pid if result.status and not ignore_status: + exc_output = result.output + if limit_exc_output > 0: + split = result.output.splitlines() + if len(split) > limit_exc_output: + exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \ + '\n'.join(split[-limit_exc_output:]) if assert_error: - raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output)) + raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output)) else: - raise CommandError(result.status, command, result.output) + raise CommandError(result.status, command, exc_output) return result @@ -110,19 +135,19 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **opti def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **options): if postconfig: - postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') - ftools.write_file(postconfig_file, postconfig) - extra_args = "-R %s" % postconfig_file + postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') + ftools.write_file(postconfig_file, postconfig) + extra_args = "-R %s" % postconfig_file else: - extra_args = "" + extra_args = "" - if isinstance(command, basestring): + if isinstance(command, str): cmd = "bitbake " + extra_args + " " + command else: cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] try: - return runCmd(cmd, ignore_status, timeout, **options) + return runCmd(cmd, ignore_status, timeout, **options) finally: if postconfig: os.remove(postconfig_file) @@ -134,21 +159,145 @@ def get_bb_env(target=None, postconfig=None): else: return bitbake("-e", postconfig=postconfig).output -def get_bb_var(var, target=None, postconfig=None): - val = None +def get_bb_vars(variables=None, target=None, postconfig=None): + """Get values of multiple bitbake variables""" bbenv = get_bb_env(target, postconfig=postconfig) + + if variables is not None: + variables = variables.copy() + var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$') + unset_re = re.compile(r'^unset (?P<var>\w+)$') + lastline = None + values = {} for line in bbenv.splitlines(): - if line.startswith(var + "="): - val = line.split('=')[1] - val = val.replace('\"','') - break - return val + match = var_re.match(line) + val = None + if match: + val = match.group('value') + else: + match = unset_re.match(line) + if match: + # Handle [unexport] variables + if lastline.startswith('# "'): + val = lastline.split('"')[1] + if val: + var = match.group('var') + if variables is None: + values[var] = val + else: + if var in variables: + values[var] = val + variables.remove(var) + # Stop after all required variables have been found + if not variables: + break + lastline = line + if variables: + # Fill in missing values + for var in variables: + values[var] = None + return values + +def get_bb_var(var, target=None, postconfig=None): + return get_bb_vars([var], target, postconfig)[var] def get_test_layer(): layers = get_bb_var("BBLAYERS").split() testlayer = None for l in layers: + if '~' in l: + l = os.path.expanduser(l) if "/meta-selftest" in l and os.path.isdir(l): testlayer = l break return testlayer + +def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'): + os.makedirs(os.path.join(templayerdir, 'conf')) + with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f: + f.write('BBPATH .= ":${LAYERDIR}"\n') + f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec) + f.write(' ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec) + f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername) + f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername) + f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority)) + f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername) + + +@contextlib.contextmanager +def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True): + """ + launch_cmd means directly run the command, don't need set rootfs or env vars. + """ + + import bb.tinfoil + import bb.build + + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(config_only=False, quiet=True) + try: + tinfoil.logger.setLevel(logging.WARNING) + import oeqa.targetcontrol + tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") + tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") + # Tell QemuTarget() whether need find rootfs/kernel or not + if launch_cmd: + tinfoil.config_data.setVar("FIND_ROOTFS", '0') + else: + tinfoil.config_data.setVar("FIND_ROOTFS", '1') + + recipedata = tinfoil.parse_recipe(pn) + for key, value in overrides.items(): + recipedata.setVar(key, value) + + # The QemuRunner log is saved out, but we need to ensure it is at the right + # log level (and then ensure that since it's a child of the BitBake logger, + # we disable propagation so we don't then see the log events on the console) + logger = logging.getLogger('BitBake.QemuRunner') + logger.setLevel(logging.DEBUG) + logger.propagate = False + logdir = recipedata.getVar("TEST_LOG_DIR") + + qemu = oeqa.targetcontrol.QemuTarget(recipedata, image_fstype) + finally: + # We need to shut down tinfoil early here in case we actually want + # to run tinfoil-using utilities with the running QEMU instance. + # Luckily QemuTarget doesn't need it after the constructor. + tinfoil.shutdown() + + # Setup bitbake logger as console handler is removed by tinfoil.shutdown + bblogger = logging.getLogger('BitBake') + bblogger.setLevel(logging.INFO) + console = logging.StreamHandler(sys.stdout) + bbformat = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + if sys.stdout.isatty(): + bbformat.enable_color() + console.setFormatter(bbformat) + bblogger.addHandler(console) + + try: + qemu.deploy() + try: + qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) + except bb.build.FuncFailed: + raise Exception('Failed to start QEMU - see the logs in %s' % logdir) + + yield qemu + + finally: + try: + qemu.stop() + except: + pass + +def updateEnv(env_file): + """ + Source a file and update environment. + """ + + cmd = ". %s; env -0" % env_file + result = runCmd(cmd) + + for line in result.output.split("\0"): + (key, _, value) = line.partition("=") + os.environ[key] = value diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index a9e67ed863..d876896921 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -10,22 +10,37 @@ import os import logging import sys import unittest +import threading +import signal +from functools import wraps #get the "result" object from one of the upper frames provided that one of these upper frames is a unittest.case frame class getResults(object): def __init__(self): #dynamically determine the unittest.case frame and use it to get the name of the test method - upperf = sys._current_frames().values()[0] + ident = threading.current_thread().ident + upperf = sys._current_frames()[ident] while (upperf.f_globals['__name__'] != 'unittest.case'): upperf = upperf.f_back - self.faillist = [ seq[0]._testMethodName for seq in upperf.f_locals['result'].failures ] - self.errorlist = [ seq[0]._testMethodName for seq in upperf.f_locals['result'].errors ] - #ignore the _ErrorHolder objects from the skipped tests list - self.skiplist = [] - for seq in upperf.f_locals['result'].skipped: - try: - self.skiplist.append(seq[0]._testMethodName) - except: pass + + def handleList(items): + ret = [] + # items is a list of tuples, (test, failure) or (_ErrorHandler(), Exception()) + for i in items: + s = i[0].id() + #Handle the _ErrorHolder objects from skipModule failures + if "setUpModule (" in s: + ret.append(s.replace("setUpModule (", "").replace(")","")) + else: + ret.append(s) + # Append also the test without the full path + testname = s.split('.')[-1] + if testname: + ret.append(testname) + return ret + self.faillist = handleList(upperf.f_locals['result'].failures) + self.errorlist = handleList(upperf.f_locals['result'].errors) + self.skiplist = handleList(upperf.f_locals['result'].skipped) def getFailList(self): return self.faillist @@ -42,11 +57,12 @@ class skipIfFailure(object): self.testcase = testcase def __call__(self,f): - def wrapped_f(*args): + @wraps(f) + def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in (res.getFailList() or res.getErrorList()): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args) + return f(*args, **kwargs) wrapped_f.__name__ = f.__name__ return wrapped_f @@ -56,11 +72,12 @@ class skipIfSkipped(object): self.testcase = testcase def __call__(self,f): - def wrapped_f(*args): + @wraps(f) + def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in res.getSkipList(): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args) + return f(*args, **kwargs) wrapped_f.__name__ = f.__name__ return wrapped_f @@ -70,76 +87,209 @@ class skipUnlessPassed(object): self.testcase = testcase def __call__(self,f): - def wrapped_f(*args): + @wraps(f) + def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in res.getSkipList() or \ self.testcase in res.getFailList() or \ self.testcase in res.getErrorList(): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args) + return f(*args, **kwargs) wrapped_f.__name__ = f.__name__ + wrapped_f._depends_on = self.testcase return wrapped_f class testcase(object): - def __init__(self, test_case): self.test_case = test_case def __call__(self, func): - def wrapped_f(*args): - return func(*args) - wrapped_f.test_case = self.test_case - return wrapped_f + @wraps(func) + def wrapped_f(*args, **kwargs): + return func(*args, **kwargs) + wrapped_f.test_case = self.test_case + wrapped_f.__name__ = func.__name__ + return wrapped_f + +class NoParsingFilter(logging.Filter): + def filter(self, record): + return record.levelno == 100 + +import inspect def LogResults(original_class): orig_method = original_class.run + from time import strftime, gmtime + caller = os.path.basename(sys.argv[0]) + timestamp = strftime('%Y%m%d%H%M%S',gmtime()) + logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log') + linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log') + + def get_class_that_defined_method(meth): + if inspect.ismethod(meth): + for cls in inspect.getmro(meth.__self__.__class__): + if cls.__dict__.get(meth.__name__) is meth: + return cls + meth = meth.__func__ # fallback to __qualname__ parsing + if inspect.isfunction(meth): + cls = getattr(inspect.getmodule(meth), + meth.__qualname__.split('.<locals>', 1)[0].rsplit('.', 1)[0]) + if isinstance(cls, type): + return cls + return None + #rewrite the run method of unittest.TestCase to add testcase logging def run(self, result, *args, **kws): orig_method(self, result, *args, **kws) - passed = True - testMethod = getattr(self, self._testMethodName) - - #if test case is decorated then use it's number, else use it's name - try: - test_case = testMethod.test_case - except AttributeError: - test_case = self._testMethodName - - #create custom logging level for filtering. - custom_log_level = 100 - logging.addLevelName(custom_log_level, 'RESULTS') - caller = os.path.basename(sys.argv[0]) - - def results(self, message, *args, **kws): - if self.isEnabledFor(custom_log_level): - self.log(custom_log_level, message, *args, **kws) - logging.Logger.results = results - - logging.basicConfig(filename=os.path.join(os.getcwd(),'results-'+caller+'.log'), + passed = True + testMethod = getattr(self, self._testMethodName) + #if test case is decorated then use it's number, else use it's name + try: + test_case = testMethod.test_case + except AttributeError: + test_case = self._testMethodName + + class_name = str(get_class_that_defined_method(testMethod)).split("'")[1] + + #create custom logging level for filtering. + custom_log_level = 100 + logging.addLevelName(custom_log_level, 'RESULTS') + + def results(self, message, *args, **kws): + if self.isEnabledFor(custom_log_level): + self.log(custom_log_level, message, *args, **kws) + logging.Logger.results = results + + logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S', level=custom_log_level) - local_log = logging.getLogger(caller) + for handler in logging.root.handlers: + handler.addFilter(NoParsingFilter()) + local_log = logging.getLogger(caller) - #check status of tests and record it + #check status of tests and record it + + tcid = self.id() for (name, msg) in result.errors: - if self._testMethodName == str(name).split(' ')[0]: - local_log.results("Testcase "+str(test_case)+": ERROR") - local_log.results("Testcase "+str(test_case)+":\n"+msg) - passed = False + if tcid == name.id(): + local_log.results("Testcase "+str(test_case)+": ERROR") + local_log.results("Testcase "+str(test_case)+":\n"+msg) + passed = False for (name, msg) in result.failures: - if self._testMethodName == str(name).split(' ')[0]: - local_log.results("Testcase "+str(test_case)+": FAILED") - local_log.results("Testcase "+str(test_case)+":\n"+msg) - passed = False + if tcid == name.id(): + local_log.results("Testcase "+str(test_case)+": FAILED") + local_log.results("Testcase "+str(test_case)+":\n"+msg) + passed = False for (name, msg) in result.skipped: - if self._testMethodName == str(name).split(' ')[0]: - local_log.results("Testcase "+str(test_case)+": SKIPPED") - passed = False - if passed: - local_log.results("Testcase "+str(test_case)+": PASSED") + if tcid == name.id(): + local_log.results("Testcase "+str(test_case)+": SKIPPED") + passed = False + if passed: + local_log.results("Testcase "+str(test_case)+": PASSED") + + # XXX: In order to avoid race condition when test if exists the linkfile + # use bb.utils.lock, the best solution is to create a unique name for the + # link file. + try: + import bb + has_bb = True + lockfilename = linkfile + '.lock' + except ImportError: + has_bb = False + + if has_bb: + lf = bb.utils.lockfile(lockfilename, block=True) + # Create symlink to the current log + if os.path.lexists(linkfile): + os.remove(linkfile) + os.symlink(logfile, linkfile) + if has_bb: + bb.utils.unlockfile(lf) original_class.run = run + return original_class + +class TimeOut(BaseException): + pass + +def timeout(seconds): + def decorator(fn): + if hasattr(signal, 'alarm'): + @wraps(fn) + def wrapped_f(*args, **kw): + current_frame = sys._getframe() + def raiseTimeOut(signal, frame): + if frame is not current_frame: + raise TimeOut('%s seconds' % seconds) + prev_handler = signal.signal(signal.SIGALRM, raiseTimeOut) + try: + signal.alarm(seconds) + return fn(*args, **kw) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, prev_handler) + return wrapped_f + else: + return fn + return decorator + +__tag_prefix = "tag__" +def tag(*args, **kwargs): + """Decorator that adds attributes to classes or functions + for use with the Attribute (-a) plugin. + """ + def wrap_ob(ob): + for name in args: + setattr(ob, __tag_prefix + name, True) + for name, value in kwargs.items(): + setattr(ob, __tag_prefix + name, value) + return ob + return wrap_ob + +def gettag(obj, key, default=None): + key = __tag_prefix + key + if not isinstance(obj, unittest.TestCase): + return getattr(obj, key, default) + tc_method = getattr(obj, obj._testMethodName) + ret = getattr(tc_method, key, getattr(obj, key, default)) + return ret + +def getAllTags(obj): + def __gettags(o): + r = {k[len(__tag_prefix):]:getattr(o,k) for k in dir(o) if k.startswith(__tag_prefix)} + return r + if not isinstance(obj, unittest.TestCase): + return __gettags(obj) + tc_method = getattr(obj, obj._testMethodName) + ret = __gettags(obj) + ret.update(__gettags(tc_method)) + return ret + +def timeout_handler(seconds): + def decorator(fn): + if hasattr(signal, 'alarm'): + @wraps(fn) + def wrapped_f(self, *args, **kw): + current_frame = sys._getframe() + def raiseTimeOut(signal, frame): + if frame is not current_frame: + try: + self.target.restart() + raise TimeOut('%s seconds' % seconds) + except: + raise TimeOut('%s seconds' % seconds) + prev_handler = signal.signal(signal.SIGALRM, raiseTimeOut) + try: + signal.alarm(seconds) + return fn(self, *args, **kw) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, prev_handler) + return wrapped_f + else: + return fn + return decorator diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py new file mode 100644 index 0000000000..5a7edc1a86 --- /dev/null +++ b/meta/lib/oeqa/utils/dump.py @@ -0,0 +1,91 @@ +import os +import sys +import errno +import datetime +import itertools +from .commands import runCmd + +class BaseDumper(object): + """ Base class to dump commands from host/target """ + + def __init__(self, cmds, parent_dir): + self.cmds = [] + # Some testing doesn't inherit testimage, so it is needed + # to set some defaults. + self.parent_dir = parent_dir or "/tmp/oe-saved-tests" + dft_cmds = """ top -bn1 + iostat -x -z -N -d -p ALL 20 2 + ps -ef + free + df + memstat + dmesg + ip -s link + netstat -an""" + if not cmds: + cmds = dft_cmds + for cmd in cmds.split('\n'): + cmd = cmd.lstrip() + if not cmd or cmd[0] == '#': + continue + self.cmds.append(cmd) + + def create_dir(self, dir_suffix): + dump_subdir = ("%s_%s" % ( + datetime.datetime.now().strftime('%Y%m%d%H%M'), + dir_suffix)) + dump_dir = os.path.join(self.parent_dir, dump_subdir) + try: + os.makedirs(dump_dir) + except OSError as err: + if err.errno != errno.EEXIST: + raise err + self.dump_dir = dump_dir + + def _write_dump(self, command, output): + if isinstance(self, HostDumper): + prefix = "host" + elif isinstance(self, TargetDumper): + prefix = "target" + else: + prefix = "unknown" + for i in itertools.count(): + filename = "%s_%02d_%s" % (prefix, i, command) + fullname = os.path.join(self.dump_dir, filename) + if not os.path.exists(fullname): + break + with open(fullname, 'w') as dump_file: + dump_file.write(output) + + +class HostDumper(BaseDumper): + """ Class to get dumps from the host running the tests """ + + def __init__(self, cmds, parent_dir): + super(HostDumper, self).__init__(cmds, parent_dir) + + def dump_host(self, dump_dir=""): + if dump_dir: + self.dump_dir = dump_dir + for cmd in self.cmds: + result = runCmd(cmd, ignore_status=True) + self._write_dump(cmd.split()[0], result.output) + +class TargetDumper(BaseDumper): + """ Class to get dumps from target, it only works with QemuRunner """ + + def __init__(self, cmds, parent_dir, runner): + super(TargetDumper, self).__init__(cmds, parent_dir) + self.runner = runner + + def dump_target(self, dump_dir=""): + if dump_dir: + self.dump_dir = dump_dir + for cmd in self.cmds: + # We can continue with the testing if serial commands fail + try: + (status, output) = self.runner.run_serial(cmd) + self._write_dump(cmd.split()[0], output) + except: + print("Tried to dump info from target but " + "serial console failed") diff --git a/meta/lib/oeqa/utils/ftools.py b/meta/lib/oeqa/utils/ftools.py index 64ebe3d217..a7233d4ca6 100644 --- a/meta/lib/oeqa/utils/ftools.py +++ b/meta/lib/oeqa/utils/ftools.py @@ -1,12 +1,19 @@ import os import re +import errno def write_file(path, data): + # In case data is None, return immediately + if data is None: + return wdata = data.rstrip() + "\n" with open(path, "w") as f: f.write(wdata) def append_file(path, data): + # In case data is None, return immediately + if data is None: + return wdata = data.rstrip() + "\n" with open(path, "a") as f: f.write(wdata) @@ -18,10 +25,22 @@ def read_file(path): return data def remove_from_file(path, data): - lines = read_file(path).splitlines() - rmdata = data.strip().splitlines() - for l in rmdata: - for c in range(0, lines.count(l)): - i = lines.index(l) - del(lines[i]) - write_file(path, "\n".join(lines)) + # In case data is None, return immediately + if data is None: + return + try: + rdata = read_file(path) + except IOError as e: + # if file does not exit, just quit, otherwise raise an exception + if e.errno == errno.ENOENT: + return + else: + raise + + contents = rdata.strip().splitlines() + for r in data.strip().splitlines(): + try: + contents.remove(r) + except ValueError: + pass + write_file(path, "\n".join(contents)) diff --git a/meta/lib/oeqa/utils/git.py b/meta/lib/oeqa/utils/git.py new file mode 100644 index 0000000000..e0cb3f0db2 --- /dev/null +++ b/meta/lib/oeqa/utils/git.py @@ -0,0 +1,80 @@ +# +# Copyright (C) 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +"""Git repository interactions""" +import os + +from oeqa.utils.commands import runCmd + + +class GitError(Exception): + """Git error handling""" + pass + +class GitRepo(object): + """Class representing a Git repository clone""" + def __init__(self, path, is_topdir=False): + git_dir = self._run_git_cmd_at(['rev-parse', '--git-dir'], path) + git_dir = git_dir if os.path.isabs(git_dir) else os.path.join(path, git_dir) + self.git_dir = os.path.realpath(git_dir) + + if self._run_git_cmd_at(['rev-parse', '--is-bare-repository'], path) == 'true': + self.bare = True + self.top_dir = self.git_dir + else: + self.bare = False + self.top_dir = self._run_git_cmd_at(['rev-parse', '--show-toplevel'], + path) + realpath = os.path.realpath(path) + if is_topdir and realpath != self.top_dir: + raise GitError("{} is not a Git top directory".format(realpath)) + + @staticmethod + def _run_git_cmd_at(git_args, cwd, **kwargs): + """Run git command at a specified directory""" + git_cmd = 'git ' if isinstance(git_args, str) else ['git'] + git_cmd += git_args + ret = runCmd(git_cmd, ignore_status=True, cwd=cwd, **kwargs) + if ret.status: + cmd_str = git_cmd if isinstance(git_cmd, str) \ + else ' '.join(git_cmd) + raise GitError("'{}' failed with exit code {}: {}".format( + cmd_str, ret.status, ret.output)) + return ret.output.strip() + + @staticmethod + def init(path, bare=False): + """Initialize a new Git repository""" + cmd = ['init'] + if bare: + cmd.append('--bare') + GitRepo._run_git_cmd_at(cmd, cwd=path) + return GitRepo(path, is_topdir=True) + + def run_cmd(self, git_args, env_update=None): + """Run Git command""" + env = None + if env_update: + env = os.environ.copy() + env.update(env_update) + return self._run_git_cmd_at(git_args, self.top_dir, env=env) + + def rev_parse(self, revision): + """Do git rev-parse""" + try: + return self.run_cmd(['rev-parse', revision]) + except GitError: + # Revision does not exist + return None + + def get_current_branch(self): + """Get current branch""" + try: + # Strip 11 chars, i.e. 'refs/heads' from the beginning + return self.run_cmd(['symbolic-ref', 'HEAD'])[11:] + except GitError: + return None + + diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index f161a1bddd..7d12331453 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -1,14 +1,17 @@ -import SimpleHTTPServer +import http.server import multiprocessing import os +from socketserver import ThreadingMixIn -class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): +class HTTPServer(ThreadingMixIn, http.server.HTTPServer): def server_start(self, root_dir): + import signal + signal.signal(signal.SIGTERM, signal.SIG_DFL) os.chdir(root_dir) self.serve_forever() -class HTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format_str, *args): pass diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py new file mode 100644 index 0000000000..b377dcd271 --- /dev/null +++ b/meta/lib/oeqa/utils/logparser.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python + +import sys +import os +import re +from . import ftools + + +# A parser that can be used to identify weather a line is a test result or a section statement. +class Lparser(object): + + def __init__(self, test_0_pass_regex, test_0_fail_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs): + # Initialize the arguments dictionary + if kwargs: + self.args = kwargs + else: + self.args = {} + + # Add the default args to the dictionary + self.args['test_0_pass_regex'] = test_0_pass_regex + self.args['test_0_fail_regex'] = test_0_fail_regex + if section_0_begin_regex: + self.args['section_0_begin_regex'] = section_0_begin_regex + if section_0_end_regex: + self.args['section_0_end_regex'] = section_0_end_regex + + self.test_possible_status = ['pass', 'fail', 'error'] + self.section_possible_status = ['begin', 'end'] + + self.initialized = False + + + # Initialize the parser with the current configuration + def init(self): + + # extra arguments can be added by the user to define new test and section categories. They must follow a pre-defined pattern: <type>_<category_name>_<status>_regex + self.test_argument_pattern = "^test_(.+?)_(%s)_regex" % '|'.join(map(str, self.test_possible_status)) + self.section_argument_pattern = "^section_(.+?)_(%s)_regex" % '|'.join(map(str, self.section_possible_status)) + + # Initialize the test and section regex dictionaries + self.test_regex = {} + self.section_regex ={} + + for arg, value in self.args.items(): + if not value: + raise Exception('The value of provided argument %s is %s. Should have a valid value.' % (key, value)) + is_test = re.search(self.test_argument_pattern, arg) + is_section = re.search(self.section_argument_pattern, arg) + if is_test: + if not is_test.group(1) in self.test_regex: + self.test_regex[is_test.group(1)] = {} + self.test_regex[is_test.group(1)][is_test.group(2)] = re.compile(value) + elif is_section: + if not is_section.group(1) in self.section_regex: + self.section_regex[is_section.group(1)] = {} + self.section_regex[is_section.group(1)][is_section.group(2)] = re.compile(value) + else: + # TODO: Make these call a traceback instead of a simple exception.. + raise Exception("The provided argument name does not correspond to any valid type. Please give one of the following types:\nfor tests: %s\nfor sections: %s" % (self.test_argument_pattern, self.section_argument_pattern)) + + self.initialized = True + + # Parse a line and return a tuple containing the type of result (test/section) and its category, status and name + def parse_line(self, line): + if not self.initialized: + raise Exception("The parser is not initialized..") + + for test_category, test_status_list in self.test_regex.items(): + for test_status, status_regex in test_status_list.items(): + test_name = status_regex.search(line) + if test_name: + return ['test', test_category, test_status, test_name.group(1)] + + for section_category, section_status_list in self.section_regex.items(): + for section_status, status_regex in section_status_list.items(): + section_name = status_regex.search(line) + if section_name: + return ['section', section_category, section_status, section_name.group(1)] + return None + + +class Result(object): + + def __init__(self): + self.result_dict = {} + + def store(self, section, test, status): + if not section in self.result_dict: + self.result_dict[section] = [] + + self.result_dict[section].append((test, status)) + + # sort tests by the test name(the first element of the tuple), for each section. This can be helpful when using git to diff for changes by making sure they are always in the same order. + def sort_tests(self): + for package in self.result_dict: + sorted_results = sorted(self.result_dict[package], key=lambda tup: tup[0]) + self.result_dict[package] = sorted_results + + # Log the results as files. The file name is the section name and the contents are the tests in that section. + def log_as_files(self, target_dir, test_status): + status_regex = re.compile('|'.join(map(str, test_status))) + if not type(test_status) == type([]): + raise Exception("test_status should be a list. Got " + str(test_status) + " instead.") + if not os.path.exists(target_dir): + raise Exception("Target directory does not exist: %s" % target_dir) + + for section, test_results in self.result_dict.items(): + prefix = '' + for x in test_status: + prefix +=x+'.' + if (section != ''): + prefix += section + section_file = os.path.join(target_dir, prefix) + # purge the file contents if it exists + open(section_file, 'w').close() + for test_result in test_results: + (test_name, status) = test_result + # we log only the tests with status in the test_status list + match_status = status_regex.search(status) + if match_status: + ftools.append_file(section_file, status + ": " + test_name) + + # Not yet implemented! + def log_to_lava(self): + pass diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py new file mode 100644 index 0000000000..cb81155e54 --- /dev/null +++ b/meta/lib/oeqa/utils/metadata.py @@ -0,0 +1,118 @@ +# Copyright (C) 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +# Functions to get metadata from the testing host used +# for analytics of test results. + +from collections import OrderedDict +from collections.abc import MutableMapping +from xml.dom.minidom import parseString +from xml.etree.ElementTree import Element, tostring + +from oeqa.utils.commands import runCmd, get_bb_vars + +def get_os_release(): + """Get info from /etc/os-release as a dict""" + data = OrderedDict() + os_release_file = '/etc/os-release' + if not os.path.exists(os_release_file): + return None + with open(os_release_file) as fobj: + for line in fobj: + key, value = line.split('=', 1) + data[key.strip().lower()] = value.strip().strip('"') + return data + +def metadata_from_bb(): + """ Returns test's metadata as OrderedDict. + + Data will be gathered using bitbake -e thanks to get_bb_vars. + """ + metadata_config_vars = ('MACHINE', 'BB_NUMBER_THREADS', 'PARALLEL_MAKE') + + info_dict = OrderedDict() + hostname = runCmd('hostname') + info_dict['hostname'] = hostname.output + data_dict = get_bb_vars() + + # Distro information + info_dict['distro'] = {'id': data_dict['DISTRO'], + 'version_id': data_dict['DISTRO_VERSION'], + 'pretty_name': '%s %s' % (data_dict['DISTRO'], data_dict['DISTRO_VERSION'])} + + # Host distro information + os_release = get_os_release() + if os_release: + info_dict['host_distro'] = OrderedDict() + for key in ('id', 'version_id', 'pretty_name'): + if key in os_release: + info_dict['host_distro'][key] = os_release[key] + + info_dict['layers'] = get_layers(data_dict['BBLAYERS']) + info_dict['bitbake'] = git_rev_info(os.path.dirname(bb.__file__)) + + info_dict['config'] = OrderedDict() + for var in sorted(metadata_config_vars): + info_dict['config'][var] = data_dict[var] + return info_dict + +def metadata_from_data_store(d): + """ Returns test's metadata as OrderedDict. + + Data will be collected from the provided data store. + """ + # TODO: Getting metadata from the data store would + # be useful when running within bitbake. + pass + +def git_rev_info(path): + """Get git revision information as a dict""" + from git import Repo, InvalidGitRepositoryError, NoSuchPathError + + info = OrderedDict() + try: + repo = Repo(path, search_parent_directories=True) + except (InvalidGitRepositoryError, NoSuchPathError): + return info + info['commit'] = repo.head.commit.hexsha + info['commit_count'] = repo.head.commit.count() + try: + info['branch'] = repo.active_branch.name + except TypeError: + info['branch'] = '(nobranch)' + return info + +def get_layers(layers): + """Returns layer information in dict format""" + layer_dict = OrderedDict() + for layer in layers.split(): + layer_name = os.path.basename(layer) + layer_dict[layer_name] = git_rev_info(layer) + return layer_dict + +def write_metadata_file(file_path, metadata): + """ Writes metadata to a XML file in directory. """ + + xml = dict_to_XML('metadata', metadata) + xml_doc = parseString(tostring(xml).decode('UTF-8')) + with open(file_path, 'w') as f: + f.write(xml_doc.toprettyxml()) + +def dict_to_XML(tag, dictionary, **kwargs): + """ Return XML element converting dicts recursively. """ + + elem = Element(tag, **kwargs) + for key, val in dictionary.items(): + if tag == 'layers': + child = (dict_to_XML('layer', val, name=key)) + elif isinstance(val, MutableMapping): + child = (dict_to_XML(key, val)) + else: + if tag == 'config': + child = Element('variable', name=key) + else: + child = Element(key) + child.text = str(val) + elem.append(child) + return elem diff --git a/meta/lib/oeqa/utils/network.py b/meta/lib/oeqa/utils/network.py new file mode 100644 index 0000000000..2768f6c5db --- /dev/null +++ b/meta/lib/oeqa/utils/network.py @@ -0,0 +1,8 @@ +import socket + +def get_free_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('', 0)) + addr = s.getsockname() + s.close() + return addr[1] diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py new file mode 100644 index 0000000000..724afb2b5e --- /dev/null +++ b/meta/lib/oeqa/utils/package_manager.py @@ -0,0 +1,210 @@ +import os +import json +import shutil + +from oeqa.core.utils.test import getCaseFile, getCaseMethod + +def get_package_manager(d, root_path): + """ + Returns an OE package manager that can install packages in root_path. + """ + from oe.package_manager import RpmPM, OpkgPM, DpkgPM + + pkg_class = d.getVar("IMAGE_PKGTYPE") + if pkg_class == "rpm": + pm = RpmPM(d, + root_path, + d.getVar('TARGET_VENDOR')) + pm.create_configs() + + elif pkg_class == "ipk": + pm = OpkgPM(d, + root_path, + d.getVar("IPKGCONF_TARGET"), + d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) + + elif pkg_class == "deb": + pm = DpkgPM(d, + root_path, + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH')) + + pm.write_index() + pm.update() + + return pm + +def find_packages_to_extract(test_suite): + """ + Returns packages to extract required by runtime tests. + """ + from oeqa.core.utils.test import getSuiteCasesFiles + + needed_packages = {} + files = getSuiteCasesFiles(test_suite) + + for f in set(files): + json_file = _get_json_file(f) + if json_file: + needed_packages.update(_get_needed_packages(json_file)) + + return needed_packages + +def _get_json_file(module_path): + """ + Returns the path of the JSON file for a module, empty if doesn't exitst. + """ + + json_file = '%s.json' % module_path.rsplit('.', 1)[0] + if os.path.isfile(module_path) and os.path.isfile(json_file): + return json_file + else: + return '' + +def _get_needed_packages(json_file, test=None): + """ + Returns a dict with needed packages based on a JSON file. + + If a test is specified it will return the dict just for that test. + """ + needed_packages = {} + + with open(json_file) as f: + test_packages = json.load(f) + for key,value in test_packages.items(): + needed_packages[key] = value + + if test: + if test in needed_packages: + needed_packages = needed_packages[test] + else: + needed_packages = {} + + return needed_packages + +def extract_packages(d, needed_packages): + """ + Extract packages that will be needed during runtime. + """ + + import bb + import oe.path + + extracted_path = d.getVar('TEST_EXTRACTED_DIR') + + for key,value in needed_packages.items(): + packages = () + if isinstance(value, dict): + packages = (value, ) + elif isinstance(value, list): + packages = value + else: + bb.fatal('Failed to process needed packages for %s; ' + 'Value must be a dict or list' % key) + + for package in packages: + pkg = package['pkg'] + rm = package.get('rm', False) + extract = package.get('extract', True) + + if extract: + #logger.debug(1, 'Extracting %s' % pkg) + dst_dir = os.path.join(extracted_path, pkg) + # Same package used for more than one test, + # don't need to extract again. + if os.path.exists(dst_dir): + continue + + # Extract package and copy it to TEST_EXTRACTED_DIR + pkg_dir = _extract_in_tmpdir(d, pkg) + oe.path.copytree(pkg_dir, dst_dir) + shutil.rmtree(pkg_dir) + + else: + #logger.debug(1, 'Copying %s' % pkg) + _copy_package(d, pkg) + +def _extract_in_tmpdir(d, pkg): + """" + Returns path to a temp directory where the package was + extracted without dependencies. + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(d.getVar('TEST_INSTALL_TMP_DIR'), pkg) + pm = get_package_manager(d, pkg_path) + extract_dir = pm.extract(pkg) + shutil.rmtree(pkg_path) + + return extract_dir + +def _copy_package(d, pkg): + """ + Copy the RPM, DEB or IPK package to dst_dir + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(d.getVar('TEST_INSTALL_TMP_DIR'), pkg) + dst_dir = d.getVar('TEST_PACKAGED_DIR') + pm = get_package_manager(d, pkg_path) + pkg_info = pm.package_info(pkg) + file_path = pkg_info[pkg]['filepath'] + shutil.copy2(file_path, dst_dir) + shutil.rmtree(pkg_path) + +def install_package(test_case): + """ + Installs package in DUT if required. + """ + needed_packages = test_needs_package(test_case) + if needed_packages: + _install_uninstall_packages(needed_packages, test_case, True) + +def uninstall_package(test_case): + """ + Uninstalls package in DUT if required. + """ + needed_packages = test_needs_package(test_case) + if needed_packages: + _install_uninstall_packages(needed_packages, test_case, False) + +def test_needs_package(test_case): + """ + Checks if a test case requires to install/uninstall packages. + """ + test_file = getCaseFile(test_case) + json_file = _get_json_file(test_file) + + if json_file: + test_method = getCaseMethod(test_case) + needed_packages = _get_needed_packages(json_file, test_method) + if needed_packages: + return needed_packages + + return None + +def _install_uninstall_packages(needed_packages, test_case, install=True): + """ + Install/Uninstall packages in the DUT without using a package manager + """ + + if isinstance(needed_packages, dict): + packages = [needed_packages] + elif isinstance(needed_packages, list): + packages = needed_packages + + for package in packages: + pkg = package['pkg'] + rm = package.get('rm', False) + extract = package.get('extract', True) + src_dir = os.path.join(test_case.tc.extract_dir, pkg) + + # Install package + if install and extract: + test_case.tc.target.copyDirTo(src_dir, '/') + + # Uninstall package + elif not install and rm: + test_case.tc.target.deleteDirStructure(src_dir, '/') diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index f1a7e24ab7..ba44b96f53 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -7,25 +7,42 @@ import subprocess import os +import sys import time import signal import re import socket import select -import bb +import errno +import string +import threading +import codecs +from oeqa.utils.dump import HostDumper + +import logging +logger = logging.getLogger("BitBake.QemuRunner") +logger.addHandler(logging.StreamHandler()) + +# Get Unicode non printable control chars +control_range = list(range(0,32))+list(range(127,160)) +control_chars = [chr(x) for x in control_range + if chr(x) not in string.printable] +re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm): # Popen object for runqemu self.runqemu = None # pid of the qemu process that runqemu will start self.qemupid = None - # target ip - from the command line + # target ip - from the command line or runqemu output self.ip = None # host ip - where qemu is running self.server_ip = None + # target ip netmask + self.netmask = None self.machine = machine self.rootfs = rootfs @@ -34,158 +51,330 @@ class QemuRunner: self.deploy_dir_image = deploy_dir_image self.logfile = logfile self.boottime = boottime + self.logged = False + self.thread = None + self.use_kvm = use_kvm self.runqemutime = 60 - - self.create_socket() - + self.host_dumper = HostDumper(dump_host_cmds, dump_dir) def create_socket(self): - - self.bootlog = '' - self.qemusock = None - try: - self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.server_socket.setblocking(0) - self.server_socket.bind(("127.0.0.1",0)) - self.server_socket.listen(2) - self.serverport = self.server_socket.getsockname()[1] - bb.note("Created listening socket for qemu serial console on: 127.0.0.1:%s" % self.serverport) - except socket.error, msg: - self.server_socket.close() - bb.fatal("Failed to create listening socket: %s" %msg[1]) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(0) + sock.bind(("127.0.0.1",0)) + sock.listen(2) + port = sock.getsockname()[1] + logger.info("Created listening socket for qemu serial console on: 127.0.0.1:%s" % port) + return (sock, port) + except socket.error: + sock.close() + raise def log(self, msg): if self.logfile: - with open(self.logfile, "a") as f: + # It is needed to sanitize the data received from qemu + # because is possible to have control characters + msg = msg.decode("utf-8", errors='ignore') + msg = re_control_char.sub('', msg) + with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) - def start(self, qemuparams = None): + def getOutput(self, o): + import fcntl + fl = fcntl.fcntl(o, fcntl.F_GETFL) + fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) + return os.read(o.fileno(), 1000000).decode("utf-8") + + + def handleSIGCHLD(self, signum, frame): + if self.runqemu and self.runqemu.poll(): + if self.runqemu.returncode: + logger.info('runqemu exited with code %d' % self.runqemu.returncode) + logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.stop() + self._dump_host() + raise SystemExit + def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): if self.display: os.environ["DISPLAY"] = self.display - else: - bb.error("To start qemu I need a X desktop, please set DISPLAY correctly (e.g. DISPLAY=:1)") - return False + # Set this flag so that Qemu doesn't do any grabs as SDL grabs + # interact badly with screensavers. + os.environ["QEMU_DONT_GRAB"] = "1" if not os.path.exists(self.rootfs): - bb.error("Invalid rootfs %s" % self.rootfs) + logger.error("Invalid rootfs %s" % self.rootfs) return False if not os.path.exists(self.tmpdir): - bb.error("Invalid TMPDIR path %s" % self.tmpdir) + logger.error("Invalid TMPDIR path %s" % self.tmpdir) return False else: os.environ["OE_TMPDIR"] = self.tmpdir if not os.path.exists(self.deploy_dir_image): - bb.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) + logger.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) return False else: os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image - # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact - # badly with screensavers. - os.environ["QEMU_DONT_GRAB"] = "1" - self.qemuparams = 'bootparams="console=tty1 console=ttyS0,115200n8" qemuparams="-serial tcp:127.0.0.1:%s"' % self.serverport + if not launch_cmd: + launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams) + if self.use_kvm: + logger.info('Using kvm for runqemu') + launch_cmd += ' kvm' + else: + logger.info('Not using kvm for runqemu') + if not self.display: + launch_cmd += ' nographic' + launch_cmd += ' %s %s' % (self.machine, self.rootfs) + + return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams) + + def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None): + try: + threadsock, threadport = self.create_socket() + self.server_socket, self.serverport = self.create_socket() + except socket.error as msg: + logger.error("Failed to create listening socket: %s" % msg[1]) + return False + + bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' + if extra_bootparams: + bootparams = bootparams + ' ' + extra_bootparams + + self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1}"'.format(bootparams, threadport) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' - launch_cmd = 'runqemu %s %s %s' % (self.machine, self.rootfs, self.qemuparams) - self.runqemu = subprocess.Popen(launch_cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,preexec_fn=os.setpgrp) + launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) - bb.note("runqemu started, pid is %s" % self.runqemu.pid) - bb.note("waiting at most %s seconds for qemu pid" % self.runqemutime) + self.origchldhandler = signal.getsignal(signal.SIGCHLD) + signal.signal(signal.SIGCHLD, self.handleSIGCHLD) + + logger.info('launchcmd=%s'%(launch_cmd)) + + # FIXME: We pass in stdin=subprocess.PIPE here to work around stty + # blocking at the end of the runqemu script when using this within + # oe-selftest (this makes stty error out immediately). There ought + # to be a proper fix but this will suffice for now. + self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp) + output = self.runqemu.stdout + + # + # We need the preexec_fn above so that all runqemu processes can easily be killed + # (by killing their process group). This presents a problem if this controlling + # process itself is killed however since those processes don't notice the death + # of the parent and merrily continue on. + # + # Rather than hack runqemu to deal with this, we add something here instead. + # Basically we fork off another process which holds an open pipe to the parent + # and also is setpgrp. If/when the pipe sees EOF from the parent dieing, it kills + # the process group. This is like pctrl's PDEATHSIG but for a process group + # rather than a single process. + # + r, w = os.pipe() + self.monitorpid = os.fork() + if self.monitorpid: + os.close(r) + self.monitorpipe = os.fdopen(w, "w") + else: + # child process + os.setpgrp() + os.close(w) + r = os.fdopen(r) + x = r.read() + os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) + sys.exit(0) + + logger.info("runqemu started, pid is %s" % self.runqemu.pid) + logger.info("waiting at most %s seconds for qemu pid" % self.runqemutime) endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: + if self.runqemu.poll(): + if self.runqemu.returncode: + # No point waiting any longer + logger.info('runqemu exited with code %d' % self.runqemu.returncode) + self._dump_host() + self.stop() + logger.info("Output from runqemu:\n%s" % self.getOutput(output)) + return False time.sleep(1) + out = self.getOutput(output) + netconf = False # network configuration is not required by default if self.is_alive(): - bb.note("qemu started - qemu procces pid is %s" % self.qemupid) - cmdline = '' - with open('/proc/%s/cmdline' % self.qemupid) as p: - cmdline = p.read() - ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - if not ips or len(ips) != 3: - bb.note("Couldn't get ip from qemu process arguments! Here is the qemu command line used: %s" % cmdline) - self.stop() + logger.info("qemu started - qemu procces pid is %s" % self.qemupid) + if get_ip: + cmdline = '' + with open('/proc/%s/cmdline' % self.qemupid) as p: + cmdline = p.read() + # It is needed to sanitize the data received + # because is possible to have control characters + cmdline = re_control_char.sub('', cmdline) + try: + ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + self.ip = ips[0] + self.server_ip = ips[1] + logger.info("qemu cmdline used:\n{}".format(cmdline)) + except (IndexError, ValueError): + # Try to get network configuration from runqemu output + match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + out, re.MULTILINE|re.DOTALL) + if match: + self.ip, self.server_ip, self.netmask = match.groups() + # network configuration is required as we couldn't get it + # from the runqemu command line, so qemu doesn't run kernel + # and guest networking is not configured + netconf = True + else: + logger.error("Couldn't get ip from qemu command line and runqemu output! " + "Here is the qemu command line used:\n%s\n" + "and output from runqemu:\n%s" % (cmdline, out)) + self._dump_host() + self.stop() + return False + + logger.info("Target IP: %s" % self.ip) + logger.info("Server IP: %s" % self.server_ip) + + self.thread = LoggingThread(self.log, threadsock, logger) + self.thread.start() + if not self.thread.connection_established.wait(self.boottime): + logger.error("Didn't receive a console connection from qemu. " + "Here is the qemu command line used:\n%s\nand " + "output from runqemu:\n%s" % (cmdline, out)) + self.stop_thread() return False - else: - self.ip = ips[0] - self.server_ip = ips[1] - bb.note("Target IP: %s" % self.ip) - bb.note("Server IP: %s" % self.server_ip) - bb.note("Waiting at most %d seconds for login banner" % self.boottime ) + + logger.info("Output from runqemu:\n%s", out) + logger.info("Waiting at most %d seconds for login banner" % self.boottime) endtime = time.time() + self.boottime socklist = [self.server_socket] reachedlogin = False stopread = False + qemusock = None + bootlog = '' + data = b'' while time.time() < endtime and not stopread: - sread, swrite, serror = select.select(socklist, [], [], 5) + try: + sread, swrite, serror = select.select(socklist, [], [], 5) + except InterruptedError: + continue for sock in sread: if sock is self.server_socket: - self.qemusock, addr = self.server_socket.accept() - self.qemusock.setblocking(0) - socklist.append(self.qemusock) + qemusock, addr = self.server_socket.accept() + qemusock.setblocking(0) + socklist.append(qemusock) socklist.remove(self.server_socket) - bb.note("Connection from %s:%s" % addr) + logger.info("Connection from %s:%s" % addr) else: - data = sock.recv(1024) + data = data + sock.recv(1024) if data: - self.log(data) - self.bootlog += data - if re.search("qemu.* login:", self.bootlog): - stopread = True - reachedlogin = True - bb.note("Reached login banner") + try: + data = data.decode("utf-8", errors="surrogateescape") + bootlog += data + data = b'' + if re.search(".* login:", bootlog): + self.server_socket = qemusock + stopread = True + reachedlogin = True + logger.info("Reached login banner") + except UnicodeDecodeError: + continue else: socklist.remove(sock) sock.close() stopread = True if not reachedlogin: - bb.note("Target didn't reached login boot in %d seconds" % self.boottime) - lines = "\n".join(self.bootlog.splitlines()[-5:]) - bb.note("Last 5 lines of text:\n%s" % lines) - bb.note("Check full boot log: %s" % self.logfile) + logger.info("Target didn't reached login boot in %d seconds" % self.boottime) + lines = "\n".join(bootlog.splitlines()[-25:]) + logger.info("Last 25 lines of text:\n%s" % lines) + logger.info("Check full boot log: %s" % self.logfile) + self._dump_host() self.stop() return False + + # If we are not able to login the tests can continue + try: + (status, output) = self.run_serial("root\n", raw=True) + if re.search("root@[a-zA-Z0-9\-]+:~#", output): + self.logged = True + logger.info("Logged as root in serial console") + if netconf: + # configure guest networking + cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) + output = self.run_serial(cmd, raw=True)[1] + if re.search("root@[a-zA-Z0-9\-]+:~#", output): + logger.info("configured ip address %s", self.ip) + else: + logger.info("Couldn't configure guest networking") + else: + logger.info("Couldn't login into serial console" + " as root using blank password") + except: + logger.info("Serial console failed while trying to login") + else: - bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime) - output = self.runqemu.stdout + logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime) + self._dump_host() self.stop() - bb.note("Output from runqemu:\n%s" % output.read()) + logger.info("Output from runqemu:\n%s" % self.getOutput(output)) return False return self.is_alive() def stop(self): - + self.stop_thread() + self.stop_qemu_system() + if hasattr(self, "origchldhandler"): + signal.signal(signal.SIGCHLD, self.origchldhandler) if self.runqemu: - bb.note("Sending SIGTERM to runqemu") - os.killpg(self.runqemu.pid, signal.SIGTERM) + if hasattr(self, "monitorpid"): + os.kill(self.monitorpid, signal.SIGKILL) + logger.info("Sending SIGTERM to runqemu") + try: + os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) + except OSError as e: + if e.errno != errno.ESRCH: + raise endtime = time.time() + self.runqemutime while self.runqemu.poll() is None and time.time() < endtime: time.sleep(1) if self.runqemu.poll() is None: - bb.note("Sending SIGKILL to runqemu") - os.killpg(self.runqemu.pid, signal.SIGKILL) + logger.info("Sending SIGKILL to runqemu") + os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) self.runqemu = None - if self.server_socket: + if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None self.qemupid = None self.ip = None + def stop_qemu_system(self): + if self.qemupid: + try: + # qemu-system behaves well and a SIGTERM is enough + os.kill(self.qemupid, signal.SIGTERM) + except ProcessLookupError as e: + logger.warn('qemu-system ended unexpectedly') + + def stop_thread(self): + if self.thread and self.thread.is_alive(): + self.thread.stop() + self.thread.join() + def restart(self, qemuparams = None): - bb.note("Restarting qemu process") + logger.info("Restarting qemu process") if self.runqemu.poll() is None: self.stop() - self.create_socket() if self.start(qemuparams): return True return False def is_alive(self): + if not self.runqemu: + return False qemu_child = self.find_child(str(self.runqemu.pid)) if qemu_child: self.qemupid = qemu_child[0] @@ -198,7 +387,7 @@ class QemuRunner: # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.split('\n') + processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} commands = {} @@ -227,11 +416,189 @@ class QemuRunner: if p not in parents: parents.append(p) newparents = next - #print "Children matching %s:" % str(parents) + #print("Children matching %s:" % str(parents)) for p in parents: - # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx" + # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" basecmd = commands[p].split()[0] basecmd = os.path.basename(basecmd) if "qemu-system" in basecmd and "-serial tcp" in commands[p]: return [int(p),commands[p]] + + def run_serial(self, command, raw=False, timeout=5): + # We assume target system have echo to get command status + if not raw: + command = "%s; echo $?\n" % command + + data = '' + status = 0 + self.server_socket.sendall(command.encode('utf-8')) + start = time.time() + end = start + timeout + while True: + now = time.time() + if now >= end: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout + break + try: + sread, _, _ = select.select([self.server_socket],[],[], end - now) + except InterruptedError: + continue + if sread: + answer = self.server_socket.recv(1024) + if answer: + data += answer.decode('utf-8') + # Search the prompt to stop + if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + break + else: + raise Exception("No data on serial console socket") + + if data: + if raw: + status = 1 + else: + # Remove first line (command line) and last line (prompt) + data = data[data.find('$?\r\n')+4:data.rfind('\r\n')] + index = data.rfind('\r\n') + if index == -1: + status_cmd = data + data = "" + else: + status_cmd = data[index+2:] + data = data[:index] + if (status_cmd == "0"): + status = 1 + return (status, str(data)) + + + def _dump_host(self): + self.host_dumper.create_dir("qemu") + logger.warn("Qemu ended unexpectedly, dump data from host" + " is in %s" % self.host_dumper.dump_dir) + self.host_dumper.dump_host() + +# This class is for reading data from a socket and passing it to logfunc +# to be processed. It's completely event driven and has a straightforward +# event loop. The mechanism for stopping the thread is a simple pipe which +# will wake up the poll and allow for tearing everything down. +class LoggingThread(threading.Thread): + def __init__(self, logfunc, sock, logger): + self.connection_established = threading.Event() + self.serversock = sock + self.logfunc = logfunc + self.logger = logger + self.readsock = None + self.running = False + + self.errorevents = select.POLLERR | select.POLLHUP | select.POLLNVAL + self.readevents = select.POLLIN | select.POLLPRI + + threading.Thread.__init__(self, target=self.threadtarget) + + def threadtarget(self): + try: + self.eventloop() + finally: + self.teardown() + + def run(self): + self.logger.info("Starting logging thread") + self.readpipe, self.writepipe = os.pipe() + threading.Thread.run(self) + + def stop(self): + self.logger.info("Stopping logging thread") + if self.running: + os.write(self.writepipe, bytes("stop", "utf-8")) + + def teardown(self): + self.logger.info("Tearing down logging thread") + self.close_socket(self.serversock) + + if self.readsock is not None: + self.close_socket(self.readsock) + + self.close_ignore_error(self.readpipe) + self.close_ignore_error(self.writepipe) + self.running = False + + def eventloop(self): + poll = select.poll() + event_read_mask = self.errorevents | self.readevents + poll.register(self.serversock.fileno()) + poll.register(self.readpipe, event_read_mask) + + breakout = False + self.running = True + self.logger.info("Starting thread event loop") + while not breakout: + events = poll.poll() + for event in events: + # An error occurred, bail out + if event[1] & self.errorevents: + raise Exception(self.stringify_event(event[1])) + + # Event to stop the thread + if self.readpipe == event[0]: + self.logger.info("Stop event received") + breakout = True + break + + # A connection request was received + elif self.serversock.fileno() == event[0]: + self.logger.info("Connection request received") + self.readsock, _ = self.serversock.accept() + self.readsock.setblocking(0) + poll.unregister(self.serversock.fileno()) + poll.register(self.readsock.fileno(), event_read_mask) + + self.logger.info("Setting connection established event") + self.connection_established.set() + + # Actual data to be logged + elif self.readsock.fileno() == event[0]: + data = self.recv(1024) + self.logfunc(data) + + # Since the socket is non-blocking make sure to honor EAGAIN + # and EWOULDBLOCK. + def recv(self, count): + try: + data = self.readsock.recv(count) + except socket.error as e: + if e.errno == errno.EAGAIN or e.errno == errno.EWOULDBLOCK: + return '' + else: + raise + + if data is None: + raise Exception("No data on read ready socket") + elif not data: + # This actually means an orderly shutdown + # happened. But for this code it counts as an + # error since the connection shouldn't go away + # until qemu exits. + raise Exception("Console connection closed unexpectedly") + + return data + + def stringify_event(self, event): + val = '' + if select.POLLERR == event: + val = 'POLLER' + elif select.POLLHUP == event: + val = 'POLLHUP' + elif select.POLLNVAL == event: + val = 'POLLNVAL' + return val + + def close_socket(self, sock): + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + def close_ignore_error(self, fd): + try: + os.close(fd) + except OSError: + pass diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py new file mode 100644 index 0000000000..1bf59007ff --- /dev/null +++ b/meta/lib/oeqa/utils/qemutinyrunner.py @@ -0,0 +1,175 @@ +# Copyright (C) 2015 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# This module provides a class for starting qemu images of poky tiny. +# It's used by testimage.bbclass. + +import subprocess +import os +import time +import signal +import re +import socket +import select +import bb +from .qemurunner import QemuRunner + +class QemuTinyRunner(QemuRunner): + + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime): + + # Popen object for runqemu + self.runqemu = None + # pid of the qemu process that runqemu will start + self.qemupid = None + # target ip - from the command line + self.ip = None + # host ip - where qemu is running + self.server_ip = None + + self.machine = machine + self.rootfs = rootfs + self.display = display + self.tmpdir = tmpdir + self.deploy_dir_image = deploy_dir_image + self.logfile = logfile + self.boottime = boottime + + self.runqemutime = 60 + self.socketfile = "console.sock" + self.server_socket = None + self.kernel = kernel + + + def create_socket(self): + tries = 3 + while tries > 0: + try: + self.server_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.server_socket.connect(self.socketfile) + bb.note("Created listening socket for qemu serial console.") + tries = 0 + except socket.error as msg: + self.server_socket.close() + bb.fatal("Failed to create listening socket.") + tries -= 1 + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s" % msg) + + def start(self, qemuparams = None, ssh=True, extra_bootparams=None, runqemuparams='', discard_writes=True): + + if self.display: + os.environ["DISPLAY"] = self.display + else: + bb.error("To start qemu I need a X desktop, please set DISPLAY correctly (e.g. DISPLAY=:1)") + return False + if not os.path.exists(self.rootfs): + bb.error("Invalid rootfs %s" % self.rootfs) + return False + if not os.path.exists(self.tmpdir): + bb.error("Invalid TMPDIR path %s" % self.tmpdir) + return False + else: + os.environ["OE_TMPDIR"] = self.tmpdir + if not os.path.exists(self.deploy_dir_image): + bb.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) + return False + else: + os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + + # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact + # badly with screensavers. + os.environ["QEMU_DONT_GRAB"] = "1" + self.qemuparams = '--append "root=/dev/ram0 console=ttyS0" -nographic -serial unix:%s,server,nowait' % self.socketfile + + launch_cmd = 'qemu-system-i386 -kernel %s -initrd %s %s' % (self.kernel, self.rootfs, self.qemuparams) + self.runqemu = subprocess.Popen(launch_cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,preexec_fn=os.setpgrp) + + bb.note("runqemu started, pid is %s" % self.runqemu.pid) + bb.note("waiting at most %s seconds for qemu pid" % self.runqemutime) + endtime = time.time() + self.runqemutime + while not self.is_alive() and time.time() < endtime: + time.sleep(1) + + if self.is_alive(): + bb.note("qemu started - qemu procces pid is %s" % self.qemupid) + self.create_socket() + else: + bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime) + output = self.runqemu.stdout + self.stop() + bb.note("Output from runqemu:\n%s" % output.read().decode("utf-8")) + return False + + return self.is_alive() + + def run_serial(self, command, timeout=5): + self.server_socket.sendall(command+'\n') + data = '' + status = 0 + stopread = False + endtime = time.time()+timeout + while time.time()<endtime and not stopread: + try: + sread, _, _ = select.select([self.server_socket],[],[],1) + except InterruptedError: + continue + for sock in sread: + answer = sock.recv(1024) + if answer: + data += answer + else: + sock.close() + stopread = True + if not data: + status = 1 + if not stopread: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout + return (status, str(data)) + + def find_child(self,parent_pid): + # + # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] + # + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] + processes = ps.decode("utf-8").split('\n') + nfields = len(processes[0].split()) - 1 + pids = {} + commands = {} + for row in processes[1:]: + data = row.split(None, nfields) + if len(data) != 3: + continue + if data[1] not in pids: + pids[data[1]] = [] + + pids[data[1]].append(data[0]) + commands[data[0]] = data[2] + + if parent_pid not in pids: + return [] + + parents = [] + newparents = pids[parent_pid] + while newparents: + next = [] + for p in newparents: + if p in pids: + for n in pids[p]: + if n not in parents and n not in next: + next.append(n) + if p not in parents: + parents.append(p) + newparents = next + #print("Children matching %s:" % str(parents)) + for p in parents: + # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" + # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" + basecmd = commands[p].split()[0] + basecmd = os.path.basename(basecmd) + if "qemu-system" in basecmd and "-serial unix" in commands[p]: + return [int(p),commands[p]] diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index 1c81795a87..05d6502550 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (C) 2013 Intel Corporation # # Released under the MIT license (see COPYING.MIT) @@ -31,12 +32,18 @@ class SSHProcess(object): self.starttime = None self.logfile = None + # Unset DISPLAY which means we won't trigger SSH_ASKPASS + env = os.environ.copy() + if "DISPLAY" in env: + del env['DISPLAY'] + self.options['env'] = env + def log(self, msg): if self.logfile: with open(self.logfile, "a") as f: f.write("%s" % msg) - def run(self, command, timeout=None, logfile=None): + def _run(self, command, timeout=None, logfile=None): self.logfile = logfile self.starttime = time.time() output = '' @@ -45,16 +52,19 @@ class SSHProcess(object): endtime = self.starttime + timeout eof = False while time.time() < endtime and not eof: - if select.select([self.process.stdout], [], [], 5)[0] != []: - data = os.read(self.process.stdout.fileno(), 1024) - if not data: - self.process.stdout.close() - eof = True - else: - output += data - self.log(data) - endtime = time.time() + timeout - + try: + if select.select([self.process.stdout], [], [], 5)[0] != []: + data = os.read(self.process.stdout.fileno(), 1024) + if not data: + self.process.stdout.close() + eof = True + else: + data = data.decode("utf-8") + output += data + self.log(data) + endtime = time.time() + timeout + except InterruptedError: + continue # process hasn't returned yet if not eof: @@ -73,8 +83,18 @@ class SSHProcess(object): self.status = self.process.wait() self.output = output.rstrip() - return (self.status, self.output) + def run(self, command, timeout=None, logfile=None): + try: + self._run(command, timeout, logfile) + except: + # Need to guard against a SystemExit or other exception occuring whilst running + # and ensure we don't leave a process behind. + if self.process.poll() is None: + self.process.kill() + self.status = self.process.wait() + raise + return (self.status, self.output) class SSHControl(object): def __init__(self, ip, logfile=None, timeout=300, user='root', port=None): @@ -120,8 +140,7 @@ class SSHControl(object): timeout=0 - no timeout, let command run until it returns """ - # We need to source /etc/profile for a proper PATH on the target - command = self.ssh + [self.ip, ' . /etc/profile; ' + command] + command = self.ssh + [self.ip, 'export PATH=/usr/sbin:/sbin:/usr/bin:/bin; ' + command] if timeout is None: return self._internal_run(command, self.defaulttimeout, self.ignore_status) @@ -130,9 +149,97 @@ class SSHControl(object): return self._internal_run(command, timeout, self.ignore_status) def copy_to(self, localpath, remotepath): - command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] - return self._internal_run(command, ignore_status=False) + if os.path.islink(localpath): + link = os.readlink(localpath) + dst_dir, dst_base = os.path.split(remotepath) + return self.run("cd %s; ln -s %s %s" % (dst_dir, link, dst_base)) + else: + command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] + return self._internal_run(command, ignore_status=False) def copy_from(self, remotepath, localpath): command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath] return self._internal_run(command, ignore_status=False) + + def copy_dir_to(self, localpath, remotepath): + """ + Copy recursively localpath directory to remotepath in target. + """ + + for root, dirs, files in os.walk(localpath): + # Create directories in the target as needed + for d in dirs: + tmp_dir = os.path.join(root, d).replace(localpath, "") + new_dir = os.path.join(remotepath, tmp_dir.lstrip("/")) + cmd = "mkdir -p %s" % new_dir + self.run(cmd) + + # Copy files into the target + for f in files: + tmp_file = os.path.join(root, f).replace(localpath, "") + dst_file = os.path.join(remotepath, tmp_file.lstrip("/")) + src_file = os.path.join(root, f) + self.copy_to(src_file, dst_file) + + + def delete_files(self, remotepath, files): + """ + Delete files in target's remote path. + """ + + cmd = "rm" + if not isinstance(files, list): + files = [files] + + for f in files: + cmd = "%s %s" % (cmd, os.path.join(remotepath, f)) + + self.run(cmd) + + + def delete_dir(self, remotepath): + """ + Delete remotepath directory in target. + """ + + cmd = "rmdir %s" % remotepath + self.run(cmd) + + + def delete_dir_structure(self, localpath, remotepath): + """ + Delete recursively localpath structure directory in target's remotepath. + + This function is very usefult to delete a package that is installed in + the DUT and the host running the test has such package extracted in tmp + directory. + + Example: + pwd: /home/user/tmp + tree: . + └── work + ├── dir1 + │ └── file1 + └── dir2 + + localpath = "/home/user/tmp" and remotepath = "/home/user" + + With the above variables this function will try to delete the + directory in the DUT in this order: + /home/user/work/dir1/file1 + /home/user/work/dir1 (if dir is empty) + /home/user/work/dir2 (if dir is empty) + /home/user/work (if dir is empty) + """ + + for root, dirs, files in os.walk(localpath, topdown=False): + # Delete files first + tmpdir = os.path.join(root).replace(localpath, "") + remotedir = os.path.join(remotepath, tmpdir.lstrip("/")) + self.delete_files(remotedir, files) + + # Remove dirs if empty + for d in dirs: + tmpdir = os.path.join(root, d).replace(localpath, "") + remotedir = os.path.join(remotepath, tmpdir.lstrip("/")) + self.delete_dir(remotepath) diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py new file mode 100644 index 0000000000..1f7d11b55c --- /dev/null +++ b/meta/lib/oeqa/utils/subprocesstweak.py @@ -0,0 +1,19 @@ +import subprocess + +class OETestCalledProcessError(subprocess.CalledProcessError): + def __str__(self): + def strify(o): + if isinstance(o, bytes): + return o.decode("utf-8", errors="replace") + else: + return o + + s = "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + if hasattr(self, "output") and self.output: + s = s + "\nStandard Output: " + strify(self.output) + if hasattr(self, "stderr") and self.stderr: + s = s + "\nStandard Error: " + strify(self.stderr) + return s + +def errors_have_output(): + subprocess.CalledProcessError = OETestCalledProcessError diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py index 32296762c0..9249fa2635 100644 --- a/meta/lib/oeqa/utils/targetbuild.py +++ b/meta/lib/oeqa/utils/targetbuild.py @@ -6,23 +6,33 @@ import os import re +import bb.utils import subprocess +import tempfile +from abc import ABCMeta, abstractmethod +class BuildProject(metaclass=ABCMeta): -class TargetBuildProject(): - - def __init__(self, target, d, uri, foldername=None): - self.target = target + def __init__(self, d, uri, foldername=None, tmpdir=None): self.d = d self.uri = uri - self.targetdir = "~/" self.archive = os.path.basename(uri) - self.localarchive = "/tmp/" + self.archive - self.fname = re.sub(r'.tar.bz2|tar.gz$', '', self.archive) + if not tmpdir: + tmpdir = self.d.getVar('WORKDIR') + if not tmpdir: + tmpdir = tempfile.mkdtemp(prefix='buildproject') + self.localarchive = os.path.join(tmpdir,self.archive) if foldername: self.fname = foldername + else: + self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive) - def download_archive(self): + # Download self.archive to self.localarchive + def _download_archive(self): + dl_dir = self.d.getVar("DL_DIR") + if dl_dir and os.path.exists(os.path.join(dl_dir, self.archive)): + bb.utils.copyfile(os.path.join(dl_dir, self.archive), self.localarchive) + return exportvars = ['HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', 'https_proxy', @@ -34,12 +44,44 @@ class TargetBuildProject(): cmd = '' for var in exportvars: - val = self.d.getVar(var, True) + val = self.d.getVar(var) if val: cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri) - subprocess.check_call(cmd, shell=True) + subprocess.check_output(cmd, shell=True) + + # This method should provide a way to run a command in the desired environment. + @abstractmethod + def _run(self, cmd): + pass + + # The timeout parameter of target.run is set to 0 to make the ssh command + # run with no timeout. + def run_configure(self, configure_args='', extra_cmds=''): + return self._run('cd %s; %s ./configure %s' % (self.targetdir, extra_cmds, configure_args)) + + def run_make(self, make_args=''): + return self._run('cd %s; make %s' % (self.targetdir, make_args)) + + def run_install(self, install_args=''): + return self._run('cd %s; make install %s' % (self.targetdir, install_args)) + + def clean(self): + self._run('rm -rf %s' % self.targetdir) + subprocess.call('rm -f %s' % self.localarchive, shell=True) + pass + +class TargetBuildProject(BuildProject): + + def __init__(self, target, d, uri, foldername=None): + self.target = target + self.targetdir = "~/" + BuildProject.__init__(self, d, uri, foldername) + + def download_archive(self): + + self._download_archive() (status, output) = self.target.copy_to(self.localarchive, self.targetdir) if status != 0: @@ -54,15 +96,44 @@ class TargetBuildProject(): # The timeout parameter of target.run is set to 0 to make the ssh command # run with no timeout. - def run_configure(self): - return self.target.run('cd %s; ./configure' % self.targetdir, 0)[0] + def _run(self, cmd): + return self.target.run(cmd, 0)[0] - def run_make(self): - return self.target.run('cd %s; make' % self.targetdir, 0)[0] - def run_install(self): - return self.target.run('cd %s; make install' % self.targetdir, 0)[0] +class SDKBuildProject(BuildProject): - def clean(self): - self.target.run('rm -rf %s' % self.targetdir) - subprocess.call('rm -f %s' % self.localarchive, shell=True) + def __init__(self, testpath, sdkenv, d, uri, foldername=None): + self.sdkenv = sdkenv + self.testdir = testpath + self.targetdir = testpath + bb.utils.mkdirhier(testpath) + self.datetime = d.getVar('DATETIME') + self.testlogdir = d.getVar("TEST_LOG_DIR") + bb.utils.mkdirhier(self.testlogdir) + self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime) + BuildProject.__init__(self, d, uri, foldername, tmpdir=testpath) + + def download_archive(self): + + self._download_archive() + + cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir) + subprocess.check_output(cmd, shell=True) + + #Change targetdir to project folder + self.targetdir = os.path.join(self.targetdir, self.fname) + + def run_configure(self, configure_args='', extra_cmds=' gnu-configize; '): + return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=extra_cmds) + + def run_install(self, install_args=''): + return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir)) + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s\n" % msg) + + def _run(self, cmd): + self.log("Running . %s; " % self.sdkenv + cmd) + return subprocess.call(". %s; " % self.sdkenv + cmd, shell=True) diff --git a/meta/lib/oeqa/utils/testexport.py b/meta/lib/oeqa/utils/testexport.py new file mode 100644 index 0000000000..be2a2110fc --- /dev/null +++ b/meta/lib/oeqa/utils/testexport.py @@ -0,0 +1,263 @@ +# Copyright (C) 2015 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Provides functions to help with exporting binaries obtained from built targets + +import os, re, glob as g, shutil as sh,sys +from time import sleep +from .commands import runCmd +from difflib import SequenceMatcher as SM + +try: + import bb +except ImportError: + class my_log(): + def __init__(self): + pass + def plain(self, msg): + if msg: + print(msg) + def warn(self, msg): + if msg: + print("WARNING: " + msg) + def fatal(self, msg): + if msg: + print("FATAL:" + msg) + sys.exit(1) + bb = my_log() + + +def determine_if_poky_env(): + """ + used to determine if we are inside the poky env or not. Usefull for remote machine where poky is not present + """ + check_env = True if ("/scripts" and "/bitbake/bin") in os.getenv("PATH") else False + return check_env + + +def get_dest_folder(tune_features, folder_list): + """ + Function to determine what rpm deploy dir to choose for a given architecture based on TUNE_FEATURES + """ + features_list = tune_features.split(" ") + features_list.reverse() + features_list = "_".join(features_list) + match_rate = 0 + best_match = None + for folder in folder_list: + curr_match_rate = SM(None, folder, features_list).ratio() + if curr_match_rate > match_rate: + match_rate = curr_match_rate + best_match = folder + return best_match + + +def process_binaries(d, params): + param_list = params + export_env = d.getVar("TEST_EXPORT_ONLY") + + def extract_binary(pth_to_pkg, dest_pth=None): + cpio_command = runCmd("which cpio") + rpm2cpio_command = runCmd("ls /usr/bin/rpm2cpio") + if (cpio_command.status != 0) and (rpm2cpio_command.status != 0): + bb.fatal("Either \"rpm2cpio\" or \"cpio\" tools are not available on your system." + "All binaries extraction processes will not be available, crashing all related tests." + "Please install them according to your OS recommendations") # will exit here + if dest_pth: + os.chdir(dest_pth) + else: + os.chdir("%s" % os.sep)# this is for native package + extract_bin_command = runCmd("%s %s | %s -idm" % (rpm2cpio_command.output, pth_to_pkg, cpio_command.output)) # semi-hardcoded because of a bug on poky's rpm2cpio + return extract_bin_command + + if determine_if_poky_env(): # machine with poky environment + exportpath = d.getVar("TEST_EXPORT_DIR") if export_env else d.getVar("DEPLOY_DIR") + rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM") + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(rpm_deploy_dir)) + arch_rpm_dir = os.path.join(rpm_deploy_dir, arch) + extracted_bin_dir = os.path.join(exportpath,"binaries", arch, "extracted_binaries") + packaged_bin_dir = os.path.join(exportpath,"binaries", arch, "packaged_binaries") + # creating necessary directory structure in case testing is done in poky env. + if export_env == "0": + if not os.path.exists(extracted_bin_dir): bb.utils.mkdirhier(extracted_bin_dir) + if not os.path.exists(packaged_bin_dir): bb.utils.mkdirhier(packaged_bin_dir) + + if param_list[3] == "native": + if export_env == "1": #this is a native package and we only need to copy it. no need for extraction + native_rpm_dir = os.path.join(rpm_deploy_dir, get_dest_folder("{} nativesdk".format(d.getVar("BUILD_SYS")), os.listdir(rpm_deploy_dir))) + native_rpm_file_list = [item for item in os.listdir(native_rpm_dir) if re.search("nativesdk-" + param_list[0] + "-([0-9]+\.*)", item)] + if not native_rpm_file_list: + bb.warn("Couldn't find any version of {} native package. Related tests will most probably fail.".format(param_list[0])) + return "" + for item in native_rpm_file_list:# will copy all versions of package. Used version will be selected on remote machine + bb.plain("Copying native package file: %s" % item) + sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries", "native")) + else: # nothing to do here; running tests under bitbake, so we asume native binaries are in sysroots dir. + if param_list[1] or param_list[4]: + bb.warn("Native binary %s %s%s. Running tests under bitbake environment. Version can't be checked except when the test itself does it" + " and binary can't be removed."%(param_list[0],"has assigned ver. " + param_list[1] if param_list[1] else "", + ", is marked for removal" if param_list[4] else "")) + else:# the package is target aka DUT intended and it is either required to be delivered in an extracted form or in a packaged version + target_rpm_file_list = [item for item in os.listdir(arch_rpm_dir) if re.search(param_list[0] + "-([0-9]+\.*)", item)] + if not target_rpm_file_list: + bb.warn("Couldn't find any version of target package %s. Please ensure it was built. " + "Related tests will probably fail." % param_list[0]) + return "" + if param_list[2] == "rpm": # binary should be deployed as rpm; (other, .deb, .ipk? ; in the near future) + for item in target_rpm_file_list: # copying all related rpm packages. "Intuition" reasons, someone may need other versions too. Deciding later on version + bb.plain("Copying target specific packaged file: %s" % item) + sh.copy(os.path.join(arch_rpm_dir, item), packaged_bin_dir) + return "copied" + else: # it is required to extract the binary + if param_list[1]: # the package is versioned + for item in target_rpm_file_list: + if re.match(".*-{}-.*\.rpm".format(param_list[1]), item): + destination = os.path.join(extracted_bin_dir,param_list[0], param_list[1]) + bb.utils.mkdirhier(destination) + extract_binary(os.path.join(arch_rpm_dir, item), destination) + break + else: + bb.warn("Couldn't find the desired version %s for target binary %s. Related test cases will probably fail." % (param_list[1], param_list[0])) + return "" + return "extracted" + else: # no version provided, just extract one binary + destination = os.path.join(extracted_bin_dir,param_list[0], + re.search(".*-([0-9]+\.[0-9]+)-.*rpm", target_rpm_file_list[0]).group(1)) + bb.utils.mkdirhier(destination) + extract_binary(os.path.join(arch_rpm_dir, target_rpm_file_list[0]), destination) + return "extracted" + else: # remote machine + binaries_path = os.getenv("bin_dir")# in order to know where the binaries are, bin_dir is set as env. variable + if param_list[3] == "native": #need to extract the native pkg here + native_rpm_dir = os.path.join(binaries_path, "native") + native_rpm_file_list = os.listdir(native_rpm_dir) + for item in native_rpm_file_list: + if param_list[1] and re.match("nativesdk-{}-{}-.*\.rpm".format(param_list[0], param_list[1]), item): # native package has version + extract_binary(os.path.join(native_rpm_dir, item)) + break + else:# just copy any related native binary + found_version = re.match("nativesdk-{}-([0-9]+\.[0-9]+)-".format(param_list[0]), item).group(1) + if found_version: + extract_binary(os.path.join(native_rpm_dir, item)) + else: + bb.warn("Couldn't find native package %s%s. Related test cases will be influenced." % + (param_list[0], " with version " + param_list[1] if param_list[1] else "")) + return + + else: # this is for target device + if param_list[2] == "rpm": + return "No need to extract, this is an .rpm file" + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(binaries_path)) + extracted_bin_path = os.path.join(binaries_path, arch, "extracted_binaries") + extracted_bin_list = [item for item in os.listdir(extracted_bin_path)] + packaged_bin_path = os.path.join(binaries_path, arch, "packaged_binaries") + packaged_bin_file_list = os.listdir(packaged_bin_path) + # see if the package is already in the extracted ones; maybe it was deployed when exported the env. + if os.path.exists(os.path.join(extracted_bin_path, param_list[0], param_list[1] if param_list[1] else "")): + return "binary %s is already extracted" % param_list[0] + else: # we need to search for it in the packaged binaries directory. It may have been shipped after export + for item in packaged_bin_file_list: + if param_list[1]: + if re.match("%s-%s.*rpm" % (param_list[0], param_list[1]), item): # package with version + if not os.path.exists(os.path.join(extracted_bin_path, param_list[0],param_list[1])): + os.makedirs(os.path.join(extracted_bin_path, param_list[0], param_list[1])) + extract_binary(os.path.join(packaged_bin_path, item), os.path.join(extracted_bin_path, param_list[0],param_list[1])) + bb.plain("Using {} for {}".format(os.path.join(packaged_bin_path, item), param_list[0])) + break + else: + if re.match("%s-.*rpm" % param_list[0], item): + found_version = re.match(".*-([0-9]+\.[0-9]+)-", item).group(1) + if not os.path.exists(os.path.join(extracted_bin_path, param_list[0], found_version)): + os.makedirs(os.path.join(extracted_bin_path, param_list[0], found_version)) + bb.plain("Used ver. %s for %s" % (found_version, param_list[0])) + extract_binary(os.path.join(packaged_bin_path, item), os.path.join(extracted_bin_path, param_list[0], found_version)) + break + else: + bb.warn("Couldn't find target package %s%s. Please ensure it is available " + "in either of these directories: extracted_binaries or packaged_binaries. " + "Related tests will probably fail." % (param_list[0], " with version " + param_list[1] if param_list[1] else "")) + return + return "Binary %s extracted successfully." % param_list[0] + + +def files_to_copy(base_dir): + """ + Produces a list of files relative to the base dir path sent as param + :return: the list of relative path files + """ + files_list = [] + dir_list = [base_dir] + count = 1 + dir_count = 1 + while (dir_count == 1 or dir_count != count): + count = dir_count + for dir in dir_list: + for item in os.listdir(dir): + if os.path.isdir(os.path.join(dir, item)) and os.path.join(dir, item) not in dir_list: + dir_list.append(os.path.join(dir, item)) + dir_count = len(dir_list) + elif os.path.join(dir, item) not in files_list and os.path.isfile(os.path.join(dir, item)): + files_list.append(os.path.join(dir, item)) + return files_list + + +def send_bin_to_DUT(d,params): + from oeqa.oetest import oeRuntimeTest + param_list = params + cleanup_list = list() + bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries") if determine_if_poky_env() \ + else os.getenv("bin_dir") + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(bins_dir)) + arch_rpms_dir = os.path.join(bins_dir, arch, "packaged_binaries") + extracted_bin_dir = os.path.join(bins_dir, arch, "extracted_binaries", param_list[0]) + + def send_extracted_binary(): + bin_local_dir = os.path.join(extracted_bin_dir, param_list[1] if param_list[1] else os.listdir(extracted_bin_dir)[0]) + for item in files_to_copy(bin_local_dir): + split_path = item.split(bin_local_dir)[1] + path_on_DUT = split_path if split_path[0] is "/" else "/" + split_path # create the path as on DUT; eg. /usr/bin/bin_file + (status, output) = oeRuntimeTest.tc.target.copy_to(item, path_on_DUT) + if status != 0: + bb.warn("Failed to copy %s binary file %s on the remote target: %s" % + (param_list[0], "ver. " + param_list[1] if param_list[1] else "", d.getVar("MACHINE"))) + return + if param_list[4] == "rm": + cleanup_list.append(path_on_DUT) + return cleanup_list + + def send_rpm(remote_path): # if it is not required to have an extracted binary, but to send an .rpm file + rpm_to_send = "" + for item in os.listdir(arch_rpms_dir): + if param_list[1] and re.match("%s-%s-.*rpm"%(param_list[0], param_list[1]), item): + rpm_to_send = item + break + elif re.match("%s-[0-9]+\.[0-9]+-.*rpm" % param_list[0], item): + rpm_to_send = item + break + else: + bb.warn("No rpm package found for %s %s in .rpm files dir %s. Skipping deployment." % + (param_list[0], "ver. " + param_list[1] if param_list[1] else "", rpms_file_dir) ) + return + (status, output) = oeRuntimeTest.tc.target.copy_to(os.path.join(arch_rpms_dir, rpm_to_send), remote_path) + if status != 0: + bb.warn("Failed to copy %s on the remote target: %s" %(param_list[0], d.getVar("MACHINE"))) + return + if param_list[4] == "rm": + cleanup_list.append(os.path.join(remote_path, rpm_to_send)) + return cleanup_list + + if param_list[2] == "rpm": # send an .rpm file + return send_rpm("/home/root") # rpms will be sent on home dir of remote machine + else: + return send_extracted_binary() + + +def rm_bin(removal_list): # need to know both if the binary is sent archived and the path where it is sent if archived + from oeqa.oetest import oeRuntimeTest + for item in removal_list: + (status,output) = oeRuntimeTest.tc.target.run("rm " + item) + if status != 0: + bb.warn("Failed to remove: %s. Please ensure connection with the target device is up and running and " + "you have the needed rights." % item) + diff --git a/meta/lib/rootfspostcommands.py b/meta/lib/rootfspostcommands.py new file mode 100644 index 0000000000..4742e0613c --- /dev/null +++ b/meta/lib/rootfspostcommands.py @@ -0,0 +1,56 @@ +import os + +def sort_file(filename, mapping): + """ + Sorts a passwd or group file based on the numeric ID in the third column. + If a mapping is given, the name from the first column is mapped via that + dictionary instead (necessary for /etc/shadow and /etc/gshadow). If not, + a new mapping is created on the fly and returned. + """ + new_mapping = {} + with open(filename, 'rb+') as f: + lines = f.readlines() + # No explicit error checking for the sake of simplicity. /etc + # files are assumed to be well-formed, causing exceptions if + # not. + for line in lines: + entries = line.split(b':') + name = entries[0] + if mapping is None: + id = int(entries[2]) + else: + id = mapping[name] + new_mapping[name] = id + # Sort by numeric id first, with entire line as secondary key + # (just in case that there is more than one entry for the same id). + lines.sort(key=lambda line: (new_mapping[line.split(b':')[0]], line)) + # We overwrite the entire file, i.e. no truncate() necessary. + f.seek(0) + f.write(b''.join(lines)) + return new_mapping + +def remove_backup(filename): + """ + Removes the backup file for files like /etc/passwd. + """ + backup_filename = filename + '-' + if os.path.exists(backup_filename): + os.unlink(backup_filename) + +def sort_passwd(sysconfdir): + """ + Sorts passwd and group files in a rootfs /etc directory by ID. + Backup files are sometimes are inconsistent and then cannot be + sorted (YOCTO #11043), and more importantly, are not needed in + the initial rootfs, so they get deleted. + """ + for main, shadow in (('passwd', 'shadow'), + ('group', 'gshadow')): + filename = os.path.join(sysconfdir, main) + remove_backup(filename) + if os.path.exists(filename): + mapping = sort_file(filename, None) + filename = os.path.join(sysconfdir, shadow) + remove_backup(filename) + if os.path.exists(filename): + sort_file(filename, mapping) |