diff options
Diffstat (limited to 'meta/lib')
212 files changed, 10727 insertions, 4700 deletions
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py new file mode 100644 index 0000000000..c5d4c73cf5 --- /dev/null +++ b/meta/lib/buildstats.py @@ -0,0 +1,158 @@ +# Implements system state sampling. Called by buildstats.bbclass. +# Because it is a real Python module, it can hold persistent state, +# like open log files and the time of the last sampling. + +import time +import re +import bb.event + +class SystemStats: + def __init__(self, d): + bn = d.getVar('BUILDNAME') + bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) + bb.utils.mkdirhier(bsdir) + + self.proc_files = [] + for filename, handler in ( + ('diskstats', self._reduce_diskstats), + ('meminfo', self._reduce_meminfo), + ('stat', self._reduce_stat), + ): + # The corresponding /proc files might not exist on the host. + # For example, /proc/diskstats is not available in virtualized + # environments like Linux-VServer. Silently skip collecting + # the data. + if os.path.exists(os.path.join('/proc', filename)): + # In practice, this class gets instantiated only once in + # the bitbake cooker process. Therefore 'append' mode is + # not strictly necessary, but using it makes the class + # more robust should two processes ever write + # concurrently. + destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if handler else '', filename)) + self.proc_files.append((filename, open(destfile, 'ab'), handler)) + self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab') + # Last time that we sampled /proc data resp. recorded disk monitoring data. + self.last_proc = 0 + self.last_disk_monitor = 0 + # Minimum number of seconds between recording a sample. This + # becames relevant when we get called very often while many + # short tasks get started. Sampling during quiet periods + # depends on the heartbeat event, which fires less often. + self.min_seconds = 1 + + self.meminfo_regex = re.compile(b'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)') + self.diskstats_regex = re.compile(b'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') + self.diskstats_ltime = None + self.diskstats_data = None + self.stat_ltimes = None + + def close(self): + self.monitor_disk.close() + for _, output, _ in self.proc_files: + output.close() + + def _reduce_meminfo(self, time, data): + """ + Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' + and writes their values into a single line, in that order. + """ + values = {} + for line in data.split(b'\n'): + m = self.meminfo_regex.match(line) + if m: + values[m.group(1)] = m.group(2) + if len(values) == 6: + return (time, + b' '.join([values[x] for x in + (b'MemTotal', b'MemFree', b'Buffers', b'Cached', b'SwapTotal', b'SwapFree')]) + b'\n') + + def _diskstats_is_relevant_line(self, linetokens): + if len(linetokens) != 14: + return False + disk = linetokens[2] + return self.diskstats_regex.match(disk) + + def _reduce_diskstats(self, time, data): + relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n'))) + diskdata = [0] * 3 + reduced = None + for tokens in relevant_tokens: + # rsect + diskdata[0] += int(tokens[5]) + # wsect + diskdata[1] += int(tokens[9]) + # use + diskdata[2] += int(tokens[12]) + if self.diskstats_ltime: + # We need to compute information about the time interval + # since the last sampling and record the result as sample + # for that point in the past. + interval = time - self.diskstats_ltime + if interval > 0: + sums = [ a - b for a, b in zip(diskdata, self.diskstats_data) ] + readTput = sums[0] / 2.0 * 100.0 / interval + writeTput = sums[1] / 2.0 * 100.0 / interval + util = float( sums[2] ) / 10 / interval + util = max(0.0, min(1.0, util)) + reduced = (self.diskstats_ltime, (readTput, writeTput, util)) + + self.diskstats_ltime = time + self.diskstats_data = diskdata + return reduced + + + def _reduce_nop(self, time, data): + return (time, data) + + def _reduce_stat(self, time, data): + if not data: + return None + # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line + tokens = data.split(b'\n', 1)[0].split() + times = [ int(token) for token in tokens[1:] ] + reduced = None + if self.stat_ltimes: + user = float((times[0] + times[1]) - (self.stat_ltimes[0] + self.stat_ltimes[1])) + system = float((times[2] + times[5] + times[6]) - (self.stat_ltimes[2] + self.stat_ltimes[5] + self.stat_ltimes[6])) + idle = float(times[3] - self.stat_ltimes[3]) + iowait = float(times[4] - self.stat_ltimes[4]) + + aSum = max(user + system + idle + iowait, 1) + reduced = (time, (user/aSum, system/aSum, iowait/aSum)) + + self.stat_ltimes = times + return reduced + + def sample(self, event, force): + now = time.time() + if (now - self.last_proc > self.min_seconds) or force: + for filename, output, handler in self.proc_files: + with open(os.path.join('/proc', filename), 'rb') as input: + data = input.read() + if handler: + reduced = handler(now, data) + else: + reduced = (now, data) + if reduced: + if isinstance(reduced[1], bytes): + # Use as it is. + data = reduced[1] + else: + # Convert to a single line. + data = (' '.join([str(x) for x in reduced[1]]) + '\n').encode('ascii') + # Unbuffered raw write, less overhead and useful + # in case that we end up with concurrent writes. + os.write(output.fileno(), + ('%.0f\n' % reduced[0]).encode('ascii') + + data + + b'\n') + self.last_proc = now + + if isinstance(event, bb.event.MonitorDiskEvent) and \ + ((now - self.last_disk_monitor > self.min_seconds) or force): + os.write(self.monitor_disk.fileno(), + ('%.0f\n' % now).encode('ascii') + + ''.join(['%s: %d\n' % (dev, sample.total_bytes - sample.free_bytes) + for dev, sample in event.disk_usage.items()]).encode('ascii') + + b'\n') + self.last_disk_monitor = now diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py index 16491a96e1..3a5b7b6b44 100644 --- a/meta/lib/oe/buildhistory_analysis.py +++ b/meta/lib/oe/buildhistory_analysis.py @@ -1,6 +1,6 @@ # Report significant differences in the buildhistory repository since a specific revision # -# Copyright (C) 2012 Intel Corporation +# Copyright (C) 2012-2013, 2016-2017 Intel Corporation # Author: Paul Eggleton <paul.eggleton@linux.intel.com> # # Note: requires GitPython 0.3.1+ @@ -13,7 +13,10 @@ import os.path import difflib import git import re +import hashlib +import collections import bb.utils +import bb.tinfoil # How to display fields @@ -69,7 +72,22 @@ class ChangeRecord: pkglist.append(k) return pkglist + def detect_renamed_dirs(aitems, bitems): + adirs = set(map(os.path.dirname, aitems)) + bdirs = set(map(os.path.dirname, bitems)) + files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \ + for name in adirs - bdirs] + files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \ + for name in bdirs - adirs] + renamed_dirs = [(dir1, dir2) for dir1, files1 in files_ab for dir2, files2 in files_ba if files1 == files2] + # remove files that belong to renamed dirs from aitems and bitems + for dir1, dir2 in renamed_dirs: + aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)] + bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)] + return renamed_dirs, aitems, bitems + if self.fieldname in list_fields or self.fieldname in list_order_fields: + renamed_dirs = [] if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue) aitems = pkglist_combine(depvera) @@ -77,16 +95,29 @@ class ChangeRecord: else: aitems = self.oldvalue.split() bitems = self.newvalue.split() + if self.fieldname == 'FILELIST': + renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + removed = list(set(aitems) - set(bitems)) added = list(set(bitems) - set(aitems)) + lines = [] + if renamed_dirs: + for dfrom, dto in renamed_dirs: + lines.append('directory renamed %s -> %s' % (dfrom, dto)) if removed or added: if removed and not bitems: - out = '%s: removed all items "%s"' % (self.fieldname, ' '.join(removed)) + lines.append('removed all items "%s"' % ' '.join(removed)) else: - out = '%s:%s%s' % (self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '') + if removed: + lines.append('removed "%s"' % ' '.join(removed)) + if added: + lines.append('added "%s"' % ' '.join(added)) else: - out = '%s changed order' % self.fieldname + lines.append('changed order') + + out = '%s: %s' % (self.fieldname, ', '.join(lines)) + elif self.fieldname in numeric_fields: aval = int(self.oldvalue or 0) bval = int(self.newvalue or 0) @@ -190,7 +221,7 @@ class FileChange: def blob_to_dict(blob): - alines = [line.decode() for line in blob.data_stream.read().splitlines()] + alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] adict = {} for line in alines: splitv = [i.strip() for i in line.split('=',1)] @@ -359,18 +390,138 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): if ' '.join(alist) == ' '.join(blist): continue + if key == 'PKGR' and not report_all: + vers = [] + # strip leading 'r' and dots + for ver in (astr.split()[0], bstr.split()[0]): + if ver.startswith('r'): + ver = ver[1:] + vers.append(ver.replace('.', '')) + maxlen = max(len(vers[0]), len(vers[1])) + try: + # pad with '0' and convert to int + vers = [int(ver.ljust(maxlen, '0')) for ver in vers] + except ValueError: + pass + else: + # skip decrements and increments + if abs(vers[0] - vers[1]) == 1: + continue + chg = ChangeRecord(path, key, astr, bstr, monitored) changes.append(chg) return changes -def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False): +def compare_siglists(a_blob, b_blob, taskdiff=False): + # FIXME collapse down a recipe's tasks? + alines = a_blob.data_stream.read().decode('utf-8').splitlines() + blines = b_blob.data_stream.read().decode('utf-8').splitlines() + keys = [] + pnmap = {} + def readsigs(lines): + sigs = {} + for line in lines: + linesplit = line.split() + if len(linesplit) > 2: + sigs[linesplit[0]] = linesplit[2] + if not linesplit[0] in keys: + keys.append(linesplit[0]) + pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0] + return sigs + adict = readsigs(alines) + bdict = readsigs(blines) + out = [] + + changecount = 0 + addcount = 0 + removecount = 0 + if taskdiff: + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + + changes = collections.OrderedDict() + + def compare_hashfiles(pn, taskname, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data) + + if not taskname: + (pn, taskname) = pn.rsplit('.', 1) + pn = pnmap.get(pn, pn) + desc = '%s.%s' % (pn, taskname) + + if len(hashfiles) == 0: + out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2)) + elif not hash1 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1)) + elif not hash2 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2)) + else: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True) + for line in out2: + m = hashlib.sha256() + m.update(line.encode('utf-8')) + entry = changes.get(m.hexdigest(), (line, [])) + if desc not in entry[1]: + changes[m.hexdigest()] = (line, entry[1] + [desc]) + + # Define recursion callback + def recursecb(key, hash1, hash2): + compare_hashfiles(key, None, hash1, hash2) + return [] + + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + changecount += 1 + (pn, taskname) = key.rsplit('.', 1) + compare_hashfiles(pn, taskname, siga, sigb) + elif siga is None: + addcount += 1 + elif sigb is None: + removecount += 1 + for key, item in changes.items(): + line, tasks = item + if len(tasks) == 1: + desc = tasks[0] + elif len(tasks) == 2: + desc = '%s and %s' % (tasks[0], tasks[1]) + else: + desc = '%s and %d others' % (tasks[-1], len(tasks)-1) + out.append('%s: %s' % (desc, line)) + else: + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + out.append('%s changed from %s to %s' % (key, siga, sigb)) + changecount += 1 + elif siga is None: + out.append('%s was added' % key) + addcount += 1 + elif sigb is None: + out.append('%s was removed' % key) + removecount += 1 + out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100))) + return '\n'.join(out) + + +def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False): repo = git.Repo(repopath) assert repo.bare == False commit = repo.commit(revision1) diff = commit.diff(revision2) changes = [] + + if sigs or sigsdiff: + for d in diff.iter_change_type('M'): + if d.a_blob.path == 'siglist.txt': + changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff)) + return changes + for d in diff.iter_change_type('M'): path = os.path.dirname(d.a_blob.path) if path.startswith('packages/'): @@ -378,34 +529,34 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif path.startswith('images/'): filename = os.path.basename(d.a_blob.path) if filename in img_monitor_files: if filename == 'files-in-image.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_file_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) elif filename == 'installed-package-names.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) else: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif filename == 'image-info.txt': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif '/image-files/' in path: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) # Look for added preinst/postinst/prerm/postrm @@ -419,7 +570,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': addedpkgs.append(path) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True) addedchanges.append(chg) for chg in addedchanges: found = False @@ -436,7 +587,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if path.startswith('packages/'): filename = os.path.basename(d.a_blob.path) if filename != 'latest' and filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read(), '', True) + chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) # Link related changes diff --git a/meta/lib/oe/classextend.py b/meta/lib/oe/classextend.py index 4c8a00070c..d2eeaf0e5c 100644 --- a/meta/lib/oe/classextend.py +++ b/meta/lib/oe/classextend.py @@ -25,7 +25,7 @@ class ClassExtender(object): return name def map_variable(self, varname, setvar = True): - var = self.d.getVar(varname, True) + var = self.d.getVar(varname) if not var: return "" var = var.split() @@ -38,7 +38,7 @@ class ClassExtender(object): return newdata def map_regexp_variable(self, varname, setvar = True): - var = self.d.getVar(varname, True) + var = self.d.getVar(varname) if not var: return "" var = var.split() @@ -60,7 +60,7 @@ class ClassExtender(object): return dep else: # Do not extend for that already have multilib prefix - var = self.d.getVar("MULTILIB_VARIANTS", True) + var = self.d.getVar("MULTILIB_VARIANTS") if var: var = var.split() for v in var: @@ -74,7 +74,7 @@ class ClassExtender(object): varname = varname + "_" + suffix orig = self.d.getVar("EXTENDPKGV", False) self.d.setVar("EXTENDPKGV", "EXTENDPKGV") - deps = self.d.getVar(varname, True) + deps = self.d.getVar(varname) if not deps: self.d.setVar("EXTENDPKGV", orig) return @@ -87,7 +87,7 @@ class ClassExtender(object): self.d.setVar("EXTENDPKGV", orig) def map_packagevars(self): - for pkg in (self.d.getVar("PACKAGES", True).split() + [""]): + for pkg in (self.d.getVar("PACKAGES").split() + [""]): self.map_depends_variable("RDEPENDS", pkg) self.map_depends_variable("RRECOMMENDS", pkg) self.map_depends_variable("RSUGGESTS", pkg) @@ -97,7 +97,7 @@ class ClassExtender(object): self.map_depends_variable("PKG", pkg) def rename_packages(self): - for pkg in (self.d.getVar("PACKAGES", True) or "").split(): + for pkg in (self.d.getVar("PACKAGES") or "").split(): if pkg.startswith(self.extname): self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg]) continue diff --git a/meta/lib/oe/classutils.py b/meta/lib/oe/classutils.py index e7856c86f2..45cd5249be 100644 --- a/meta/lib/oe/classutils.py +++ b/meta/lib/oe/classutils.py @@ -36,7 +36,7 @@ abstract base classes out of the registry).""" @classmethod def prioritized(tcls): return sorted(list(tcls.registry.values()), - key=lambda v: v.priority, reverse=True) + key=lambda v: (v.priority, v.name), reverse=True) def unregister(cls): for key in cls.registry.keys(): diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py index eddf5bb2da..a372904183 100644 --- a/meta/lib/oe/copy_buildsystem.py +++ b/meta/lib/oe/copy_buildsystem.py @@ -4,11 +4,15 @@ import stat import shutil def _smart_copy(src, dest): + import subprocess # smart_copy will choose the correct function depending on whether the # source is a file or a directory. mode = os.stat(src).st_mode if stat.S_ISDIR(mode): - shutil.copytree(src, dest, symlinks=True, ignore=shutil.ignore_patterns('.git')) + bb.utils.mkdirhier(dest) + cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else: shutil.copyfile(src, dest) shutil.copymode(src, dest) @@ -17,8 +21,8 @@ class BuildSystem(object): def __init__(self, context, d): self.d = d self.context = context - self.layerdirs = d.getVar('BBLAYERS', True).split() - self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split() + self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] + self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() def copy_bitbake_and_layers(self, destdir, workspace_name=None): # Copy in all metadata layers + bitbake (as repositories) @@ -26,7 +30,7 @@ class BuildSystem(object): bb.utils.mkdirhier(destdir) layers = list(self.layerdirs) - corebase = self.d.getVar('COREBASE', True) + corebase = os.path.abspath(self.d.getVar('COREBASE')) layers.append(corebase) # Exclude layers @@ -42,7 +46,7 @@ class BuildSystem(object): extranum += 1 workspace_newname = '%s-%d' % (workspace_name, extranum) - corebase_files = self.d.getVar('COREBASE_FILES', True).split() + corebase_files = self.d.getVar('COREBASE_FILES').split() corebase_files = [corebase + '/' +x for x in corebase_files] # Make sure bitbake goes in bitbake_dir = bb.__file__.rsplit('/', 3)[0] @@ -96,7 +100,7 @@ class BuildSystem(object): # Drop all bbappends except the one for the image the SDK is being built for # (because of externalsrc, the workspace bbappends will interfere with the # locked signatures if present, and we don't need them anyway) - image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE', True)))[0] + '.bbappend' + image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend' appenddir = os.path.join(layerdestpath, 'appends') if os.path.isdir(appenddir): for fn in os.listdir(appenddir): @@ -145,7 +149,7 @@ def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output invalue = True f.write(line) -def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output): +def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): merged = {} arch_order = [] with open(lockedsigs_main, 'r') as f: @@ -195,23 +199,46 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu fulltypes.append(typename) f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) - write_sigs_file(copy_output, list(tocopy.keys()), tocopy) + if copy_output: + write_sigs_file(copy_output, list(tocopy.keys()), tocopy) if merged_output: write_sigs_file(merged_output, arch_order, merged) -def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): +def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): + import shutil bb.note('Generating sstate-cache...') - nativelsbstring = d.getVar('NATIVELSBSTRING', True) - bb.process.run("gen-lockedsig-cache %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring)) - if fixedlsbstring: + nativelsbstring = d.getVar('NATIVELSBSTRING') + bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) + if fixedlsbstring and nativelsbstring != fixedlsbstring: nativedir = output_sstate_cache + '/' + nativelsbstring if os.path.isdir(nativedir): destdir = os.path.join(output_sstate_cache, fixedlsbstring) - bb.utils.mkdirhier(destdir) + for root, _, files in os.walk(nativedir): + for fn in files: + src = os.path.join(root, fn) + dest = os.path.join(destdir, os.path.relpath(src, nativedir)) + if os.path.exists(dest): + # Already exists, and it'll be the same file, so just delete it + os.unlink(src) + else: + bb.utils.mkdirhier(os.path.dirname(dest)) + shutil.move(src, dest) + +def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): + import subprocess - dirlist = os.listdir(nativedir) - for i in dirlist: - src = os.path.join(nativedir, i) - dest = os.path.join(destdir, i) - os.rename(src, dest) + bb.note('Generating sstate task list...') + + if not cwd: + cwd = os.getcwd() + if logfile: + logparam = '-l %s' % logfile + else: + logparam = '' + cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + env = dict(d.getVar('BB_ORIGENV', False)) + env.pop('BUILDDIR', '') + pathitems = env['PATH'].split(':') + env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) + bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py index ee48950a82..b8901e63f5 100644 --- a/meta/lib/oe/data.py +++ b/meta/lib/oe/data.py @@ -1,9 +1,10 @@ +import json import oe.maketype def typed_value(key, d): """Construct a value for the specified metadata variable, using its flags to determine the type and parameters for construction.""" - var_type = d.getVarFlag(key, 'type', True) + var_type = d.getVarFlag(key, 'type') flags = d.getVarFlags(key) if flags is not None: flags = dict((flag, d.expand(value)) @@ -12,6 +13,35 @@ def typed_value(key, d): flags = {} try: - return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags) + return oe.maketype.create(d.getVar(key) or '', var_type, **flags) except (TypeError, ValueError) as exc: bb.msg.fatal("Data", "%s: %s" % (key, str(exc))) + +def export2json(d, json_file, expand=True, searchString="",replaceString=""): + data2export = {} + keys2export = [] + + for key in d.keys(): + if key.startswith("_"): + continue + elif key.startswith("BB"): + continue + elif key.startswith("B_pn"): + continue + elif key.startswith("do_"): + continue + elif d.getVarFlag(key, "func"): + continue + + keys2export.append(key) + + for key in keys2export: + try: + data2export[key] = d.getVar(key, expand).replace(searchString,replaceString) + except bb.data_smart.ExpansionError: + data2export[key] = '' + except AttributeError: + pass + + with open(json_file, "w") as f: + json.dump(data2export, f, skipkeys=True, indent=4, sort_keys=True) diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py index f1f1fbbb28..37f04ed359 100644 --- a/meta/lib/oe/distro_check.py +++ b/meta/lib/oe/distro_check.py @@ -1,53 +1,20 @@ -from contextlib import contextmanager -@contextmanager def create_socket(url, d): - import urllib.request, urllib.parse, urllib.error - socket = urllib.request.urlopen(url, proxies=get_proxies(d)) - try: - yield socket - finally: - socket.close() + import urllib + from bb.utils import export_proxies -def get_proxies(d): - proxies = {} - for key in ['http', 'https', 'ftp', 'ftps', 'no', 'all']: - proxy = d.getVar(key + '_proxy', True) - if proxy: - proxies[key] = proxy - return proxies + export_proxies(d) + return urllib.request.urlopen(url) def get_links_from_url(url, d): "Return all the href links found on the web location" - import sgmllib - - class LinksParser(sgmllib.SGMLParser): - def parse(self, s): - "Parse the given string 's'." - self.feed(s) - self.close() - - def __init__(self, verbose=0): - "Initialise an object passing 'verbose' to the superclass." - sgmllib.SGMLParser.__init__(self, verbose) - self.hyperlinks = [] - - def start_a(self, attributes): - "Process a hyperlink and its 'attributes'." - for name, value in attributes: - if name == "href": - self.hyperlinks.append(value.strip('/')) - - def get_hyperlinks(self): - "Return the list of hyperlinks." - return self.hyperlinks - - with create_socket(url,d) as sock: - webpage = sock.read() - - linksparser = LinksParser() - linksparser.parse(webpage) - return linksparser.get_hyperlinks() + from bs4 import BeautifulSoup, SoupStrainer + + soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) + hyperlinks = [] + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" @@ -55,6 +22,7 @@ def find_latest_numeric_release(url, d): maxstr="" for link in get_links_from_url(url, d): try: + # TODO use LooseVersion release = float(link) except: release = 0 @@ -65,140 +33,112 @@ def find_latest_numeric_release(url, d): def is_src_rpm(name): "Check if the link is pointing to a src.rpm file" - if name[-8:] == ".src.rpm": - return True - else: - return False + return name.endswith(".src.rpm") def package_name_from_srpm(srpm): "Strip out the package name from the src.rpm filename" - strings = srpm.split('-') - package_name = strings[0] - for i in range(1, len (strings) - 1): - str = strings[i] - if not str[0].isdigit(): - package_name += '-' + str - return package_name - -def clean_package_list(package_list): - "Removes multiple entries of packages and sorts the list" - set = {} - map(set.__setitem__, package_list, []) - return set.keys() - -def get_latest_released_meego_source_package_list(d): - "Returns list of all the name os packages in the latest meego distro" - - package_names = [] - try: - f = open("/tmp/Meego-1.1", "r") - for line in f: - package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end - except IOError: pass - package_list=clean_package_list(package_names) - return "1.0", package_list + # ca-certificates-2016.2.7-1.0.fc24.src.rpm + # ^name ^ver ^release^removed + (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) + return name def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) links = get_links_from_url(url, d) - srpms = list(filter(is_src_rpm, links)) - names_list = list(map(package_name_from_srpm, srpms)) + srpms = filter(is_src_rpm, links) + names_list = map(package_name_from_srpm, srpms) - new_pkgs = [] + new_pkgs = set() for pkgs in names_list: - new_pkgs.append(pkgs + ":" + section) - + new_pkgs.add(pkgs + ":" + section) return new_pkgs +def get_source_package_list_from_url_by_letter(url, section, d): + import string + from urllib.error import HTTPError + packages = set() + for letter in (string.ascii_lowercase + string.digits): + # Not all subfolders may exist, so silently handle 404 + try: + packages |= get_source_package_list_from_url(url + "/" + letter, section, d) + except HTTPError as e: + if e.code != 404: raise + return packages + def get_latest_released_fedora_source_package_list(d): "Returns list of all the name os packages in the latest fedora distro" latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) - - package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d) - -# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") - package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) - - package_list=clean_package_list(package_names) - - return latest, package_list + package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) + package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) + return latest, package_names def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) - package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d) - - package_list=clean_package_list(package_names) - return latest, package_list + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d) + return latest, package_names def get_latest_released_mandriva_source_package_list(d): "Returns list of all the name os packages in the latest mandriva distro" latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) -# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") - package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) + package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) + return latest, package_names - package_list=clean_package_list(package_names) - return latest, package_list +def get_latest_released_clear_source_package_list(d): + latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) + package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) + return latest, package_names def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" - releases = [] - for link in get_links_from_url(url, d): - if link[:6] == "Debian": - if ';' not in link: - releases.append(link) + releases = [link.replace("Debian", "") + for link in get_links_from_url(url, d) + if link.startswith("Debian")] releases.sort() try: - return releases.pop()[6:] + return releases[-1] except: return "_NotFound_" def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - with create_socket(url,d) as sock: - webpage = sock.read() - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - tmpfile.close() import gzip - bb.note("Reading %s: %s" % (url, section)) - - f = gzip.open(tmpfilename) - package_names = [] - for line in f: - if line[:9] == "Package: ": - package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end - os.unlink(tmpfilename) + package_names = set() + for line in gzip.open(create_socket(url, d), mode="rt"): + if line.startswith("Package:"): + pkg = line.split(":", 1)[1].strip() + package_names.add(pkg + ":" + section) return package_names def get_latest_released_debian_source_package_list(d): - "Returns list of all the name os packages in the latest debian distro" + "Returns list of all the name of packages in the latest debian distro" latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) - url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" + url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" package_names = get_debian_style_source_package_list(url, "main", d) -# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" -# package_names += get_debian_style_source_package_list(url, "contrib") - url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" - package_names += get_debian_style_source_package_list(url, "updates", d) - package_list=clean_package_list(package_names) - return latest, package_list + url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names def find_latest_ubuntu_release(url, d): - "Find the latest listed ubuntu release on the given url" + """ + Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. + + To avoid matching development releases look for distributions that have + updates, so the resulting distro could be any supported release. + """ url += "?C=M;O=D" # Descending Sort by Last Modified for link in get_links_from_url(url, d): - if link[-8:] == "-updates": - return link[:-8] + if "-updates" in link: + distro = link.replace("-updates", "") + return distro return "_NotFound_" def get_latest_released_ubuntu_source_package_list(d): @@ -206,52 +146,45 @@ def get_latest_released_ubuntu_source_package_list(d): latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest package_names = get_debian_style_source_package_list(url, "main", d) -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "multiverse") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "universe") url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest - package_names += get_debian_style_source_package_list(url, "updates", d) - package_list=clean_package_list(package_names) - return latest, package_list + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names def create_distro_packages_list(distro_check_dir, d): + import shutil + pkglst_dir = os.path.join(distro_check_dir, "package_lists") - if not os.path.isdir (pkglst_dir): - os.makedirs(pkglst_dir) - # first clear old stuff - for file in os.listdir(pkglst_dir): - os.unlink(os.path.join(pkglst_dir, file)) - - per_distro_functions = [ - ["Debian", get_latest_released_debian_source_package_list], - ["Ubuntu", get_latest_released_ubuntu_source_package_list], - ["Fedora", get_latest_released_fedora_source_package_list], - ["OpenSuSE", get_latest_released_opensuse_source_package_list], - ["Mandriva", get_latest_released_mandriva_source_package_list], - ["Meego", get_latest_released_meego_source_package_list] - ] - - from datetime import datetime - begin = datetime.now() - for distro in per_distro_functions: - name = distro[0] - release, package_list = distro[1](d) + bb.utils.remove(pkglst_dir, True) + bb.utils.mkdirhier(pkglst_dir) + + per_distro_functions = ( + ("Debian", get_latest_released_debian_source_package_list), + ("Ubuntu", get_latest_released_ubuntu_source_package_list), + ("Fedora", get_latest_released_fedora_source_package_list), + ("OpenSuSE", get_latest_released_opensuse_source_package_list), + ("Mandriva", get_latest_released_mandriva_source_package_list), + ("Clear", get_latest_released_clear_source_package_list), + ) + + for name, fetcher_func in per_distro_functions: + try: + release, package_list = fetcher_func(d) + except Exception as e: + bb.warn("Cannot fetch packages for %s: %s" % (name, e)) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) + if len(package_list) == 0: + bb.error("Didn't fetch any packages for %s %s" % (name, release)) + package_list_file = os.path.join(pkglst_dir, name + "-" + release) - f = open(package_list_file, "w+b") - for pkg in package_list: - f.write(pkg + "\n") - f.close() - end = datetime.now() - delta = end - begin - bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) + with open(package_list_file, 'w') as f: + for pkg in sorted(package_list): + f.write(pkg + "\n") def update_distro_data(distro_check_dir, datetime, d): """ - If distro packages list data is old then rebuild it. - The operations has to be protected by a lock so that - only one thread performes it at a time. + If distro packages list data is old then rebuild it. + The operations has to be protected by a lock so that + only one thread performes it at a time. """ if not os.path.isdir (distro_check_dir): try: @@ -266,9 +199,9 @@ def update_distro_data(distro_check_dir, datetime, d): import fcntl try: if not os.path.exists(datetime_file): - open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail - f = open(datetime_file, "r+b") + f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: @@ -278,71 +211,59 @@ def update_distro_data(distro_check_dir, datetime, d): f.seek(0) f.write(datetime) - except OSError: - raise Exception('Unable to read/write this file: %s' % (datetime_file)) + except OSError as e: + raise Exception('Unable to open timestamp: %s' % e) finally: fcntl.lockf(f, fcntl.LOCK_UN) f.close() - + def compare_in_distro_packages_list(distro_check_dir, d): if not os.path.isdir(distro_check_dir): raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") - + localdata = bb.data.createCopy(d) pkglst_dir = os.path.join(distro_check_dir, "package_lists") matching_distros = [] - pn = d.getVar('PN', True) - recipe_name = d.getVar('PN', True) + pn = recipe_name = d.getVar('PN') bb.note("Checking: %s" % pn) - trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"}) - if pn.find("-native") != -1: pnstripped = pn.split("-native") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.startswith("nativesdk-"): pnstripped = pn.split("nativesdk-") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[1] if pn.find("-cross") != -1: pnstripped = pn.split("-cross") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.find("-initial") != -1: pnstripped = pn.split("-initial") - localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] bb.note("Recipe: %s" % recipe_name) - tmp = localdata.getVar('DISTRO_PN_ALIAS', True) distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) - - if tmp: - list = tmp.split(' ') - for str in list: - if str and str.find("=") == -1 and distro_exceptions[str]: - matching_distros.append(str) + tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" + for str in tmp.split(): + if str and str.find("=") == -1 and distro_exceptions[str]: + matching_distros.append(str) distro_pn_aliases = {} - if tmp: - list = tmp.split(' ') - for str in list: - if str.find("=") != -1: - (dist, pn_alias) = str.split('=') - distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() - + for str in tmp.split(): + if "=" in str: + (dist, pn_alias) = str.split('=') + distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() + for file in os.listdir(pkglst_dir): (distro, distro_release) = file.split("-") - f = open(os.path.join(pkglst_dir, file), "rb") + f = open(os.path.join(pkglst_dir, file), "r") for line in f: (pkg, section) = line.split(":") if distro.lower() in distro_pn_aliases: @@ -355,38 +276,34 @@ def compare_in_distro_packages_list(distro_check_dir, d): break f.close() - - if tmp != None: - list = tmp.split(' ') - for item in list: - matching_distros.append(item) + for item in tmp.split(): + matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros def create_log_file(d, logname): - import subprocess - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) logfn, logsuffix = os.path.splitext(logname) - logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix)) + logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) if not os.path.exists(logfile): slogfile = os.path.join(logpath, logname) if os.path.exists(slogfile): os.remove(slogfile) - subprocess.call("touch %s" % logfile, shell=True) + open(logfile, 'w+').close() os.symlink(logfile, slogfile) d.setVar('LOG_FILE', logfile) return logfile def save_distro_check_result(result, datetime, result_file, d): - pn = d.getVar('PN', True) - logdir = d.getVar('LOG_DIR', True) + pn = d.getVar('PN') + logdir = d.getVar('LOG_DIR') if not logdir: bb.error("LOG_DIR variable is not defined, can't write the distro_check results") return - if not os.path.isdir(logdir): - os.makedirs(logdir) + bb.utils.mkdirhier(logdir) + line = pn for i in result: line = line + "," + i diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py index a8a478aa95..7ce767ee0a 100644 --- a/meta/lib/oe/gpg_sign.py +++ b/meta/lib/oe/gpg_sign.py @@ -7,10 +7,11 @@ import oe.utils class LocalSigner(object): """Class for handling local (on the build host) signing""" def __init__(self, d): - self.gpg_bin = d.getVar('GPG_BIN', True) or \ + self.gpg_bin = d.getVar('GPG_BIN') or \ bb.utils.which(os.getenv('PATH'), 'gpg') - self.gpg_path = d.getVar('GPG_PATH', True) - self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpm") + self.gpg_path = d.getVar('GPG_PATH') + self.gpg_version = self.get_gpg_version() + self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign") def export_pubkey(self, output_file, keyid, armor=True): """Export GPG public key to a file""" @@ -24,22 +25,26 @@ class LocalSigner(object): status, output = oe.utils.getstatusoutput(cmd) if status: raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % - (keyid, output.decode("utf-8"))) + (keyid, output)) def sign_rpms(self, files, keyid, passphrase): """Sign RPM files""" cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid - cmd += "--define '_gpg_passphrase %s' " % passphrase + gpg_args = '--batch --passphrase=%s' % passphrase + if self.gpg_version > (2,1,): + gpg_args += ' --pinentry-mode=loopback' + cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args if self.gpg_bin: cmd += "--define '%%__gpg %s' " % self.gpg_bin if self.gpg_path: cmd += "--define '_gpg_path %s' " % self.gpg_path - cmd += ' '.join(files) - status, output = oe.utils.getstatusoutput(cmd) - if status: - raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output.decode("utf-8")) + # Sign in chunks of 100 packages + for i in range(0, len(files), 100): + status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100])) + if status: + raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): """Create a detached signature of a file""" @@ -58,9 +63,7 @@ class LocalSigner(object): #gpg > 2.1 supports password pipes only through the loopback interface #gpg < 2.1 errors out if given unknown parameters - dots = self.get_gpg_version().split('.') - assert len(dots) >= 2 - if int(dots[0]) >= 2 and int(dots[1]) >= 1: + if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback'] cmd += [input_file] @@ -87,10 +90,11 @@ class LocalSigner(object): def get_gpg_version(self): - """Return the gpg version""" + """Return the gpg version as a tuple of ints""" import subprocess try: - return subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + return tuple([int(i) for i in ver_str.split('.')]) except subprocess.CalledProcessError as e: raise bb.build.FuncFailed("Could not get gpg version: %s" % e) @@ -113,4 +117,3 @@ def get_signer(d, backend): return LocalSigner(d) else: bb.fatal("Unsupported signing backend '%s'" % backend) - diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index 39ef9654fc..8d2fd1709c 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -215,3 +215,21 @@ def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') return (manifest.licensestr, manifest.licenses) + +class ListVisitor(LicenseVisitor): + """Record all different licenses found in the license string""" + def __init__(self): + self.licenses = set() + + def visit_Str(self, node): + self.licenses.add(node.s) + +def list_licenses(licensestr): + """Simply get a list of all licenses mentioned in a license string. + Binary operators are not applied or taken into account in any way""" + visitor = ListVisitor() + try: + visitor.visit_string(licensestr) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + return visitor.licenses diff --git a/meta/lib/oe/lsb.py b/meta/lib/oe/lsb.py index e0bdfba255..3a945e0fce 100644 --- a/meta/lib/oe/lsb.py +++ b/meta/lib/oe/lsb.py @@ -1,26 +1,54 @@ -def release_dict(): - """Return the output of lsb_release -ir as a dictionary""" +def release_dict_osr(): + """ Populate a dict with pertinent values from /etc/os-release """ + if not os.path.exists('/etc/os-release'): + return None + + data = {} + with open('/etc/os-release') as f: + for line in f: + try: + key, val = line.rstrip().split('=', 1) + except ValueError: + continue + if key == 'ID': + data['DISTRIB_ID'] = val.strip('"') + if key == 'VERSION_ID': + data['DISTRIB_RELEASE'] = val.strip('"') + + return data + +def release_dict_lsb(): + """ Return the output of lsb_release -ir as a dictionary """ from subprocess import PIPE try: output, err = bb.process.run(['lsb_release', '-ir'], stderr=PIPE) except bb.process.CmdError as exc: - return None + return {} + + lsb_map = { 'Distributor ID': 'DISTRIB_ID', + 'Release': 'DISTRIB_RELEASE'} + lsb_keys = lsb_map.keys() data = {} for line in output.splitlines(): - if line.startswith("-e"): line = line[3:] + if line.startswith("-e"): + line = line[3:] try: key, value = line.split(":\t", 1) except ValueError: continue - else: - data[key] = value + if key in lsb_keys: + data[lsb_map[key]] = value + + if len(data.keys()) != 2: + return None + return data def release_dict_file(): - """ Try to gather LSB release information manually when lsb_release tool is unavailable """ - data = None + """ Try to gather release information manually when other methods fail """ + data = {} try: if os.path.exists('/etc/lsb-release'): data = {} @@ -37,14 +65,6 @@ def release_dict_file(): if match: data['DISTRIB_ID'] = match.group(1) data['DISTRIB_RELEASE'] = match.group(2) - elif os.path.exists('/etc/os-release'): - data = {} - with open('/etc/os-release') as f: - for line in f: - if line.startswith('NAME='): - data['DISTRIB_ID'] = line[5:].rstrip().strip('"') - if line.startswith('VERSION_ID='): - data['DISTRIB_RELEASE'] = line[11:].rstrip().strip('"') elif os.path.exists('/etc/SuSE-release'): data = {} data['DISTRIB_ID'] = 'SUSE LINUX' @@ -55,7 +75,7 @@ def release_dict_file(): break except IOError: - return None + return {} return data def distro_identifier(adjust_hook=None): @@ -64,15 +84,17 @@ def distro_identifier(adjust_hook=None): import re - lsb_data = release_dict() - if lsb_data: - distro_id, release = lsb_data['Distributor ID'], lsb_data['Release'] - else: - lsb_data_file = release_dict_file() - if lsb_data_file: - distro_id, release = lsb_data_file['DISTRIB_ID'], lsb_data_file.get('DISTRIB_RELEASE', None) - else: - distro_id, release = None, None + # Try /etc/os-release first, then the output of `lsb_release -ir` and + # finally fall back on parsing various release files in order to determine + # host distro name and version. + distro_data = release_dict_osr() + if not distro_data: + distro_data = release_dict_lsb() + if not distro_data: + distro_data = release_dict_file() + + distro_id = distro_data.get('DISTRIB_ID', '') + release = distro_data.get('DISTRIB_RELEASE', '') if adjust_hook: distro_id, release = adjust_hook(distro_id, release) @@ -82,7 +104,7 @@ def distro_identifier(adjust_hook=None): distro_id = re.sub(r'\W', '', distro_id) if release: - id_str = '{0}-{1}'.format(distro_id, release) + id_str = '{0}-{1}'.format(distro_id.lower(), release) else: id_str = distro_id return id_str.replace(' ','-').replace('/','-') diff --git a/meta/lib/oe/manifest.py b/meta/lib/oe/manifest.py index 95f8eb2df3..60c49be0e9 100644 --- a/meta/lib/oe/manifest.py +++ b/meta/lib/oe/manifest.py @@ -59,9 +59,9 @@ class Manifest(object, metaclass=ABCMeta): if manifest_dir is None: if manifest_type != self.MANIFEST_TYPE_IMAGE: - self.manifest_dir = self.d.getVar('SDK_DIR', True) + self.manifest_dir = self.d.getVar('SDK_DIR') else: - self.manifest_dir = self.d.getVar('WORKDIR', True) + self.manifest_dir = self.d.getVar('WORKDIR') else: self.manifest_dir = manifest_dir @@ -82,7 +82,7 @@ class Manifest(object, metaclass=ABCMeta): This will be used for testing until the class is implemented properly! """ def _create_dummy_initial(self): - image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) + image_rootfs = self.d.getVar('IMAGE_ROOTFS') pkg_list = dict() if image_rootfs.find("core-image-sato-sdk") > 0: pkg_list[self.PKG_TYPE_MUST_INSTALL] = \ @@ -104,7 +104,7 @@ class Manifest(object, metaclass=ABCMeta): pkg_list['lgp'] = \ "locale-base-en-us locale-base-en-gb" elif image_rootfs.find("core-image-minimal") > 0: - pkg_list[self.PKG_TYPE_MUST_INSTALL] = "run-postinsts packagegroup-core-boot" + pkg_list[self.PKG_TYPE_MUST_INSTALL] = "packagegroup-core-boot" with open(self.initial_manifest, "w+") as manifest: manifest.write(self.initial_manifest_file_header) @@ -195,7 +195,7 @@ class RpmManifest(Manifest): for pkg in pkg_list.split(): pkg_type = self.PKG_TYPE_MUST_INSTALL - ml_variants = self.d.getVar('MULTILIB_VARIANTS', True).split() + ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() for ml_variant in ml_variants: if pkg.startswith(ml_variant + '-'): @@ -216,13 +216,13 @@ class RpmManifest(Manifest): for var in self.var_maps[self.manifest_type]: if var in self.vars_to_split: - split_pkgs = self._split_multilib(self.d.getVar(var, True)) + split_pkgs = self._split_multilib(self.d.getVar(var)) if split_pkgs is not None: pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: - pkg_list = self.d.getVar(var, True) + pkg_list = self.d.getVar(var) if pkg_list is not None: - pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var, True) + pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) for pkg_type in pkgs: for pkg in pkgs[pkg_type].split(): @@ -245,7 +245,7 @@ class OpkgManifest(Manifest): for pkg in pkg_list.split(): pkg_type = self.PKG_TYPE_MUST_INSTALL - ml_variants = self.d.getVar('MULTILIB_VARIANTS', True).split() + ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() for ml_variant in ml_variants: if pkg.startswith(ml_variant + '-'): @@ -266,13 +266,13 @@ class OpkgManifest(Manifest): for var in self.var_maps[self.manifest_type]: if var in self.vars_to_split: - split_pkgs = self._split_multilib(self.d.getVar(var, True)) + split_pkgs = self._split_multilib(self.d.getVar(var)) if split_pkgs is not None: pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: - pkg_list = self.d.getVar(var, True) + pkg_list = self.d.getVar(var) if pkg_list is not None: - pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var, True) + pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) for pkg_type in pkgs: for pkg in pkgs[pkg_type].split(): @@ -310,7 +310,7 @@ class DpkgManifest(Manifest): manifest.write(self.initial_manifest_file_header) for var in self.var_maps[self.manifest_type]: - pkg_list = self.d.getVar(var, True) + pkg_list = self.d.getVar(var) if pkg_list is None: continue @@ -332,7 +332,7 @@ def create_manifest(d, final_manifest=False, manifest_dir=None, 'ipk': OpkgManifest, 'deb': DpkgManifest} - manifest = manifest_map[d.getVar('IMAGE_PKGTYPE', True)](d, manifest_dir, manifest_type) + manifest = manifest_map[d.getVar('IMAGE_PKGTYPE')](d, manifest_dir, manifest_type) if final_manifest: manifest.create_final() diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index faa0ab2edb..52c5f16cf8 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -18,23 +18,24 @@ def runstrip(arg): newmode = origmode | stat.S_IWRITE | stat.S_IREAD os.chmod(file, newmode) - extraflags = "" + stripcmd = [strip] # kernel module if elftype & 16: - extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates" + stripcmd.extend(["--strip-debug", "--remove-section=.comment", + "--remove-section=.note", "--preserve-dates"]) # .so and shared library elif ".so" in file and elftype & 8: - extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded" + stripcmd.extend(["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"]) # shared or executable: elif elftype & 8 or elftype & 4: - extraflags = "--remove-section=.comment --remove-section=.note" + stripcmd.extend(["--remove-section=.comment", "--remove-section=.note"]) - stripcmd = "'%s' %s '%s'" % (strip, extraflags, file) + stripcmd.append(file) bb.debug(1, "runstrip: %s" % stripcmd) try: - output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.error("runstrip: '%s' strip command failed with %s (%s)" % (stripcmd, e.returncode, e.output)) @@ -56,7 +57,7 @@ def file_translate(file): def filedeprunner(arg): import re, subprocess, shlex - (pkg, pkgfiles, rpmdeps, pkgdest) = arg + (pkg, pkgfiles, rpmdeps, pkgdest, magic) = arg provides = {} requires = {} @@ -89,8 +90,11 @@ def filedeprunner(arg): return provides, requires + env = os.environ.copy() + env["MAGIC"] = magic + try: - dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE) + dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE, env=env) provides, requires = process_deps(dep_popen.stdout, pkg, pkgdest, provides, requires) except OSError as e: bb.error("rpmdeps: '%s' command failed, '%s'" % (shlex.split(rpmdeps) + pkgfiles, e)) @@ -103,7 +107,7 @@ def read_shlib_providers(d): import re shlib_provider = {} - shlibs_dirs = d.getVar('SHLIBSDIRS', True).split() + shlibs_dirs = d.getVar('SHLIBSDIRS').split() list_re = re.compile('^(.*)\.list$') # Go from least to most specific since the last one found wins for dir in reversed(shlibs_dirs): @@ -114,7 +118,12 @@ def read_shlib_providers(d): m = list_re.match(file) if m: dep_pkg = m.group(1) - fd = open(os.path.join(dir, file)) + try: + fd = open(os.path.join(dir, file)) + except IOError: + # During a build unrelated shlib files may be deleted, so + # handle files disappearing between the listdirs and open. + continue lines = fd.readlines() fd.close() for l in lines: @@ -144,6 +153,7 @@ def npm_split_package_dirs(pkgdir): continue pkgitems.append(pathitem) pkgname = '-'.join(pkgitems).replace('_', '-') + pkgname = pkgname.replace('@', '') pkgfile = os.path.join(root, dn, 'package.json') data = None if os.path.exists(pkgfile): diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py index 71e5b502e7..f1b65bdbbc 100644 --- a/meta/lib/oe/package_manager.py +++ b/meta/lib/oe/package_manager.py @@ -5,9 +5,11 @@ import subprocess import shutil import multiprocessing import re +import collections import bb import tempfile import oe.utils +import oe.path import string from oe.gpg_sign import get_signer @@ -100,113 +102,14 @@ class Indexer(object, metaclass=ABCMeta): class RpmIndexer(Indexer): - def get_ml_prefix_and_os_list(self, arch_var=None, os_var=None): - package_archs = { - 'default': [], - } - - target_os = { - 'default': "", - } - - if arch_var is not None and os_var is not None: - package_archs['default'] = self.d.getVar(arch_var, True).split() - package_archs['default'].reverse() - target_os['default'] = self.d.getVar(os_var, True).strip() - else: - package_archs['default'] = self.d.getVar("PACKAGE_ARCHS", True).split() - # arch order is reversed. This ensures the -best- match is - # listed first! - package_archs['default'].reverse() - target_os['default'] = self.d.getVar("TARGET_OS", True).strip() - multilibs = self.d.getVar('MULTILIBS', True) or "" - for ext in multilibs.split(): - eext = ext.split(':') - if len(eext) > 1 and eext[0] == 'multilib': - localdata = bb.data.createCopy(self.d) - default_tune_key = "DEFAULTTUNE_virtclass-multilib-" + eext[1] - default_tune = localdata.getVar(default_tune_key, False) - if default_tune is None: - default_tune_key = "DEFAULTTUNE_ML_" + eext[1] - default_tune = localdata.getVar(default_tune_key, False) - if default_tune: - localdata.setVar("DEFAULTTUNE", default_tune) - bb.data.update_data(localdata) - package_archs[eext[1]] = localdata.getVar('PACKAGE_ARCHS', - True).split() - package_archs[eext[1]].reverse() - target_os[eext[1]] = localdata.getVar("TARGET_OS", - True).strip() - - ml_prefix_list = dict() - for mlib in package_archs: - if mlib == 'default': - ml_prefix_list[mlib] = package_archs[mlib] - else: - ml_prefix_list[mlib] = list() - for arch in package_archs[mlib]: - if arch in ['all', 'noarch', 'any']: - ml_prefix_list[mlib].append(arch) - else: - ml_prefix_list[mlib].append(mlib + "_" + arch) - - return (ml_prefix_list, target_os) - def write_index(self): - sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS', True) or "").replace('-', '_').split() - all_mlb_pkg_archs = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split() - - mlb_prefix_list = self.get_ml_prefix_and_os_list()[0] - - archs = set() - for item in mlb_prefix_list: - archs = archs.union(set(i.replace('-', '_') for i in mlb_prefix_list[item])) - - if len(archs) == 0: - archs = archs.union(set(all_mlb_pkg_archs)) - - archs = archs.union(set(sdk_pkg_archs)) - - rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo") - if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': - signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND', True)) - else: - signer = None - index_cmds = [] - repomd_files = [] - rpm_dirs_found = False - for arch in archs: - dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb', arch) - if os.path.exists(dbpath): - bb.utils.remove(dbpath, True) - arch_dir = os.path.join(self.deploy_dir, arch) - if not os.path.isdir(arch_dir): - continue + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + raise NotImplementedError('Package feed signing not yet implementd for rpm') - index_cmds.append("%s --dbpath %s --update -q %s" % \ - (rpm_createrepo, dbpath, arch_dir)) - repomd_files.append(os.path.join(arch_dir, 'repodata', 'repomd.xml')) - - rpm_dirs_found = True - - if not rpm_dirs_found: - bb.note("There are no packages in %s" % self.deploy_dir) - return - - # Create repodata - result = oe.utils.multiprocess_exec(index_cmds, create_index) + createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c") + result = create_index("%s --update -q %s" % (createrepo_c, self.deploy_dir)) if result: - bb.fatal('%s' % ('\n'.join(result))) - # Sign repomd - if signer: - for repomd in repomd_files: - feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE', True) - is_ascii_sig = (feed_sig_type.upper() != "BIN") - signer.detach_sign(repomd, - self.d.getVar('PACKAGE_FEED_GPG_NAME', True), - self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE', True), - armor=is_ascii_sig) - + bb.fatal(result) class OpkgIndexer(Indexer): def write_index(self): @@ -215,8 +118,8 @@ class OpkgIndexer(Indexer): "MULTILIB_ARCHS"] opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") - if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': - signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND', True)) + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) else: signer = None @@ -226,7 +129,7 @@ class OpkgIndexer(Indexer): index_cmds = set() index_sign_files = set() for arch_var in arch_vars: - archs = self.d.getVar(arch_var, True) + archs = self.d.getVar(arch_var) if archs is None: continue @@ -254,12 +157,12 @@ class OpkgIndexer(Indexer): bb.fatal('%s' % ('\n'.join(result))) if signer: - feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE', True) + feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') is_ascii_sig = (feed_sig_type.upper() != "BIN") for f in index_sign_files: signer.detach_sign(f, - self.d.getVar('PACKAGE_FEED_GPG_NAME', True), - self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE', True), + self.d.getVar('PACKAGE_FEED_GPG_NAME'), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), armor=is_ascii_sig) @@ -293,16 +196,16 @@ class DpkgIndexer(Indexer): os.environ['APT_CONFIG'] = self.apt_conf_file - pkg_archs = self.d.getVar('PACKAGE_ARCHS', True) + pkg_archs = self.d.getVar('PACKAGE_ARCHS') if pkg_archs is not None: arch_list = pkg_archs.split() - sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS', True) + sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS') if sdk_pkg_archs is not None: for a in sdk_pkg_archs.split(): if a not in pkg_archs: arch_list.append(a) - all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").split() + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list) apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive") @@ -335,7 +238,7 @@ class DpkgIndexer(Indexer): result = oe.utils.multiprocess_exec(index_cmds, create_index) if result: bb.fatal('%s' % ('\n'.join(result))) - if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': raise NotImplementedError('Package feed signing not implementd for dpkg') @@ -349,119 +252,9 @@ class PkgsList(object, metaclass=ABCMeta): def list_pkgs(self): pass - class RpmPkgsList(PkgsList): - def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): - super(RpmPkgsList, self).__init__(d, rootfs_dir) - - self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") - self.image_rpmlib = os.path.join(self.rootfs_dir, 'var/lib/rpm') - - self.ml_prefix_list, self.ml_os_list = \ - RpmIndexer(d, rootfs_dir).get_ml_prefix_and_os_list(arch_var, os_var) - - # Determine rpm version - cmd = "%s --version" % self.rpm_cmd - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") - except subprocess.CalledProcessError as e: - bb.fatal("Getting rpm version failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - - ''' - Translate the RPM/Smart format names to the OE multilib format names - ''' - def _pkg_translate_smart_to_oe(self, pkg, arch): - new_pkg = pkg - new_arch = arch - fixed_arch = arch.replace('_', '-') - found = 0 - for mlib in self.ml_prefix_list: - for cmp_arch in self.ml_prefix_list[mlib]: - fixed_cmp_arch = cmp_arch.replace('_', '-') - if fixed_arch == fixed_cmp_arch: - if mlib == 'default': - new_pkg = pkg - new_arch = cmp_arch - else: - new_pkg = mlib + '-' + pkg - # We need to strip off the ${mlib}_ prefix on the arch - new_arch = cmp_arch.replace(mlib + '_', '') - - # Workaround for bug 3565. Simply look to see if we - # know of a package with that name, if not try again! - filename = os.path.join(self.d.getVar('PKGDATA_DIR', True), - 'runtime-reverse', - new_pkg) - if os.path.exists(filename): - found = 1 - break - - if found == 1 and fixed_arch == fixed_cmp_arch: - break - #bb.note('%s, %s -> %s, %s' % (pkg, arch, new_pkg, new_arch)) - return new_pkg, new_arch - - def _list_pkg_deps(self): - cmd = [bb.utils.which(os.getenv('PATH'), "rpmresolve"), - "-t", self.image_rpmlib] - - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") - except subprocess.CalledProcessError as e: - bb.fatal("Cannot get the package dependencies. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - - return output - def list_pkgs(self): - cmd = self.rpm_cmd + ' --root ' + self.rootfs_dir - cmd += ' -D "_dbpath /var/lib/rpm" -qa' - cmd += " --qf '[%{NAME} %{ARCH} %{VERSION} %{PACKAGEORIGIN}\n]'" - - try: - # bb.note(cmd) - tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") - except subprocess.CalledProcessError as e: - bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - - output = dict() - deps = dict() - dependencies = self._list_pkg_deps() - - # Populate deps dictionary for better manipulation - for line in dependencies.splitlines(): - try: - pkg, dep = line.split("|") - if not pkg in deps: - deps[pkg] = list() - if not dep in deps[pkg]: - deps[pkg].append(dep) - except: - # Ignore any other lines they're debug or errors - pass - - for line in tmp_output.split('\n'): - if len(line.strip()) == 0: - continue - pkg = line.split()[0] - arch = line.split()[1] - ver = line.split()[2] - dep = deps.get(pkg, []) - - # Skip GPG keys - if pkg == 'gpg-pubkey': - continue - - pkgorigin = line.split()[3] - new_pkg, new_arch = self._pkg_translate_smart_to_oe(pkg, arch) - - output[new_pkg] = {"arch":new_arch, "ver":ver, - "filename":pkgorigin, "deps":dep} - - return output - + return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR')).list_installed() class OpkgPkgsList(PkgsList): def __init__(self, d, rootfs_dir, config_file): @@ -469,7 +262,7 @@ class OpkgPkgsList(PkgsList): self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) - self.opkg_args += self.d.getVar("OPKG_ARGS", True) + self.opkg_args += self.d.getVar("OPKG_ARGS") def list_pkgs(self, format=None): cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) @@ -517,9 +310,6 @@ class PackageManager(object, metaclass=ABCMeta): self.d = d self.deploy_dir = None self.deploy_lock = None - self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or "" - self.feed_base_paths = self.d.getVar('PACKAGE_FEED_BASE_PATHS', True) or "" - self.feed_archs = self.d.getVar('PACKAGE_FEED_ARCHS', True) """ Update the package manager package database. @@ -559,8 +349,24 @@ class PackageManager(object, metaclass=ABCMeta): def list_installed(self): pass + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + """ + @abstractmethod + def extract(self, pkg): + pass + + """ + Add remote package feeds into repository manager configuration. The parameters + for the feeds are set by feed_uris, feed_base_paths and feed_archs. + See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS + for their description. + """ @abstractmethod - def insert_feeds_uris(self): + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): pass """ @@ -571,20 +377,11 @@ class PackageManager(object, metaclass=ABCMeta): installation """ def install_complementary(self, globs=None): - # we need to write the list of installed packages to a file because the - # oe-pkgdata-util reads it from a file - installed_pkgs_file = os.path.join(self.d.getVar('WORKDIR', True), - "installed_pkgs.txt") - with open(installed_pkgs_file, "w+") as installed_pkgs: - pkgs = self.list_installed() - output = oe.utils.format_pkg_list(pkgs, "arch") - installed_pkgs.write(output) - if globs is None: - globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY', True) + globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY') split_linguas = set() - for translation in self.d.getVar('IMAGE_LINGUAS', True).split(): + for translation in self.d.getVar('IMAGE_LINGUAS').split(): split_linguas.add(translation) split_linguas.add(translation.split('-')[0]) @@ -596,22 +393,28 @@ class PackageManager(object, metaclass=ABCMeta): if globs is None: return - cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), - "-p", self.d.getVar('PKGDATA_DIR', True), "glob", installed_pkgs_file, - globs] - exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY', True) - if exclude: - cmd.extend(['-x', exclude]) - try: - bb.note("Installing complementary packages ...") - bb.note('Running %s' % cmd) - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") - except subprocess.CalledProcessError as e: - bb.fatal("Could not compute complementary packages list. Command " - "'%s' returned %d:\n%s" % - (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - self.install(complementary_pkgs.split(), attempt_only=True) - os.remove(installed_pkgs_file) + # we need to write the list of installed packages to a file because the + # oe-pkgdata-util reads it from a file + with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs: + pkgs = self.list_installed() + output = oe.utils.format_pkg_list(pkgs, "arch") + installed_pkgs.write(output) + + cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), + "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name, + globs] + exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY') + if exclude: + cmd.extend(['--exclude=' + '|'.join(exclude.split())]) + try: + bb.note("Installing complementary packages ...") + bb.note('Running %s' % cmd) + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not compute complementary packages list. Command " + "'%s' returned %d:\n%s" % + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + self.install(complementary_pkgs.split(), attempt_only=True) def deploy_dir_lock(self): if self.deploy_dir is None: @@ -638,8 +441,8 @@ class PackageManager(object, metaclass=ABCMeta): def construct_uris(self, uris, base_paths): def _append(arr1, arr2, sep='/'): res = [] - narr1 = [string.rstrip(a, sep) for a in arr1] - narr2 = [string.lstrip(string.rstrip(a, sep), sep) for a in arr2] + narr1 = [a.rstrip(sep) for a in arr1] + narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2] for a1 in narr1: if arr2: for a2 in narr2: @@ -662,805 +465,291 @@ class RpmPM(PackageManager): self.target_rootfs = target_rootfs self.target_vendor = target_vendor self.task_name = task_name - self.providename = providename - self.fullpkglist = list() - self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM', True) - self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm") - self.install_dir_name = "oe_install" - self.install_dir_path = os.path.join(self.target_rootfs, self.install_dir_name) - self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") - self.smart_cmd = bb.utils.which(os.getenv('PATH'), "smart") - # 0 = default, only warnings - # 1 = --log-level=info (includes information about executing scriptlets and their output) - # 2 = --log-level=debug - # 3 = --log-level=debug plus dumps of scriplet content and command invocation - self.debug_level = int(d.getVar('ROOTFS_RPM_DEBUG', True) or "0") - self.smart_opt = "--log-level=%s --data-dir=%s" % \ - ("warning" if self.debug_level == 0 else - "info" if self.debug_level == 1 else - "debug", - os.path.join(target_rootfs, 'var/lib/smart')) - self.scriptlet_wrapper = self.d.expand('${WORKDIR}/scriptlet_wrapper') + if arch_var == None: + self.archs = self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS').replace("-","_") + else: + self.archs = self.d.getVar(arch_var).replace("-","_") + if task_name == "host": + self.primary_arch = self.d.getVar('SDK_ARCH') + else: + self.primary_arch = self.d.getVar('MACHINE_ARCH') + + self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), "oe-rootfs-repo") + bb.utils.mkdirhier(self.rpm_repo_dir) + oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True) + + self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name) + if not os.path.exists(self.d.expand('${T}/saved_packaging_data')): + bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data')) + self.packaging_data_dirs = ['var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf'] self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % self.task_name) - self.saved_rpmlib = self.d.expand('${T}/saved/%s' % self.task_name) - self.image_rpmlib = os.path.join(self.target_rootfs, 'var/lib/rpm') - if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.indexer = RpmIndexer(self.d, self.deploy_dir) - self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, arch_var, os_var) - - self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) - - def insert_feeds_uris(self): - if self.feed_uris == "": - return - - arch_list = [] - if self.feed_archs is not None: - # User define feed architectures - arch_list = self.feed_archs.split() + def _configure_dnf(self): + # libsolv handles 'noarch' internally, we don't need to specify it explicitly + archs = [i for i in reversed(self.archs.split()) if i not in ["any", "all", "noarch"]] + # This prevents accidental matching against libsolv's built-in policies + if len(archs) <= 1: + archs = archs + ["bogusarch"] + archconfdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/") + bb.utils.mkdirhier(archconfdir) + open(archconfdir + "arch", 'w').write(":".join(archs)) + + open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w').write("") + + + def _configure_rpm(self): + # We need to configure rpm to use our primary package architecture as the installation architecture, + # and to make it compatible with other package architectures that we use. + # Otherwise it will refuse to proceed with packages installation. + platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/") + rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/") + bb.utils.mkdirhier(platformconfdir) + open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch) + open(rpmrcconfdir + "rpmrc", 'w').write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch)) + + open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n") + if self.d.getVar('RPM_PREFER_ELF_ARCH'): + open(platformconfdir + "macros", 'a').write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH'))) else: - # List must be prefered to least preferred order - default_platform_extra = set() - platform_extra = set() - bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" - for mlib in self.ml_os_list: - for arch in self.ml_prefix_list[mlib]: - plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] - if mlib == bbextendvariant: - default_platform_extra.add(plt) - else: - platform_extra.add(plt) + open(platformconfdir + "macros", 'a').write("%_prefer_color 7") + + if self.d.getVar('RPM_SIGN_PACKAGES') == '1': + signer = get_signer(self.d, self.d.getVar('RPM_GPG_BACKEND')) + pubkey_path = oe.path.join(self.d.getVar('B'), 'rpm-key') + signer.export_pubkey(pubkey_path, self.d.getVar('RPM_GPG_NAME')) + rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmkeys") + cmd = [rpm_bin, '--root=%s' % self.target_rootfs, '--import', pubkey_path] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Importing GPG key failed. Command '%s' " + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - platform_extra = platform_extra.union(default_platform_extra) + def create_configs(self): + self._configure_dnf() + self._configure_rpm() - for canonical_arch in platform_extra: - arch = canonical_arch.split('-')[0] - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - arch_list.append(arch) + def write_index(self): + lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock" + lf = bb.utils.lockfile(lockfilename, False) + RpmIndexer(self.d, self.rpm_repo_dir).write_index() + bb.utils.unlockfile(lf) - feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split()) - - uri_iterator = 0 - channel_priority = 10 + 5 * len(feed_uris) * (len(arch_list) if arch_list else 1) - - for uri in feed_uris: - if arch_list: - for arch in arch_list: - bb.note('Adding Smart channel url%d%s (%s)' % - (uri_iterator, arch, channel_priority)) - self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y' - % (uri_iterator, arch, uri, arch)) - self._invoke_smart('channel --set url%d-%s priority=%d' % - (uri_iterator, arch, channel_priority)) - channel_priority -= 5 - else: - bb.note('Adding Smart channel url%d (%s)' % - (uri_iterator, channel_priority)) - self._invoke_smart('channel --add url%d type=rpm-md baseurl=%s -y' - % (uri_iterator, uri)) - self._invoke_smart('channel --set url%d priority=%d' % - (uri_iterator, channel_priority)) - channel_priority -= 5 + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + from urllib.parse import urlparse - uri_iterator += 1 + if feed_uris == "": + return - ''' - Create configs for rpm and smart, and multilib is supported - ''' - def create_configs(self): - target_arch = self.d.getVar('TARGET_ARCH', True) - platform = '%s%s-%s' % (target_arch.replace('-', '_'), - self.target_vendor, - self.ml_os_list['default']) - - # List must be prefered to least preferred order - default_platform_extra = list() - platform_extra = list() - bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" - for mlib in self.ml_os_list: - for arch in self.ml_prefix_list[mlib]: - plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] - if mlib == bbextendvariant: - if plt not in default_platform_extra: - default_platform_extra.append(plt) - else: - if plt not in platform_extra: - platform_extra.append(plt) - platform_extra = default_platform_extra + platform_extra + bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d")) + remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + for uri in remote_uris: + repo_base = "oe-remote-repo" + "-".join(urlparse(uri).path.split("/")) + if feed_archs is not None: + for arch in feed_archs.split(): + repo_uri = uri + "/" + arch + repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/")) + repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/")) + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write( + "[%s]\nname=%s\nbaseurl=%s\n\n" % (repo_id, repo_name, repo_uri)) + else: + repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/")) + repo_uri = uri + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write( + "[%s]\nname=%s\nbaseurl=%s\n" % (repo_base, repo_name, repo_uri)) - self._create_configs(platform, platform_extra) + def _prepare_pkg_transaction(self): + os.environ['D'] = self.target_rootfs + os.environ['OFFLINE_ROOT'] = self.target_rootfs + os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['INTERCEPT_DIR'] = oe.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') - def _invoke_smart(self, args): - cmd = "%s %s %s" % (self.smart_cmd, self.smart_opt, args) - # bb.note(cmd) - try: - complementary_pkgs = subprocess.check_output(cmd, - stderr=subprocess.STDOUT, - shell=True).decode("utf-8") - # bb.note(complementary_pkgs) - return complementary_pkgs - except subprocess.CalledProcessError as e: - bb.fatal("Could not invoke smart. Command " - "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - - def _search_pkg_name_in_feeds(self, pkg, feed_archs): - for arch in feed_archs: - arch = arch.replace('-', '_') - regex_match = re.compile(r"^%s-[^-]*-[^-]*@%s$" % \ - (re.escape(pkg), re.escape(arch))) - for p in self.fullpkglist: - if regex_match.match(p) is not None: - # First found is best match - # bb.note('%s -> %s' % (pkg, pkg + '@' + arch)) - return pkg + '@' + arch - - # Search provides if not found by pkgname. - bb.note('Not found %s by name, searching provides ...' % pkg) - cmd = "%s %s query --provides %s --show-format='$name-$version'" % \ - (self.smart_cmd, self.smart_opt, pkg) - cmd += " | sed -ne 's/ *Provides://p'" - bb.note('cmd: %s' % cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") - # Found a provider - if output: - bb.note('Found providers for %s: %s' % (pkg, output)) - for p in output.split(): - for arch in feed_archs: - arch = arch.replace('-', '_') - if p.rstrip().endswith('@' + arch): - return p - - return "" - ''' - Translate the OE multilib format names to the RPM/Smart format names - It searched the RPM/Smart format names in probable multilib feeds first, - and then searched the default base feed. - ''' - def _pkg_translate_oe_to_smart(self, pkgs, attempt_only=False): - new_pkgs = list() - - for pkg in pkgs: - new_pkg = pkg - # Search new_pkg in probable multilibs first - for mlib in self.ml_prefix_list: - # Jump the default archs - if mlib == 'default': - continue + def install(self, pkgs, attempt_only = False): + if len(pkgs) == 0: + return + self._prepare_pkg_transaction() - subst = pkg.replace(mlib + '-', '') - # if the pkg in this multilib feed - if subst != pkg: - feed_archs = self.ml_prefix_list[mlib] - new_pkg = self._search_pkg_name_in_feeds(subst, feed_archs) - if not new_pkg: - # Failed to translate, package not found! - err_msg = '%s not found in the %s feeds (%s).\n' % \ - (pkg, mlib, " ".join(feed_archs)) - if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) - bb.warn(err_msg) - else: - new_pkgs.append(new_pkg) - - break - - # Apparently not a multilib package... - if pkg == new_pkg: - # Search new_pkg in default archs - default_archs = self.ml_prefix_list['default'] - new_pkg = self._search_pkg_name_in_feeds(pkg, default_archs) - if not new_pkg: - err_msg = '%s not found in the base feeds (%s).\n' % \ - (pkg, ' '.join(default_archs)) - if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) - bb.warn(err_msg) - else: - new_pkgs.append(new_pkg) - - return new_pkgs - - def _create_configs(self, platform, platform_extra): - # Setup base system configuration - bb.note("configuring RPM platform settings") - - # Configure internal RPM environment when using Smart - os.environ['RPM_ETCRPM'] = self.etcrpm_dir - bb.utils.mkdirhier(self.etcrpm_dir) - - # Setup temporary directory -- install... - if os.path.exists(self.install_dir_path): - bb.utils.remove(self.install_dir_path, True) - bb.utils.mkdirhier(os.path.join(self.install_dir_path, 'tmp')) - - channel_priority = 5 - platform_dir = os.path.join(self.etcrpm_dir, "platform") - sdkos = self.d.getVar("SDK_OS", True) - with open(platform_dir, "w+") as platform_fd: - platform_fd.write(platform + '\n') - for pt in platform_extra: - channel_priority += 5 - if sdkos: - tmp = re.sub("-%s$" % sdkos, "-%s\n" % sdkos, pt) - tmp = re.sub("-linux.*$", "-linux.*\n", tmp) - platform_fd.write(tmp) - - # Tell RPM that the "/" directory exist and is available - bb.note("configuring RPM system provides") - sysinfo_dir = os.path.join(self.etcrpm_dir, "sysinfo") - bb.utils.mkdirhier(sysinfo_dir) - with open(os.path.join(sysinfo_dir, "Dirnames"), "w+") as dirnames: - dirnames.write("/\n") - - if self.providename: - providename_dir = os.path.join(sysinfo_dir, "Providename") - if not os.path.exists(providename_dir): - providename_content = '\n'.join(self.providename) - providename_content += '\n' - open(providename_dir, "w+").write(providename_content) - - # Configure RPM... we enforce these settings! - bb.note("configuring RPM DB settings") - # After change the __db.* cache size, log file will not be - # generated automatically, that will raise some warnings, - # so touch a bare log for rpm write into it. - rpmlib_log = os.path.join(self.image_rpmlib, 'log', 'log.0000000001') - if not os.path.exists(rpmlib_log): - bb.utils.mkdirhier(os.path.join(self.image_rpmlib, 'log')) - open(rpmlib_log, 'w+').close() - - DB_CONFIG_CONTENT = "# ================ Environment\n" \ - "set_data_dir .\n" \ - "set_create_dir .\n" \ - "set_lg_dir ./log\n" \ - "set_tmp_dir ./tmp\n" \ - "set_flags db_log_autoremove on\n" \ - "\n" \ - "# -- thread_count must be >= 8\n" \ - "set_thread_count 64\n" \ - "\n" \ - "# ================ Logging\n" \ - "\n" \ - "# ================ Memory Pool\n" \ - "set_cachesize 0 1048576 0\n" \ - "set_mp_mmapsize 268435456\n" \ - "\n" \ - "# ================ Locking\n" \ - "set_lk_max_locks 16384\n" \ - "set_lk_max_lockers 16384\n" \ - "set_lk_max_objects 16384\n" \ - "mutex_set_max 163840\n" \ - "\n" \ - "# ================ Replication\n" - - db_config_dir = os.path.join(self.image_rpmlib, 'DB_CONFIG') - if not os.path.exists(db_config_dir): - open(db_config_dir, 'w+').write(DB_CONFIG_CONTENT) - - # Create database so that smart doesn't complain (lazy init) - opt = "-qa" - cmd = "%s --root %s --dbpath /var/lib/rpm %s > /dev/null" % ( - self.rpm_cmd, self.target_rootfs, opt) - try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as e: - bb.fatal("Create rpm database failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - # Import GPG key to RPM database of the target system - if self.d.getVar('RPM_SIGN_PACKAGES', True) == '1': - pubkey_path = self.d.getVar('RPM_GPG_PUBKEY', True) - cmd = "%s --root %s --dbpath /var/lib/rpm --import %s > /dev/null" % ( - self.rpm_cmd, self.target_rootfs, pubkey_path) - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + bad_recommendations = self.d.getVar('BAD_RECOMMENDATIONS') + package_exclude = self.d.getVar('PACKAGE_EXCLUDE') + exclude_pkgs = (bad_recommendations.split() if bad_recommendations else []) + (package_exclude.split() if package_exclude else []) - # Configure smart - bb.note("configuring Smart settings") - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), - True) - self._invoke_smart('config --set rpm-root=%s' % self.target_rootfs) - self._invoke_smart('config --set rpm-dbpath=/var/lib/rpm') - self._invoke_smart('config --set rpm-extra-macros._var=%s' % - self.d.getVar('localstatedir', True)) - cmd = "config --set rpm-extra-macros._tmppath=/%s/tmp" % (self.install_dir_name) - - prefer_color = self.d.getVar('RPM_PREFER_ELF_ARCH', True) - if prefer_color: - if prefer_color not in ['0', '1', '2', '4']: - bb.fatal("Invalid RPM_PREFER_ELF_ARCH: %s, it should be one of:\n" - "\t1: ELF32 wins\n" - "\t2: ELF64 wins\n" - "\t4: ELF64 N32 wins (mips64 or mips64el only)" % - prefer_color) - if prefer_color == "4" and self.d.getVar("TUNE_ARCH", True) not in \ - ['mips64', 'mips64el']: - bb.fatal("RPM_PREFER_ELF_ARCH = \"4\" is for mips64 or mips64el " - "only.") - self._invoke_smart('config --set rpm-extra-macros._prefer_color=%s' - % prefer_color) - - self._invoke_smart(cmd) - self._invoke_smart('config --set rpm-ignoresize=1') - - # Write common configuration for host and target usage - self._invoke_smart('config --set rpm-nolinktos=1') - self._invoke_smart('config --set rpm-noparentdirs=1') - check_signature = self.d.getVar('RPM_CHECK_SIGNATURES', True) - if check_signature and check_signature.strip() == "0": - self._invoke_smart('config --set rpm-check-signatures=false') - for i in self.d.getVar('BAD_RECOMMENDATIONS', True).split(): - self._invoke_smart('flag --set ignore-recommends %s' % i) - - # Do the following configurations here, to avoid them being - # saved for field upgrade - if self.d.getVar('NO_RECOMMENDATIONS', True).strip() == "1": - self._invoke_smart('config --set ignore-all-recommends=1') - pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE', True) or "" - for i in pkg_exclude.split(): - self._invoke_smart('flag --set exclude-packages %s' % i) - - # Optional debugging - # self._invoke_smart('config --set rpm-log-level=debug') - # cmd = 'config --set rpm-log-file=/tmp/smart-debug-logfile' - # self._invoke_smart(cmd) - ch_already_added = [] - for canonical_arch in platform_extra: - arch = canonical_arch.split('-')[0] - arch_channel = os.path.join(self.deploy_dir, arch) - if os.path.exists(arch_channel) and not arch in ch_already_added: - bb.note('Adding Smart channel %s (%s)' % - (arch, channel_priority)) - self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y' - % (arch, arch_channel)) - self._invoke_smart('channel --set %s priority=%d' % - (arch, channel_priority)) - channel_priority -= 5 - - ch_already_added.append(arch) - - bb.note('adding Smart RPM DB channel') - self._invoke_smart('channel --add rpmsys type=rpm-sys -y') - - # Construct install scriptlet wrapper. - # Scripts need to be ordered when executed, this ensures numeric order. - # If we ever run into needing more the 899 scripts, we'll have to. - # change num to start with 1000. - # - scriptletcmd = "$2 $1/$3 $4\n" - scriptpath = "$1/$3" - - # When self.debug_level >= 3, also dump the content of the - # executed scriptlets and how they get invoked. We have to - # replace "exit 1" and "ERR" because printing those as-is - # would trigger a log analysis failure. - if self.debug_level >= 3: - dump_invocation = 'echo "Executing ${name} ${kind} with: ' + scriptletcmd + '"\n' - dump_script = 'cat ' + scriptpath + '| sed -e "s/exit 1/exxxit 1/g" -e "s/ERR/IRR/g"; echo\n' - else: - dump_invocation = 'echo "Executing ${name} ${kind}"\n' - dump_script = '' - - SCRIPTLET_FORMAT = "#!/bin/bash\n" \ - "\n" \ - "export PATH=%s\n" \ - "export D=%s\n" \ - 'export OFFLINE_ROOT="$D"\n' \ - 'export IPKG_OFFLINE_ROOT="$D"\n' \ - 'export OPKG_OFFLINE_ROOT="$D"\n' \ - "export INTERCEPT_DIR=%s\n" \ - "export NATIVE_ROOT=%s\n" \ - "\n" \ - "name=`head -1 " + scriptpath + " | cut -d\' \' -f 2`\n" \ - "kind=`head -1 " + scriptpath + " | cut -d\' \' -f 4`\n" \ - + dump_invocation \ - + dump_script \ - + scriptletcmd + \ - "ret=$?\n" \ - "echo Result of ${name} ${kind}: ${ret}\n" \ - "if [ ${ret} -ne 0 ]; then\n" \ - " if [ $4 -eq 1 ]; then\n" \ - " mkdir -p $1/etc/rpm-postinsts\n" \ - " num=100\n" \ - " while [ -e $1/etc/rpm-postinsts/${num}-* ]; do num=$((num + 1)); done\n" \ - ' echo "#!$2" > $1/etc/rpm-postinsts/${num}-${name}\n' \ - ' echo "# Arg: $4" >> $1/etc/rpm-postinsts/${num}-${name}\n' \ - " cat " + scriptpath + " >> $1/etc/rpm-postinsts/${num}-${name}\n" \ - " chmod +x $1/etc/rpm-postinsts/${num}-${name}\n" \ - ' echo "Info: deferring ${name} ${kind} install scriptlet to first boot"\n' \ - " else\n" \ - ' echo "Error: ${name} ${kind} remove scriptlet failed"\n' \ - " fi\n" \ - "fi\n" - - intercept_dir = self.d.expand('${WORKDIR}/intercept_scripts') - native_root = self.d.getVar('STAGING_DIR_NATIVE', True) - scriptlet_content = SCRIPTLET_FORMAT % (os.environ['PATH'], - self.target_rootfs, - intercept_dir, - native_root) - open(self.scriptlet_wrapper, 'w+').write(scriptlet_content) - - bb.note("configuring RPM cross-install scriptlet_wrapper") - os.chmod(self.scriptlet_wrapper, 0o755) - cmd = 'config --set rpm-extra-macros._cross_scriptlet_wrapper=%s' % \ - self.scriptlet_wrapper - self._invoke_smart(cmd) - - # Debug to show smart config info - # bb.note(self._invoke_smart('config --show')) + output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) + + (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) + + (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) + + (["--nogpgcheck"] if self.d.getVar('RPM_SIGN_PACKAGES') != '1' else ["--setopt=gpgcheck=True"]) + + ["install"] + + pkgs) - def update(self): - self._invoke_smart('update rpmsys') - - def get_rdepends_recursively(self, pkgs): - # pkgs will be changed during the loop, so use [:] to make a copy. - for pkg in pkgs[:]: - sub_data = oe.packagedata.read_subpkgdata(pkg, self.d) - sub_rdep = sub_data.get("RDEPENDS_" + pkg) - if not sub_rdep: - continue - done = list(bb.utils.explode_dep_versions2(sub_rdep).keys()) - next = done - # Find all the rdepends on dependency chain - while next: - new = [] - for sub_pkg in next: - sub_data = oe.packagedata.read_subpkgdata(sub_pkg, self.d) - sub_pkg_rdep = sub_data.get("RDEPENDS_" + sub_pkg) - if not sub_pkg_rdep: - continue - for p in bb.utils.explode_dep_versions2(sub_pkg_rdep): - # Already handled, skip it. - if p in done or p in pkgs: - continue - # It's a new dep - if oe.packagedata.has_subpkgdata(p, self.d): - done.append(p) - new.append(p) - next = new - pkgs.extend(done) - return pkgs + failed_scriptlets_pkgnames = collections.OrderedDict() + for line in output.splitlines(): + if line.startswith("Non-fatal POSTIN scriptlet failure in rpm package"): + failed_scriptlets_pkgnames[line.split()[-1]] = True - ''' - Install pkgs with smart, the pkg name is oe format - ''' - def install(self, pkgs, attempt_only=False): + for pkg in failed_scriptlets_pkgnames.keys(): + self.save_rpmpostinst(pkg) - if not pkgs: - bb.note("There are no packages to install") + def remove(self, pkgs, with_dependencies = True): + if len(pkgs) == 0: return - bb.note("Installing the following packages: %s" % ' '.join(pkgs)) - if not attempt_only: - # Pull in multilib requires since rpm may not pull in them - # correctly, for example, - # lib32-packagegroup-core-standalone-sdk-target requires - # lib32-libc6, but rpm may pull in libc6 rather than lib32-libc6 - # since it doesn't know mlprefix (lib32-), bitbake knows it and - # can handle it well, find out the RDEPENDS on the chain will - # fix the problem. Both do_rootfs and do_populate_sdk have this - # issue. - # The attempt_only packages don't need this since they are - # based on the installed ones. - # - # Separate pkgs into two lists, one is multilib, the other one - # is non-multilib. - ml_pkgs = [] - non_ml_pkgs = pkgs[:] - for pkg in pkgs: - for mlib in (self.d.getVar("MULTILIB_VARIANTS", True) or "").split(): - if pkg.startswith(mlib + '-'): - ml_pkgs.append(pkg) - non_ml_pkgs.remove(pkg) - - if len(ml_pkgs) > 0 and len(non_ml_pkgs) > 0: - # Found both foo and lib-foo - ml_pkgs = self.get_rdepends_recursively(ml_pkgs) - non_ml_pkgs = self.get_rdepends_recursively(non_ml_pkgs) - # Longer list makes smart slower, so only keep the pkgs - # which have the same BPN, and smart can handle others - # correctly. - pkgs_new = [] - for pkg in non_ml_pkgs: - for mlib in (self.d.getVar("MULTILIB_VARIANTS", True) or "").split(): - mlib_pkg = mlib + "-" + pkg - if mlib_pkg in ml_pkgs: - pkgs_new.append(pkg) - pkgs_new.append(mlib_pkg) - for pkg in pkgs: - if pkg not in pkgs_new: - pkgs_new.append(pkg) - pkgs = pkgs_new - new_depends = {} - deps = bb.utils.explode_dep_versions2(" ".join(pkgs)) - for depend in deps: - data = oe.packagedata.read_subpkgdata(depend, self.d) - key = "PKG_%s" % depend - if key in data: - new_depend = data[key] - else: - new_depend = depend - new_depends[new_depend] = deps[depend] - pkgs = bb.utils.join_deps(new_depends, commasep=True).split(', ') - pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only) - if not attempt_only: - bb.note('to be installed: %s' % ' '.join(pkgs)) - cmd = "%s %s install -y %s" % \ - (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) - bb.note(cmd) - else: - bb.note('installing attempt only packages...') - bb.note('Attempting %s' % ' '.join(pkgs)) - cmd = "%s %s install --attempt -y %s" % \ - (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) - try: - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") - bb.note(output) - except subprocess.CalledProcessError as e: - bb.fatal("Unable to install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + self._prepare_pkg_transaction() - ''' - Remove pkgs with smart, the pkg name is smart/rpm format - ''' - def remove(self, pkgs, with_dependencies=True): - bb.note('to be removed: ' + ' '.join(pkgs)) - - if not with_dependencies: - cmd = "%s -e --nodeps " % self.rpm_cmd - cmd += "--root=%s " % self.target_rootfs - cmd += "--dbpath=/var/lib/rpm " - cmd += "--define='_cross_scriptlet_wrapper %s' " % \ - self.scriptlet_wrapper - cmd += "--define='_tmppath /%s/tmp' %s" % (self.install_dir_name, ' '.join(pkgs)) + if with_dependencies: + self._invoke_dnf(["remove"] + pkgs) else: - # for pkg in pkgs: - # bb.note('Debug: What required: %s' % pkg) - # bb.note(self._invoke_smart('query %s --show-requiredby' % pkg)) + cmd = bb.utils.which(os.getenv('PATH'), "rpm") + args = ["-e", "--nodeps", "--root=%s" %self.target_rootfs] - cmd = "%s %s remove -y %s" % (self.smart_cmd, - self.smart_opt, - ' '.join(pkgs)) - - try: - bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") - bb.note(output) - except subprocess.CalledProcessError as e: - bb.note("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + try: + output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not invoke rpm. Command " + "'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8"))) def upgrade(self): - bb.note('smart upgrade') - self._invoke_smart('upgrade') + self._prepare_pkg_transaction() + self._invoke_dnf(["upgrade"]) - def write_index(self): - result = self.indexer.write_index() - - if result is not None: - bb.fatal(result) + def autoremove(self): + self._prepare_pkg_transaction() + self._invoke_dnf(["autoremove"]) def remove_packaging_data(self): - bb.utils.remove(self.image_rpmlib, True) - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), - True) - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/opkg'), True) - - # remove temp directory - bb.utils.remove(self.install_dir_path, True) + self._invoke_dnf(["clean", "all"]) + for dir in self.packaging_data_dirs: + bb.utils.remove(oe.path.join(self.target_rootfs, dir), True) def backup_packaging_data(self): - # Save the rpmlib for increment rpm image generation - if os.path.exists(self.saved_rpmlib): - bb.utils.remove(self.saved_rpmlib, True) - shutil.copytree(self.image_rpmlib, - self.saved_rpmlib, - symlinks=True) + # Save the packaging dirs for increment rpm image generation + if os.path.exists(self.saved_packaging_data): + bb.utils.remove(self.saved_packaging_data, True) + for i in self.packaging_data_dirs: + source_dir = oe.path.join(self.target_rootfs, i) + target_dir = oe.path.join(self.saved_packaging_data, i) + shutil.copytree(source_dir, target_dir, symlinks=True) def recovery_packaging_data(self): # Move the rpmlib back - if os.path.exists(self.saved_rpmlib): - if os.path.exists(self.image_rpmlib): - bb.utils.remove(self.image_rpmlib, True) - - bb.note('Recovery packaging data') - shutil.copytree(self.saved_rpmlib, - self.image_rpmlib, + if os.path.exists(self.saved_packaging_data): + for i in self.packaging_data_dirs: + target_dir = oe.path.join(self.target_rootfs, i) + if os.path.exists(target_dir): + bb.utils.remove(target_dir, True) + source_dir = oe.path.join(self.saved_packaging_data, i) + shutil.copytree(source_dir, + target_dir, symlinks=True) def list_installed(self): - return self.pkgs_list.list_pkgs() - - ''' - If incremental install, we need to determine what we've got, - what we need to add, and what to remove... - The dump_install_solution will dump and save the new install - solution. - ''' - def dump_install_solution(self, pkgs): - bb.note('creating new install solution for incremental install') - if len(pkgs) == 0: - return - - pkgs = self._pkg_translate_oe_to_smart(pkgs, False) - install_pkgs = list() + output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{sourcerpm}\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"], + print_output = False) + packages = {} + current_package = None + current_deps = None + current_state = "initial" + for line in output.splitlines(): + if line.startswith("Package:"): + package_info = line.split(" ")[1:] + current_package = package_info[0] + package_arch = package_info[1] + package_version = package_info[2] + package_srpm = package_info[3] + packages[current_package] = {"arch":package_arch, "ver":package_version, "filename":package_srpm} + current_deps = [] + elif line.startswith("Dependencies:"): + current_state = "dependencies" + elif line.startswith("Recommendations"): + current_state = "recommendations" + elif line.startswith("DependenciesEndHere:"): + current_state = "initial" + packages[current_package]["deps"] = current_deps + elif len(line) > 0: + if current_state == "dependencies": + current_deps.append(line) + elif current_state == "recommendations": + current_deps.append("%s [REC]" % line) + + return packages - cmd = "%s %s install -y --dump %s 2>%s" % \ - (self.smart_cmd, - self.smart_opt, - ' '.join(pkgs), - self.solution_manifest) + def update(self): + self._invoke_dnf(["makecache"]) + + def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ): + os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs + + dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf") + standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y", + "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), + "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")), + "--repofrompath=oe-repo,%s" % (self.rpm_repo_dir), + "--installroot=%s" % (self.target_rootfs), + "--setopt=logdir=%s" % (self.d.getVar('T')) + ] + cmd = [dnf_cmd] + standard_dnf_args + dnf_args try: - # Disable rpmsys channel for the fake install - self._invoke_smart('channel --disable rpmsys') - - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - with open(self.solution_manifest, 'r') as manifest: - for pkg in manifest.read().split('\n'): - if '@' in pkg: - install_pkgs.append(pkg) + output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8") + if print_output: + bb.note(output) + return output except subprocess.CalledProcessError as e: - bb.note("Unable to dump install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - # Recovery rpmsys channel - self._invoke_smart('channel --enable rpmsys') - return install_pkgs + if print_output: + (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command " + "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + else: + (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command " + "'%s' returned %d:" % (' '.join(cmd), e.returncode)) + return e.output.decode("utf-8") + + def dump_install_solution(self, pkgs): + open(self.solution_manifest, 'w').write(" ".join(pkgs)) + return pkgs - ''' - If incremental install, we need to determine what we've got, - what we need to add, and what to remove... - The load_old_install_solution will load the previous install - solution - ''' def load_old_install_solution(self): - bb.note('load old install solution for incremental install') - installed_pkgs = list() if not os.path.exists(self.solution_manifest): - bb.note('old install solution not exist') - return installed_pkgs - - with open(self.solution_manifest, 'r') as manifest: - for pkg in manifest.read().split('\n'): - if '@' in pkg: - installed_pkgs.append(pkg.strip()) - - return installed_pkgs + return [] - ''' - Dump all available packages in feeds, it should be invoked after the - newest rpm index was created - ''' - def dump_all_available_pkgs(self): - available_manifest = self.d.expand('${T}/saved/available_pkgs.txt') - available_pkgs = list() - cmd = "%s %s query --output %s" % \ - (self.smart_cmd, self.smart_opt, available_manifest) - try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - with open(available_manifest, 'r') as manifest: - for pkg in manifest.read().split('\n'): - if '@' in pkg: - available_pkgs.append(pkg.strip()) - except subprocess.CalledProcessError as e: - bb.note("Unable to list all available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - - self.fullpkglist = available_pkgs + return open(self.solution_manifest, 'r').read().split() - return + def _script_num_prefix(self, path): + files = os.listdir(path) + numbers = set() + numbers.add(99) + for f in files: + numbers.add(int(f.split("-")[0])) + return max(numbers) + 1 def save_rpmpostinst(self, pkg): - mlibs = (self.d.getVar('MULTILIB_GLOBAL_VARIANTS', False) or "").split() - - new_pkg = pkg - # Remove any multilib prefix from the package name - for mlib in mlibs: - if mlib in pkg: - new_pkg = pkg.replace(mlib + '-', '') - break - - bb.note(' * postponing %s' % new_pkg) - saved_dir = self.target_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') + new_pkg - - cmd = self.rpm_cmd + ' -q --scripts --root ' + self.target_rootfs - cmd += ' --dbpath=/var/lib/rpm ' + new_pkg - cmd += ' | sed -n -e "/^postinstall scriptlet (using .*):$/,/^.* scriptlet (using .*):$/ {/.*/p}"' - cmd += ' | sed -e "/postinstall scriptlet (using \(.*\)):$/d"' - cmd += ' -e "/^.* scriptlet (using .*):$/d" > %s' % saved_dir + bb.note("Saving postinstall script of %s" % (pkg)) + cmd = bb.utils.which(os.getenv('PATH'), "rpm") + args = ["-q", "--root=%s" % self.target_rootfs, "--queryformat", "%{postin}", pkg] try: - bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") - bb.note(output) - os.chmod(saved_dir, 0o755) + output = subprocess.check_output([cmd] + args,stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: - bb.fatal("Invoke save_rpmpostinst failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + bb.fatal("Could not invoke rpm. Command " + "'%s' returned %d:\n%s" % (' '.join([cmd] + args), e.returncode, e.output.decode("utf-8"))) - '''Write common configuration for target usage''' - def rpm_setup_smart_target_config(self): - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), - True) + # may need to prepend #!/bin/sh to output - self._invoke_smart('config --set rpm-nolinktos=1') - self._invoke_smart('config --set rpm-noparentdirs=1') - for i in self.d.getVar('BAD_RECOMMENDATIONS', True).split(): - self._invoke_smart('flag --set ignore-recommends %s' % i) - self._invoke_smart('channel --add rpmsys type=rpm-sys -y') + target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/')) + bb.utils.mkdirhier(target_path) + num = self._script_num_prefix(target_path) + saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg)) + open(saved_script_name, 'w').write(output) + os.chmod(saved_script_name, 0o755) - ''' - The rpm db lock files were produced after invoking rpm to query on - build system, and they caused the rpm on target didn't work, so we - need to unlock the rpm db by removing the lock files. - ''' - def unlock_rpm_db(self): - # Remove rpm db lock files - rpm_db_locks = glob.glob('%s/var/lib/rpm/__db.*' % self.target_rootfs) - for f in rpm_db_locks: - bb.utils.remove(f, True) - - """ - Returns a dictionary with the package info. - """ - def package_info(self, pkg): - cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg) - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as e: - bb.fatal("Unable to list available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) - - # Set default values to avoid UnboundLocalError - arch = "" - ver = "" - filename = "" - - #Parse output - for line in output.splitlines(): - line = line.rstrip() - if line.startswith("Name:"): - pkg = line.split(": ")[1] - elif line.startswith("Version:"): - tmp_str = line.split(": ")[1] - ver, arch = tmp_str.split("@") - break - - # Get filename - index = re.search("^URLs", output, re.MULTILINE) - tmp_str = output[index.end():] - for line in tmp_str.splitlines(): - if "/" in line: - line = line.lstrip() - filename = line.split(" ")[0] - break - - # To have the same data type than other package_info methods - filepath = os.path.join(self.deploy_dir, arch, filename) - pkg_dict = {} - pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename, - "filepath": filepath} - - return pkg_dict - - """ - Returns the path to a tmpdir where resides the contents of a package. - - Deleting the tmpdir is responsability of the caller. - - """ def extract(self, pkg): - pkg_info = self.package_info(pkg) - if not pkg_info: - bb.fatal("Unable to get information for package '%s' while " - "trying to extract the package." % pkg) - - pkg_path = pkg_info[pkg]["filepath"] + output = self._invoke_dnf(["repoquery", "--queryformat", "%{location}", pkg]) + pkg_name = output.splitlines()[-1] + if not pkg_name.endswith(".rpm"): + bb.fatal("dnf could not find package %s in repository: %s" %(pkg, output)) + pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name) cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") @@ -1506,7 +795,7 @@ class OpkgDpkgPM(PackageManager): def package_info(self, pkg, cmd): try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Unable to list available packages. Command '%s' " "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) @@ -1532,20 +821,24 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) + if self.d.getVar('IMAGE_PKGTYPE') == 'deb': + data_tar = 'data.tar.xz' + else: + data_tar = 'data.tar.gz' try: - cmd = "%s x %s" % (ar_cmd, pkg_path) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) - cmd = "%s xf data.tar.*" % tar_cmd - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + cmd = [ar_cmd, 'x', pkg_path] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + cmd = [tar_cmd, 'xf', data_tar] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.utils.remove(tmp_dir, recurse=True) bb.fatal("Unable to extract %s package. Command '%s' " - "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8"))) except OSError as e: bb.utils.remove(tmp_dir, recurse=True) bb.fatal("Unable to extract %s package. Command '%s' " - "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename)) bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) @@ -1564,13 +857,13 @@ class OpkgPM(OpkgDpkgPM): self.pkg_archs = archs self.task_name = task_name - self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True) + self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK") self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") - self.opkg_args = "--volatile-cache -f %s -o %s " % (self.config_file, target_rootfs) - self.opkg_args += self.d.getVar("OPKG_ARGS", True) + self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) + self.opkg_args += self.d.getVar("OPKG_ARGS") - opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') if opkg_lib_dir[0] == "/": opkg_lib_dir = opkg_lib_dir[1:] @@ -1582,7 +875,7 @@ class OpkgPM(OpkgDpkgPM): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") == "1" + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" if self.from_feeds: self._create_custom_config() else: @@ -1627,7 +920,7 @@ class OpkgPM(OpkgDpkgPM): config_file.write("arch %s %d\n" % (arch, priority)) priority += 5 - for line in (self.d.getVar('IPK_FEED_URIS', True) or "").split(): + for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): feed_match = re.match("^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) if feed_match is not None: @@ -1644,27 +937,29 @@ class OpkgPM(OpkgDpkgPM): specified as compatible for the current machine. NOTE: Development-helper feature, NOT a full-fledged feed. """ - if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True) or "") != "": + if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": for arch in self.pkg_archs.split(): cfg_file_name = os.path.join(self.target_rootfs, - self.d.getVar("sysconfdir", True), + self.d.getVar("sysconfdir"), "opkg", "local-%s-feed.conf" % arch) with open(cfg_file_name, "w+") as cfg_file: cfg_file.write("src/gz local-%s %s/%s" % (arch, - self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True), + self.d.getVar('FEED_DEPLOYDIR_BASE_URI'), arch)) - if self.opkg_dir != '/var/lib/opkg': + if self.d.getVar('OPKGLIBDIR') != '/var/lib': # There is no command line option for this anymore, we need to add # info_dir and status_file to config file, if OPKGLIBDIR doesn't have # the default value of "/var/lib" as defined in opkg: - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" - cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) - cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" + cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) + cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) + cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) def _create_config(self): @@ -1682,31 +977,33 @@ class OpkgPM(OpkgDpkgPM): config_file.write("src oe-%s file:%s\n" % (arch, pkgs_dir)) - if self.opkg_dir != '/var/lib/opkg': + if self.d.getVar('OPKGLIBDIR') != '/var/lib': # There is no command line option for this anymore, we need to add # info_dir and status_file to config file, if OPKGLIBDIR doesn't have # the default value of "/var/lib" as defined in opkg: - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" - config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) - config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) - - def insert_feeds_uris(self): - if self.feed_uris == "": + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" + config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) + config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) + config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) + + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + if feed_uris == "": return rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' % self.target_rootfs) - feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split()) - archs = self.pkg_archs.split() if self.feed_archs is None else self.feed_archs.split() + feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split() with open(rootfs_config, "w+") as config_file: uri_iterator = 0 for uri in feed_uris: if archs: for arch in archs: - if (self.feed_archs is None) and (not os.path.exists(os.path.join(self.deploy_dir, arch))): + if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))): continue bb.note('Adding opkg feed url-%s-%d (%s)' % (arch, uri_iterator, uri)) @@ -1744,9 +1041,9 @@ class OpkgPM(OpkgDpkgPM): os.environ['OFFLINE_ROOT'] = self.target_rootfs os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs - os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR', True), + os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") - os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE', True) + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') try: bb.note("Installing the following packages: %s" % ' '.join(pkgs)) @@ -1760,7 +1057,7 @@ class OpkgPM(OpkgDpkgPM): def remove(self, pkgs, with_dependencies=True): if with_dependencies: - cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \ + cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) else: cmd = "%s %s --force-depends remove %s" % \ @@ -1797,7 +1094,7 @@ class OpkgPM(OpkgDpkgPM): return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs() def handle_bad_recommendations(self): - bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True) or "" + bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS") or "" if bad_recommendations.strip() == "": return @@ -1844,11 +1141,14 @@ class OpkgPM(OpkgDpkgPM): # Create an temp dir as opkg root for dummy installation temp_rootfs = self.d.expand('${T}/opkg') - temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg') + opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + if opkg_lib_dir[0] == "/": + opkg_lib_dir = opkg_lib_dir[1:] + temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') bb.utils.mkdirhier(temp_opkg_dir) opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) - opkg_args += self.d.getVar("OPKG_ARGS", True) + opkg_args += self.d.getVar("OPKG_ARGS") cmd = "%s %s update" % (self.opkg_cmd, opkg_args) try: @@ -1924,7 +1224,7 @@ class DpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): super(DpkgPM, self).__init__(d) self.target_rootfs = target_rootfs - self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB', True) + self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB') if apt_conf_dir is None: self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt") else: @@ -1933,10 +1233,10 @@ class DpkgPM(OpkgDpkgPM): self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") - self.apt_args = d.getVar("APT_ARGS", True) + self.apt_args = d.getVar("APT_ARGS") self.all_arch_list = archs.split() - all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").split() + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list) self._create_configs(archs, base_archs) @@ -1977,7 +1277,10 @@ class DpkgPM(OpkgDpkgPM): """ def run_pre_post_installs(self, package_name=None): info_dir = self.target_rootfs + "/var/lib/dpkg/info" - suffixes = [(".preinst", "Preinstall"), (".postinst", "Postinstall")] + ControlScript = collections.namedtuple("ControlScript", ["suffix", "name", "argument"]) + control_scripts = [ + ControlScript(".preinst", "Preinstall", "install"), + ControlScript(".postinst", "Postinstall", "configure")] status_file = self.target_rootfs + "/var/lib/dpkg/status" installed_pkgs = [] @@ -1994,22 +1297,25 @@ class DpkgPM(OpkgDpkgPM): os.environ['OFFLINE_ROOT'] = self.target_rootfs os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs - os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR', True), + os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") - os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE', True) + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') failed_pkgs = [] for pkg_name in installed_pkgs: - for suffix in suffixes: - p_full = os.path.join(info_dir, pkg_name + suffix[0]) + for control_script in control_scripts: + p_full = os.path.join(info_dir, pkg_name + control_script.suffix) if os.path.exists(p_full): try: bb.note("Executing %s for package: %s ..." % - (suffix[1].lower(), pkg_name)) - subprocess.check_output(p_full, stderr=subprocess.STDOUT) + (control_script.name.lower(), pkg_name)) + output = subprocess.check_output([p_full, control_script.argument], + stderr=subprocess.STDOUT).decode("utf-8") + bb.note(output) except subprocess.CalledProcessError as e: bb.note("%s for package %s failed with %d:\n%s" % - (suffix[1], pkg_name, e.returncode, e.output.decode("utf-8"))) + (control_script.name, pkg_name, e.returncode, + e.output.decode("utf-8"))) failed_pkgs.append(pkg_name) break @@ -2089,23 +1395,23 @@ class DpkgPM(OpkgDpkgPM): if result is not None: bb.fatal(result) - def insert_feeds_uris(self): - if self.feed_uris == "": + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + if feed_uris == "": return sources_conf = os.path.join("%s/etc/apt/sources.list" % self.target_rootfs) arch_list = [] - if self.feed_archs is None: + if feed_archs is None: for arch in self.all_arch_list: if not os.path.exists(os.path.join(self.deploy_dir, arch)): continue arch_list.append(arch) else: - arch_list = self.feed_archs.split() + arch_list = feed_archs.split() - feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split()) + feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) with open(sources_conf, "w+") as sources_file: for uri in feed_uris: @@ -2145,7 +1451,7 @@ class DpkgPM(OpkgDpkgPM): priority += 5 - pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE', True) or "" + pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or "" for pkg in pkg_exclude.split(): prefs_file.write( "Package: %s\n" @@ -2160,14 +1466,13 @@ class DpkgPM(OpkgDpkgPM): os.path.join(self.deploy_dir, arch)) base_arch_list = base_archs.split() - multilib_variants = self.d.getVar("MULTILIB_VARIANTS", True); + multilib_variants = self.d.getVar("MULTILIB_VARIANTS"); for variant in multilib_variants.split(): localdata = bb.data.createCopy(self.d) variant_tune = localdata.getVar("DEFAULTTUNE_virtclass-multilib-" + variant, False) - orig_arch = localdata.getVar("DPKG_ARCH", True) + orig_arch = localdata.getVar("DPKG_ARCH") localdata.setVar("DEFAULTTUNE", variant_tune) - bb.data.update_data(localdata) - variant_arch = localdata.getVar("DPKG_ARCH", True) + variant_arch = localdata.getVar("DPKG_ARCH") if variant_arch not in base_arch_list: base_arch_list.append(variant_arch) @@ -2198,7 +1503,7 @@ class DpkgPM(OpkgDpkgPM): def remove_packaging_data(self): bb.utils.remove(os.path.join(self.target_rootfs, - self.d.getVar('opkglibdir', True)), True) + self.d.getVar('opkglibdir')), True) bb.utils.remove(self.target_rootfs + "/var/lib/dpkg/", True) def fix_broken_dependencies(self): @@ -2246,12 +1551,12 @@ class DpkgPM(OpkgDpkgPM): return tmp_dir def generate_index_files(d): - classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split() + classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split() indexer_map = { - "rpm": (RpmIndexer, d.getVar('DEPLOY_DIR_RPM', True)), - "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK', True)), - "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB', True)) + "rpm": (RpmIndexer, d.getVar('DEPLOY_DIR_RPM')), + "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK')), + "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB')) } result = None diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index 21d4de914f..32e5c82a94 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -57,7 +57,7 @@ def read_subpkgdata_dict(pkg, d): def _pkgmap(d): """Return a dictionary mapping package to recipe name.""" - pkgdatadir = d.getVar("PKGDATA_DIR", True) + pkgdatadir = d.getVar("PKGDATA_DIR") pkgmap = {} try: diff --git a/meta/lib/oe/packagegroup.py b/meta/lib/oe/packagegroup.py index 97819279b7..4bc5d3e4bb 100644 --- a/meta/lib/oe/packagegroup.py +++ b/meta/lib/oe/packagegroup.py @@ -1,17 +1,17 @@ import itertools def is_optional(feature, d): - packages = d.getVar("FEATURE_PACKAGES_%s" % feature, True) + packages = d.getVar("FEATURE_PACKAGES_%s" % feature) if packages: - return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional", True)) + return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional")) else: - return bool(d.getVarFlag("PACKAGE_GROUP_%s" % feature, "optional", True)) + return bool(d.getVarFlag("PACKAGE_GROUP_%s" % feature, "optional")) def packages(features, d): for feature in features: - packages = d.getVar("FEATURE_PACKAGES_%s" % feature, True) + packages = d.getVar("FEATURE_PACKAGES_%s" % feature) if not packages: - packages = d.getVar("PACKAGE_GROUP_%s" % feature, True) + packages = d.getVar("PACKAGE_GROUP_%s" % feature) for pkg in (packages or "").split(): yield pkg diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py index 4a0d3f7149..f1ab3dd809 100644 --- a/meta/lib/oe/patch.py +++ b/meta/lib/oe/patch.py @@ -81,7 +81,7 @@ class PatchSet(object): patch[param] = PatchSet.defaults[param] if patch.get("remote"): - patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d) + patch["file"] = self.d.expand(bb.fetch2.localpath(patch["remote"], self.d)) patch["filemd5"] = bb.utils.md5_file(patch["file"]) @@ -117,43 +117,50 @@ class PatchSet(object): return None return os.sep.join(filesplit[striplevel:]) - copiedmode = False - filelist = [] - with open(patchfile) as f: - for line in f: - if line.startswith('--- '): - patchpth = patchedpath(line) - if not patchpth: - break - if copiedmode: - addedfile = patchpth - else: - removedfile = patchpth - elif line.startswith('+++ '): - addedfile = patchedpath(line) - if not addedfile: - break - elif line.startswith('*** '): - copiedmode = True - removedfile = patchedpath(line) - if not removedfile: - break - else: - removedfile = None - addedfile = None - - if addedfile and removedfile: - if removedfile == '/dev/null': - mode = 'A' - elif addedfile == '/dev/null': - mode = 'D' - else: - mode = 'M' - if srcdir: - fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) - else: - fullpath = addedfile - filelist.append((fullpath, mode)) + for encoding in ['utf-8', 'latin-1']: + try: + copiedmode = False + filelist = [] + with open(patchfile) as f: + for line in f: + if line.startswith('--- '): + patchpth = patchedpath(line) + if not patchpth: + break + if copiedmode: + addedfile = patchpth + else: + removedfile = patchpth + elif line.startswith('+++ '): + addedfile = patchedpath(line) + if not addedfile: + break + elif line.startswith('*** '): + copiedmode = True + removedfile = patchedpath(line) + if not removedfile: + break + else: + removedfile = None + addedfile = None + + if addedfile and removedfile: + if removedfile == '/dev/null': + mode = 'A' + elif addedfile == '/dev/null': + mode = 'D' + else: + mode = 'M' + if srcdir: + fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) + else: + fullpath = addedfile + filelist.append((fullpath, mode)) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to decode %s' % patchfile) return filelist @@ -274,18 +281,27 @@ class GitApplyTree(PatchTree): def __init__(self, dir, d): PatchTree.__init__(self, dir, d) + self.commituser = d.getVar('PATCH_GIT_USER_NAME') + self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL') @staticmethod def extractPatchHeader(patchfile): """ Extract just the header lines from the top of a patch file """ - lines = [] - with open(patchfile, 'r') as f: - for line in f.readlines(): - if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): - break - lines.append(line) + for encoding in ['utf-8', 'latin-1']: + lines = [] + try: + with open(patchfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): + break + lines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % patchfile) return lines @staticmethod @@ -301,6 +317,7 @@ class GitApplyTree(PatchTree): def interpretPatchHeader(headerlines): import re author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') + from_commit_re = re.compile('^From [a-z0-9]{40} .*') outlines = [] author = None date = None @@ -330,11 +347,39 @@ class GitApplyTree(PatchTree): # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval + elif from_commit_re.match(line): + # We don't want the From <commit> line - if it's present it will break rebasing + continue outlines.append(line) + + if not subject: + firstline = None + for line in headerlines: + line = line.strip() + if firstline: + if line: + # Second line is not blank, the first line probably isn't usable + firstline = None + break + elif line: + firstline = line + if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100: + subject = firstline + return outlines, author, date, subject @staticmethod - def prepareCommit(patchfile): + def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None): + if d: + commituser = d.getVar('PATCH_GIT_USER_NAME') + commitemail = d.getVar('PATCH_GIT_USER_EMAIL') + if commituser: + cmd += ['-c', 'user.name="%s"' % commituser] + if commitemail: + cmd += ['-c', 'user.email="%s"' % commitemail] + + @staticmethod + def prepareCommit(patchfile, commituser=None, commitemail=None): """ Prepare a git commit command line based on the header from a patch file (typically this is useful for patches that cannot be applied with "git am" due to formatting) @@ -343,21 +388,24 @@ class GitApplyTree(PatchTree): # Process patch header and extract useful information lines = GitApplyTree.extractPatchHeader(patchfile) outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) - if not author or not subject: + if not author or not subject or not date: try: - shellcmd = ["git", "log", "--format=email", "--diff-filter=A", "--", patchfile] + shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile] out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) except CmdError: out = None if out: _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) - if not author or not date: - # These really need to go together + if not author: + # If we're setting the author then the date should be set as well author = newauthor date = newdate + elif not date: + # If we don't do this we'll get the current date, at least this will be closer + date = newdate if not subject: subject = newsubject - if subject: + if subject and outlines and not outlines[0].strip() == subject: outlines.insert(0, '%s\n\n' % subject.strip()) # Write out commit message to a file @@ -366,7 +414,9 @@ class GitApplyTree(PatchTree): for line in outlines: tf.write(line) # Prepare git command - cmd = ["git", "commit", "-F", tmpfile] + cmd = ["git"] + GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) + cmd += ["commit", "-F", tmpfile] # git doesn't like plain email addresses as authors if author and '<' in author: cmd.append('--author="%s"' % author) @@ -378,6 +428,7 @@ class GitApplyTree(PatchTree): def extractPatches(tree, startcommit, outdir, paths=None): import tempfile import shutil + import re tempdir = tempfile.mkdtemp(prefix='oepatch') try: shellcmd = ["git", "format-patch", startcommit, "-o", tempdir] @@ -387,16 +438,27 @@ class GitApplyTree(PatchTree): out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) if out: for srcfile in out.split(): - patchlines = [] - outfile = None - with open(srcfile, 'r') as f: - for line in f: - if line.startswith(GitApplyTree.patch_line_prefix): - outfile = line.split()[-1].strip() - continue - if line.startswith(GitApplyTree.ignore_commit_prefix): - continue - patchlines.append(line) + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + outfile = None + try: + with open(srcfile, 'r', encoding=encoding) as f: + for line in f: + checkline = line + if checkline.startswith('Subject: '): + checkline = re.sub(r'\[.+?\]\s*', '', checkline[9:]) + if checkline.startswith(GitApplyTree.patch_line_prefix): + outfile = line.split()[-1].strip() + continue + if checkline.startswith(GitApplyTree.ignore_commit_prefix): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + if not outfile: outfile = os.path.basename(srcfile) with open(os.path.join(outdir, outfile), 'w') as of: @@ -442,7 +504,9 @@ class GitApplyTree(PatchTree): try: patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) try: - shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot, "am", "-3", "--keep-cr", "-p%s" % patch['strippath']] + shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] + self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) + shellcmd += ["am", "-3", "--keep-cr", "-p%s" % patch['strippath']] return _applypatchhelper(shellcmd, patch, force, reverse, run) except CmdError: # Need to abort the git am, or we'll still be within it at the end @@ -472,7 +536,7 @@ class GitApplyTree(PatchTree): shellcmd = ["git", "reset", "HEAD", self.patchdir] output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) # Commit the result - (tmpfile, shellcmd) = self.prepareCommit(patch['file']) + (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) try: shellcmd.insert(0, patchfilevar) output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) @@ -487,7 +551,7 @@ class GitApplyTree(PatchTree): class QuiltTree(PatchSet): def _runcmd(self, args, run = True): - quiltrc = self.d.getVar('QUILTRCFILE', True) + quiltrc = self.d.getVar('QUILTRCFILE') if not run: return ["quilt"] + ["--quiltrc"] + [quiltrc] + args runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir) @@ -663,7 +727,7 @@ class UserResolver(Resolver): # Patch application failed patchcmd = self.patchset.Push(True, False, False) - t = self.patchset.d.getVar('T', True) + t = self.patchset.d.getVar('T') if not t: bb.msg.fatal("Build", "T not set") bb.utils.mkdirhier(t) @@ -705,3 +769,110 @@ class UserResolver(Resolver): os.chdir(olddir) raise os.chdir(olddir) + + +def patch_path(url, fetch, workdir, expand=True): + """Return the local path of a patch, or None if this isn't a patch""" + + local = fetch.localpath(url) + base, ext = os.path.splitext(os.path.basename(local)) + if ext in ('.gz', '.bz2', '.xz', '.Z'): + if expand: + local = os.path.join(workdir, base) + ext = os.path.splitext(base)[1] + + urldata = fetch.ud[url] + if "apply" in urldata.parm: + apply = oe.types.boolean(urldata.parm["apply"]) + if not apply: + return + elif ext not in (".diff", ".patch"): + return + + return local + +def src_patches(d, all=False, expand=True): + workdir = d.getVar('WORKDIR') + fetch = bb.fetch2.Fetch([], d) + patches = [] + sources = [] + for url in fetch.urls: + local = patch_path(url, fetch, workdir, expand) + if not local: + if all: + local = fetch.localpath(url) + sources.append(local) + continue + + urldata = fetch.ud[url] + parm = urldata.parm + patchname = parm.get('pname') or os.path.basename(local) + + apply, reason = should_apply(parm, d) + if not apply: + if reason: + bb.note("Patch %s %s" % (patchname, reason)) + continue + + patchparm = {'patchname': patchname} + if "striplevel" in parm: + striplevel = parm["striplevel"] + elif "pnum" in parm: + #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url) + striplevel = parm["pnum"] + else: + striplevel = '1' + patchparm['striplevel'] = striplevel + + patchdir = parm.get('patchdir') + if patchdir: + patchparm['patchdir'] = patchdir + + localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm)) + patches.append(localurl) + + if all: + return sources + + return patches + + +def should_apply(parm, d): + if "mindate" in parm or "maxdate" in parm: + pn = d.getVar('PN') + srcdate = d.getVar('SRCDATE_%s' % pn) + if not srcdate: + srcdate = d.getVar('SRCDATE') + + if srcdate == "now": + srcdate = d.getVar('DATE') + + if "maxdate" in parm and parm["maxdate"] < srcdate: + return False, 'is outdated' + + if "mindate" in parm and parm["mindate"] > srcdate: + return False, 'is predated' + + + if "minrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and srcrev < parm["minrev"]: + return False, 'applies to later revisions' + + if "maxrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and srcrev > parm["maxrev"]: + return False, 'applies to earlier revisions' + + if "rev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and parm["rev"] not in srcrev: + return False, "doesn't apply to revision" + + if "notrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and parm["notrev"] in srcrev: + return False, "doesn't apply to revision" + + return True, None + diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py index 413ebfb395..448a2b944e 100644 --- a/meta/lib/oe/path.py +++ b/meta/lib/oe/path.py @@ -50,9 +50,30 @@ def make_relative_symlink(path): os.remove(path) os.symlink(base, path) +def replace_absolute_symlinks(basedir, d): + """ + Walk basedir looking for absolute symlinks and replacing them with relative ones. + The absolute links are assumed to be relative to basedir + (compared to make_relative_symlink above which tries to compute common ancestors + using pattern matching instead) + """ + for walkroot, dirs, files in os.walk(basedir): + for file in files + dirs: + path = os.path.join(walkroot, file) + if not os.path.islink(path): + continue + link = os.readlink(path) + if not os.path.isabs(link): + continue + walkdir = os.path.dirname(path.rpartition(basedir)[2]) + base = os.path.relpath(link, walkdir) + bb.debug(2, "Replacing absolute path %s with relative path %s" % (link, base)) + os.remove(path) + os.symlink(base, path) + def format_display(path, metadata): """ Prepare a path for display to the user. """ - rel = relative(metadata.getVar("TOPDIR", True), path) + rel = relative(metadata.getVar("TOPDIR"), path) if len(rel) > len(path): return path else: @@ -65,27 +86,43 @@ def copytree(src, dst): # This way we also preserve hardlinks between files in the tree. bb.utils.mkdirhier(dst) - cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) def copyhardlinktree(src, dst): """ Make the hard link when possible, otherwise copy. """ bb.utils.mkdirhier(dst) if os.path.isdir(src) and not len(os.listdir(src)): - return + return if (os.stat(src).st_dev == os.stat(dst).st_dev): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time - cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --files-from - --no-recursion | tar -xf - -C %s' % (src, src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) - cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + source = '' + if os.path.isdir(src): + if len(glob.glob('%s/.??*' % src)) > 0: + source = './.??* ' + source += './*' + s_dir = src + else: + source = src + s_dir = os.getcwd() + cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst)) + subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) else: copytree(src, dst) def remove(path, recurse=True): - """Equivalent to rm -f or rm -rf""" + """ + Equivalent to rm -f or rm -rf + NOTE: be careful about passing paths that may contain filenames with + wildcards in them (as opposed to passing an actual wildcarded path) - + since we use glob.glob() to expand the path. Filenames containing + square brackets are particularly problematic since the they may not + actually expand to match the original filename. + """ for name in glob.glob(path): try: os.unlink(name) @@ -105,47 +142,6 @@ def symlink(source, destination, force=False): if e.errno != errno.EEXIST or os.readlink(destination) != source: raise -class CalledProcessError(Exception): - def __init__(self, retcode, cmd, output = None): - self.retcode = retcode - self.cmd = cmd - self.output = output - def __str__(self): - return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output) - -# Not needed when we move to python 2.7 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - def find(dir, **walkoptions): """ Given a directory, recurses into that directory, returning all files as absolute paths. """ diff --git a/meta/lib/oe/prservice.py b/meta/lib/oe/prservice.py index 0054f954cc..32dfc15e88 100644 --- a/meta/lib/oe/prservice.py +++ b/meta/lib/oe/prservice.py @@ -1,7 +1,7 @@ def prserv_make_conn(d, check = False): import prserv.serv - host_params = list([_f for _f in (d.getVar("PRSERV_HOST", True) or '').split(':') if _f]) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) try: conn = None conn = prserv.serv.PRServerConnection(host_params[0], int(host_params[1])) @@ -15,11 +15,11 @@ def prserv_make_conn(d, check = False): return conn def prserv_dump_db(d): - if not d.getVar('PRSERV_HOST', True): + if not d.getVar('PRSERV_HOST'): bb.error("Not using network based PR service") return None - conn = d.getVar("__PRSERV_CONN", True) + conn = d.getVar("__PRSERV_CONN") if conn is None: conn = prserv_make_conn(d) if conn is None: @@ -27,18 +27,18 @@ def prserv_dump_db(d): return None #dump db - opt_version = d.getVar('PRSERV_DUMPOPT_VERSION', True) - opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH', True) - opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM', True) - opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL', True)) + opt_version = d.getVar('PRSERV_DUMPOPT_VERSION') + opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH') + opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM') + opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL')) return conn.export(opt_version, opt_pkgarch, opt_checksum, opt_col) def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None): - if not d.getVar('PRSERV_HOST', True): + if not d.getVar('PRSERV_HOST'): bb.error("Not using network based PR service") return None - conn = d.getVar("__PRSERV_CONN", True) + conn = d.getVar("__PRSERV_CONN") if conn is None: conn = prserv_make_conn(d) if conn is None: @@ -58,7 +58,7 @@ def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksu (filter_checksum and filter_checksum != checksum): continue try: - value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum, True)) + value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum)) except BaseException as exc: bb.debug("Not valid value of %s:%s" % (v,str(exc))) continue @@ -72,8 +72,8 @@ def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksu def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): import bb.utils #initilize the output file - bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR', True)) - df = d.getVar('PRSERV_DUMPFILE', True) + bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR')) + df = d.getVar('PRSERV_DUMPFILE') #write data lf = bb.utils.lockfile("%s.lock" % df) f = open(df, "a") @@ -114,7 +114,7 @@ def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): bb.utils.unlockfile(lf) def prserv_check_avail(d): - host_params = list([_f for _f in (d.getVar("PRSERV_HOST", True) or '').split(':') if _f]) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) try: if len(host_params) != 2: raise TypeError diff --git a/meta/lib/oe/qa.py b/meta/lib/oe/qa.py index 75e7df8546..3231e60cea 100644 --- a/meta/lib/oe/qa.py +++ b/meta/lib/oe/qa.py @@ -1,4 +1,4 @@ -import os, struct +import os, struct, mmap class NotELFFileError(Exception): pass @@ -23,9 +23,9 @@ class ELFFile: EV_CURRENT = 1 # possible values for EI_DATA - ELFDATANONE = 0 - ELFDATA2LSB = 1 - ELFDATA2MSB = 2 + EI_DATA_NONE = 0 + EI_DATA_LSB = 1 + EI_DATA_MSB = 2 PT_INTERP = 3 @@ -34,51 +34,46 @@ class ELFFile: #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name) raise NotELFFileError("%s is not an ELF" % self.name) - def __init__(self, name, bits = 0): + def __init__(self, name): self.name = name - self.bits = bits self.objdump_output = {} - def open(self): - if not os.path.isfile(self.name): - raise NotELFFileError("%s is not a normal file" % self.name) + # Context Manager functions to close the mmap explicitly + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.data.close() + def open(self): with open(self.name, "rb") as f: - # Read 4k which should cover most of the headers we're after - self.data = f.read(4096) + try: + self.data = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + except ValueError: + # This means the file is empty + raise NotELFFileError("%s is empty" % self.name) + # Check the file has the minimum number of ELF table entries if len(self.data) < ELFFile.EI_NIDENT + 4: raise NotELFFileError("%s is not an ELF" % self.name) + # ELF header self.my_assert(self.data[0], 0x7f) self.my_assert(self.data[1], ord('E')) self.my_assert(self.data[2], ord('L')) self.my_assert(self.data[3], ord('F')) - if self.bits == 0: - if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: - self.bits = 32 - elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: - self.bits = 64 - else: - # Not 32-bit or 64.. lets assert - raise NotELFFileError("ELF but not 32 or 64 bit.") - elif self.bits == 32: - self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS32) - elif self.bits == 64: - self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS64) + if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: + self.bits = 32 + elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: + self.bits = 64 else: - raise NotELFFileError("Must specify unknown, 32 or 64 bit size.") + # Not 32-bit or 64.. lets assert + raise NotELFFileError("ELF but not 32 or 64 bit.") self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT) - self.sex = self.data[ELFFile.EI_DATA] - if self.sex == ELFFile.ELFDATANONE: - raise NotELFFileError("self.sex == ELFDATANONE") - elif self.sex == ELFFile.ELFDATA2LSB: - self.sex = "<" - elif self.sex == ELFFile.ELFDATA2MSB: - self.sex = ">" - else: - raise NotELFFileError("Unknown self.sex") + self.endian = self.data[ELFFile.EI_DATA] + if self.endian not in (ELFFile.EI_DATA_LSB, ELFFile.EI_DATA_MSB): + raise NotELFFileError("Unexpected EI_DATA %x" % self.endian) def osAbi(self): return self.data[ELFFile.EI_OSABI] @@ -90,16 +85,20 @@ class ELFFile: return self.bits def isLittleEndian(self): - return self.sex == "<" + return self.endian == ELFFile.EI_DATA_LSB def isBigEndian(self): - return self.sex == ">" + return self.endian == ELFFile.EI_DATA_MSB + + def getStructEndian(self): + return {ELFFile.EI_DATA_LSB: "<", + ELFFile.EI_DATA_MSB: ">"}[self.endian] def getShort(self, offset): - return struct.unpack_from(self.sex+"H", self.data, offset)[0] + return struct.unpack_from(self.getStructEndian() + "H", self.data, offset)[0] def getWord(self, offset): - return struct.unpack_from(self.sex+"i", self.data, offset)[0] + return struct.unpack_from(self.getStructEndian() + "i", self.data, offset)[0] def isDynamic(self): """ @@ -118,7 +117,7 @@ class ELFFile: def machine(self): """ - We know the sex stored in self.sex and we + We know the endian stored in self.endian and we know the position """ return self.getShort(ELFFile.E_MACHINE) @@ -130,11 +129,11 @@ class ELFFile: if cmd in self.objdump_output: return self.objdump_output[cmd] - objdump = d.getVar('OBJDUMP', True) + objdump = d.getVar('OBJDUMP') env = os.environ.copy() env["LC_ALL"] = "C" - env["PATH"] = d.getVar('PATH', True) + env["PATH"] = d.getVar('PATH') try: bb.note("%s %s %s" % (objdump, cmd, self.name)) @@ -144,8 +143,29 @@ class ELFFile: bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e)) return "" +def elf_machine_to_string(machine): + """ + Return the name of a given ELF e_machine field or the hex value as a string + if it isn't recognised. + """ + try: + return { + 0x02: "SPARC", + 0x03: "x86", + 0x08: "MIPS", + 0x14: "PowerPC", + 0x28: "ARM", + 0x2A: "SuperH", + 0x32: "IA-64", + 0x3E: "x86-64", + 0xB7: "AArch64" + }[machine] + except: + return "Unknown (%s)" % repr(machine) + if __name__ == "__main__": import sys - elf = ELFFile(sys.argv[1]) - elf.open() - print(elf.isDynamic()) + + with ELFFile(sys.argv[1]) as elf: + elf.open() + print(elf.isDynamic()) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index e3c4b8a759..a7fdd36e40 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -2,7 +2,7 @@ # # Some code borrowed from the OE layer index # -# Copyright (C) 2013-2015 Intel Corporation +# Copyright (C) 2013-2016 Intel Corporation # import sys @@ -15,6 +15,7 @@ from . import utils import shutil import re import fnmatch +import glob from collections import OrderedDict, defaultdict @@ -26,20 +27,11 @@ list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] -def pn_to_recipe(cooker, pn): +def pn_to_recipe(cooker, pn, mc=''): """Convert a recipe name (PN) to the path to the recipe file""" - import bb.providers - - if pn in cooker.recipecache.pkg_pn: - best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn) - return best[3] - elif pn in cooker.recipecache.providers: - filenames = cooker.recipecache.providers[pn] - eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecache) - filename = eligible[0] - return filename - else: - return None + + best = cooker.findBestProvider(pn, mc) + return best[3] def get_unavailable_reasons(cooker, pn): @@ -49,38 +41,17 @@ def get_unavailable_reasons(cooker, pn): return taskdata.get_reasons(pn) -def parse_recipe(fn, appendfiles, d): +def parse_recipe(cooker, fn, appendfiles): """ Parse an individual recipe file, optionally with a list of bbappend files. """ import bb.cache - envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d) + parser = bb.cache.NoCache(cooker.databuilder) + envdata = parser.loadDataFull(fn, appendfiles) return envdata -def parse_recipe_simple(cooker, pn, d, appends=True): - """ - Parse a recipe and optionally all bbappends that apply to it - in the current configuration. - """ - import bb.providers - - recipefile = pn_to_recipe(cooker, pn) - if not recipefile: - skipreasons = get_unavailable_reasons(cooker, pn) - # We may as well re-use bb.providers.NoProvider here - if skipreasons: - raise bb.providers.NoProvider(skipreasons) - else: - raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn) - if appends: - appendfiles = cooker.collection.get_file_appends(recipefile) - else: - appendfiles = None - return parse_recipe(recipefile, appendfiles, d) - - def get_var_files(fn, varlist, d): """Find the file in which each of a list of variables is set. Note: requires variable history to be enabled when parsing. @@ -258,7 +229,7 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True): changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) if remainingnames: - if tolines[-1].strip() != '': + if tolines and tolines[-1].strip() != '': tolines.append('\n') for k in remainingnames.keys(): outputvalue(k, tolines) @@ -357,15 +328,16 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): # FIXME need a warning if the unexpanded SRC_URI value contains variable references - uris = (d.getVar('SRC_URI', True) or "").split() + uris = (d.getVar('SRC_URI') or "").split() fetch = bb.fetch2.Fetch(uris, d) if download: fetch.download() # Copy local files to target directory and gather any remote files - bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep + bb_dir = os.path.dirname(d.getVar('FILE')) + os.sep remotes = [] - includes = [path for path in d.getVar('BBINCLUDED', True).split() if + copied = [] + includes = [path for path in d.getVar('BBINCLUDED').split() if path.startswith(bb_dir) and os.path.exists(path)] for path in fetch.localpaths() + includes: # Only import files that are under the meta directory @@ -376,37 +348,58 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): if not os.path.exists(subdir): os.makedirs(subdir) shutil.copy2(path, os.path.join(tgt_dir, relpath)) + copied.append(relpath) else: remotes.append(path) # Simply copy whole meta dir, if requested if whole_dir: shutil.copytree(bb_dir, tgt_dir) - return remotes + return copied, remotes -def get_recipe_local_files(d, patches=False): +def get_recipe_local_files(d, patches=False, archives=False): """Get a list of local files in SRC_URI within a recipe.""" - uris = (d.getVar('SRC_URI', True) or "").split() + import oe.patch + uris = (d.getVar('SRC_URI') or "").split() fetch = bb.fetch2.Fetch(uris, d) + # FIXME this list should be factored out somewhere else (such as the + # fetcher) though note that this only encompasses actual container formats + # i.e. that can contain multiple files as opposed to those that only + # contain a compressed stream (i.e. .tar.gz as opposed to just .gz) + archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z'] ret = {} for uri in uris: if fetch.ud[uri].type == 'file': if (not patches and - bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')): + oe.patch.patch_path(uri, fetch, '', expand=False)): continue # Skip files that are referenced by absolute path - if not os.path.isabs(fetch.ud[uri].basepath): - ret[fetch.ud[uri].basepath] = fetch.localpath(uri) + fname = fetch.ud[uri].basepath + if os.path.isabs(fname): + continue + # Handle subdir= + subdir = fetch.ud[uri].parm.get('subdir', '') + if subdir: + if os.path.isabs(subdir): + continue + fname = os.path.join(subdir, fname) + localpath = fetch.localpath(uri) + if not archives: + # Ignore archives that will be unpacked + if localpath.endswith(tuple(archive_exts)): + unpack = fetch.ud[uri].parm.get('unpack', True) + if unpack: + continue + ret[fname] = localpath return ret def get_recipe_patches(d): """Get a list of the patches included in SRC_URI within a recipe.""" + import oe.patch + patches = oe.patch.src_patches(d, expand=False) patchfiles = [] - # Execute src_patches() defined in patch.bbclass - this works since that class - # is inherited globally - patches = bb.utils.exec_flat_python_func('src_patches', d) for patch in patches: _, _, local, _, _, parm = bb.fetch.decodeurl(patch) patchfiles.append(local) @@ -423,22 +416,20 @@ def get_recipe_patched_files(d): change mode ('A' for add, 'D' for delete or 'M' for modify) """ import oe.patch - # Execute src_patches() defined in patch.bbclass - this works since that class - # is inherited globally - patches = bb.utils.exec_flat_python_func('src_patches', d) + patches = oe.patch.src_patches(d, expand=False) patchedfiles = {} for patch in patches: _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch) striplevel = int(parm['striplevel']) - patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S', True), parm.get('patchdir', ''))) + patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', ''))) return patchedfiles def validate_pn(pn): """Perform validation on a recipe name (PN) for a new recipe.""" reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] - if not re.match('[0-9a-z-.]+', pn): - return 'Recipe name "%s" is invalid: only characters 0-9, a-z, - and . are allowed' % pn + if not re.match('^[0-9a-z-.+]+$', pn): + return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn elif pn in reserved_names: return 'Recipe name "%s" is invalid: is a reserved keyword' % pn elif pn.startswith('pn-'): @@ -448,13 +439,67 @@ def validate_pn(pn): return '' +def get_bbfile_path(d, destdir, extrapathhint=None): + """ + Determine the correct path for a recipe within a layer + Parameters: + d: Recipe-specific datastore + destdir: destination directory. Can be the path to the base of the layer or a + partial path somewhere within the layer. + extrapathhint: a path relative to the base of the layer to try + """ + import bb.cookerdata + + destdir = os.path.abspath(destdir) + destlayerdir = find_layerdir(destdir) + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + pn = d.getVar('PN') + + bbfilespecs = (confdata.getVar('BBFILES') or '').split() + if destdir == destlayerdir: + for bbfilespec in bbfilespecs: + if not bbfilespec.endswith('.bbappend'): + for match in glob.glob(bbfilespec): + splitext = os.path.splitext(os.path.basename(match)) + if splitext[1] == '.bb': + mpn = splitext[0].split('_')[0] + if mpn == pn: + return os.path.dirname(match) + + # Try to make up a path that matches BBFILES + # this is a little crude, but better than nothing + bpn = d.getVar('BPN') + recipefn = os.path.basename(d.getVar('FILE')) + pathoptions = [destdir] + if extrapathhint: + pathoptions.append(os.path.join(destdir, extrapathhint)) + if destdir == destlayerdir: + pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) + pathoptions.append(os.path.join(destdir, 'recipes', bpn)) + pathoptions.append(os.path.join(destdir, bpn)) + elif not destdir.endswith(('/' + pn, '/' + bpn)): + pathoptions.append(os.path.join(destdir, bpn)) + closepath = '' + for pathoption in pathoptions: + bbfilepath = os.path.join(pathoption, 'test.bb') + for bbfilespec in bbfilespecs: + if fnmatch.fnmatchcase(bbfilepath, bbfilespec): + return pathoption + return None + def get_bbappend_path(d, destlayerdir, wildcardver=False): """Determine how a bbappend for a recipe should be named and located within another layer""" import bb.cookerdata destlayerdir = os.path.abspath(destlayerdir) - recipefile = d.getVar('FILE', True) + recipefile = d.getVar('FILE') recipefn = os.path.splitext(os.path.basename(recipefile))[0] if wildcardver and '_' in recipefn: recipefn = recipefn.split('_', 1)[0] + '_%' @@ -474,7 +519,7 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False): appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn) closepath = '' pathok = True - for bbfilespec in confdata.getVar('BBFILES', True).split(): + for bbfilespec in confdata.getVar('BBFILES').split(): if fnmatch.fnmatchcase(appendpath, bbfilespec): # Our append path works, we're done break @@ -547,7 +592,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, # FIXME check if the bbappend doesn't get overridden by a higher priority layer? - layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()] + layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] if not os.path.abspath(destlayerdir) in layerdirs: bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') @@ -583,7 +628,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, else: bbappendlines.append((varname, op, value)) - destsubdir = rd.getVar('PN', True) + destsubdir = rd.getVar('PN') if srcfiles: bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:')) @@ -602,7 +647,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, srcurientry = 'file://%s' % srcfile # Double-check it's not there already # FIXME do we care if the entry is added by another bbappend that might go away? - if not srcurientry in rd.getVar('SRC_URI', True).split(): + if not srcurientry in rd.getVar('SRC_URI').split(): if machine: appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry) else: @@ -720,7 +765,11 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, for newfile, srcfile in copyfiles.items(): filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile)) if os.path.abspath(newfile) != os.path.abspath(filedest): - bb.note('Copying %s to %s' % (newfile, filedest)) + if newfile.startswith(tempfile.gettempdir()): + newfiledisp = os.path.basename(newfile) + else: + newfiledisp = newfile + bb.note('Copying %s to %s' % (newfiledisp, filedest)) bb.utils.mkdirhier(os.path.dirname(filedest)) shutil.copyfile(newfile, filedest) @@ -728,14 +777,16 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, def find_layerdir(fn): - """ Figure out relative path to base of layer for a file (e.g. a recipe)""" - pth = os.path.dirname(fn) + """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" + pth = fn layerdir = '' while pth: if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): layerdir = pth break pth = os.path.dirname(pth) + if pth == '/': + return None return layerdir @@ -745,7 +796,7 @@ def replace_dir_vars(path, d): # Sort by length so we get the variables we're interested in first for var in sorted(list(d.keys()), key=len): if var.endswith('dir') and var.lower() == var: - value = d.getVar(var, True) + value = d.getVar(var) if value.startswith('/') and not '\n' in value and value not in dirvars: dirvars[value] = var for dirpath in sorted(list(dirvars.keys()), reverse=True): @@ -799,12 +850,12 @@ def get_recipe_upstream_version(rd): ru['type'] = 'U' ru['datetime'] = '' - pv = rd.getVar('PV', True) + pv = rd.getVar('PV') # XXX: If don't have SRC_URI means that don't have upstream sources so # returns the current recipe version, so that upstream version check # declares a match. - src_uris = rd.getVar('SRC_URI', True) + src_uris = rd.getVar('SRC_URI') if not src_uris: ru['version'] = pv ru['type'] = 'M' @@ -815,13 +866,13 @@ def get_recipe_upstream_version(rd): src_uri = src_uris.split()[0] uri_type, _, _, _, _, _ = decodeurl(src_uri) - manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION", True) + manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION") if manual_upstream_version: # manual tracking of upstream version. ru['version'] = manual_upstream_version ru['type'] = 'M' - manual_upstream_date = rd.getVar("CHECK_DATE", True) + manual_upstream_date = rd.getVar("CHECK_DATE") if manual_upstream_date: date = datetime.strptime(manual_upstream_date, "%b %d, %Y") else: diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py index 1fc35bdc78..96591f370e 100644 --- a/meta/lib/oe/rootfs.py +++ b/meta/lib/oe/rootfs.py @@ -15,11 +15,13 @@ class Rootfs(object, metaclass=ABCMeta): This is an abstract class. Do not instantiate this directly. """ - def __init__(self, d): + def __init__(self, d, progress_reporter=None, logcatcher=None): self.d = d self.pm = None - self.image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) - self.deploy_dir_image = self.d.getVar('DEPLOY_DIR_IMAGE', True) + self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') + self.deploydir = self.d.getVar('IMGDEPLOYDIR') + self.progress_reporter = progress_reporter + self.logcatcher = logcatcher self.install_order = Manifest.INSTALL_ORDER @@ -52,6 +54,8 @@ class Rootfs(object, metaclass=ABCMeta): messages = [] with open(log_path, 'r') as log: for line in log: + if self.logcatcher and self.logcatcher.contains(line.rstrip()): + continue for ee in excludes: m = ee.search(line) if m: @@ -68,7 +72,7 @@ class Rootfs(object, metaclass=ABCMeta): else: msg = '%d %s messages' % (len(messages), type) msg = '[log_check] %s: found %s in the logfile:\n%s' % \ - (self.d.getVar('PN', True), msg, ''.join(messages)) + (self.d.getVar('PN'), msg, ''.join(messages)) if type == 'error': bb.fatal(msg) else: @@ -83,7 +87,10 @@ class Rootfs(object, metaclass=ABCMeta): def _insert_feed_uris(self): if bb.utils.contains("IMAGE_FEATURES", "package-management", True, False, self.d): - self.pm.insert_feeds_uris() + self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", + self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", + self.d.getVar('PACKAGE_FEED_ARCHS')) + @abstractmethod def _handle_intercept_failure(self, failed_script): @@ -99,7 +106,7 @@ class Rootfs(object, metaclass=ABCMeta): pass def _setup_dbg_rootfs(self, dirs): - gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS', True) or '0' + gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' if gen_debugfs != '1': return @@ -152,7 +159,7 @@ class Rootfs(object, metaclass=ABCMeta): os.rename(self.image_rootfs + '-orig', self.image_rootfs) def _exec_shell_cmd(self, cmd): - fakerootcmd = self.d.getVar('FAKEROOT', True) + fakerootcmd = self.d.getVar('FAKEROOT') if fakerootcmd is not None: exec_cmd = [fakerootcmd, cmd] else: @@ -167,37 +174,36 @@ class Rootfs(object, metaclass=ABCMeta): def create(self): bb.note("###### Generate rootfs #######") - pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND", True) - post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND", True) - rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND', True) + pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") + post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") + rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') - postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR", True) + postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR") if not postinst_intercepts_dir: postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts") - intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True), + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") bb.utils.remove(intercepts_dir, True) bb.utils.mkdirhier(self.image_rootfs) - bb.utils.mkdirhier(self.deploy_dir_image) + bb.utils.mkdirhier(self.deploydir) shutil.copytree(postinst_intercepts_dir, intercepts_dir) - shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"), - self.deploy_dir_image + - "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt") - execute_pre_post_process(self.d, pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # call the package manager dependent create method self._create() - sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir', True) + sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') bb.utils.mkdirhier(sysconfdir) with open(sysconfdir + "/version", "w+") as ver: - ver.write(self.d.getVar('BUILDNAME', True) + "\n") + ver.write(self.d.getVar('BUILDNAME') + "\n") execute_pre_post_process(self.d, rootfs_post_install_cmds) @@ -205,6 +211,9 @@ class Rootfs(object, metaclass=ABCMeta): execute_pre_post_process(self.d, post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d): delayed_postinsts = self._get_delayed_postinsts() @@ -213,37 +222,51 @@ class Rootfs(object, metaclass=ABCMeta): "offline and rootfs is read-only: %s" % delayed_postinsts) - if self.d.getVar('USE_DEVFS', True) != "1": + if self.d.getVar('USE_DEVFS') != "1": self._create_devfs() self._uninstall_unneeded() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._insert_feed_uris() self._run_ldconfig() - if self.d.getVar('USE_DEPMOD', True) != "0": + if self.d.getVar('USE_DEPMOD') != "0": self._generate_kernel_module_deps() self._cleanup() self._log_check() + if self.progress_reporter: + self.progress_reporter.next_stage() + + def _uninstall_unneeded(self): # Remove unneeded init script symlinks delayed_postinsts = self._get_delayed_postinsts() if delayed_postinsts is None: if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")): self._exec_shell_cmd(["update-rc.d", "-f", "-r", - self.d.getVar('IMAGE_ROOTFS', True), + self.d.getVar('IMAGE_ROOTFS'), "run-postinsts", "remove"]) image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d) - if image_rorfs: + image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') + + if image_rorfs or image_rorfs_force == "1": # Remove components that we don't need if it's a read-only rootfs - unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED", True).split() + unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() pkgs_installed = image_list_installed_packages(self.d) - pkgs_to_remove = [pkg for pkg in pkgs_installed if pkg in unneeded_pkgs] + # Make sure update-alternatives is last on the command line, so + # that it is removed last. This makes sure that its database is + # available while uninstalling packages, allowing alternative + # symlinks of packages to be uninstalled to be managed correctly. + provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") + pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) if len(pkgs_to_remove) > 0: self.pm.remove(pkgs_to_remove, False) @@ -254,7 +277,7 @@ class Rootfs(object, metaclass=ABCMeta): bb.warn("There are post install scripts " "in a read-only rootfs") - post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND", True) + post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") execute_pre_post_process(self.d, post_uninstall_cmds) runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", @@ -264,12 +287,12 @@ class Rootfs(object, metaclass=ABCMeta): self.pm.remove_packaging_data() def _run_intercepts(self): - intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True), + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts") bb.note("Running intercept scripts:") os.environ['D'] = self.image_rootfs - os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE', True) + os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE') for script in os.listdir(intercepts_dir): script_full = os.path.join(intercepts_dir, script) @@ -279,10 +302,10 @@ class Rootfs(object, metaclass=ABCMeta): bb.note("> Executing %s intercept ..." % script) try: - subprocess.check_call(script_full) + subprocess.check_output(script_full) except subprocess.CalledProcessError as e: - bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details!" % - (script, e.returncode)) + bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details! (Output: %s)" % + (script, e.returncode, e.output)) with open(script_full) as intercept: registered_pkgs = None @@ -301,7 +324,7 @@ class Rootfs(object, metaclass=ABCMeta): self._handle_intercept_failure(registered_pkgs) def _run_ldconfig(self): - if self.d.getVar('LDCONFIGDEPEND', True): + if self.d.getVar('LDCONFIGDEPEND'): bb.note("Executing: ldconfig -r" + self.image_rootfs + "-c new -v") self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 'new', '-v']) @@ -321,7 +344,7 @@ class Rootfs(object, metaclass=ABCMeta): bb.note("No Kernel Modules found, not running depmod") return - kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR', True), "kernel-depmod", + kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod", 'kernel-abiversion') if not os.path.exists(kernel_abi_ver_file): bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) @@ -343,15 +366,15 @@ class Rootfs(object, metaclass=ABCMeta): """ def _create_devfs(self): devtable_list = [] - devtable = self.d.getVar('IMAGE_DEVICE_TABLE', True) + devtable = self.d.getVar('IMAGE_DEVICE_TABLE') if devtable is not None: devtable_list.append(devtable) else: - devtables = self.d.getVar('IMAGE_DEVICE_TABLES', True) + devtables = self.d.getVar('IMAGE_DEVICE_TABLES') if devtables is None: devtables = 'files/device_table-minimal.txt' for devtable in devtables.split(): - devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH', True), devtable)) + devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) for devtable in devtable_list: self._exec_shell_cmd(["makedevs", "-r", @@ -359,24 +382,24 @@ class Rootfs(object, metaclass=ABCMeta): class RpmRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(RpmRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(RpmRootfs, self).__init__(d, progress_reporter, logcatcher) self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ '|exit 1|ERROR: |Error: |Error |ERROR '\ '|Failed |Failed: |Failed$|Failed\(\d+\):)' self.manifest = RpmManifest(d, manifest_dir) self.pm = RpmPM(d, - d.getVar('IMAGE_ROOTFS', True), - self.d.getVar('TARGET_VENDOR', True) + d.getVar('IMAGE_ROOTFS'), + self.d.getVar('TARGET_VENDOR') ) - self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN', True) + self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN') if self.inc_rpm_image_gen != "1": bb.utils.remove(self.image_rootfs, True) else: self.pm.recovery_packaging_data() - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) self.pm.create_configs() @@ -408,21 +431,27 @@ class RpmRootfs(Rootfs): bb.note('incremental removed: %s' % ' '.join(pkg_to_remove)) self.pm.remove(pkg_to_remove) + self.pm.autoremove() + def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() - rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS', True) - rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS', True) + rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') + rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') # update PM index files self.pm.write_index() execute_pre_post_process(self.d, rpm_pre_process_cmds) - self.pm.dump_all_available_pkgs() + if self.progress_reporter: + self.progress_reporter.next_stage() if self.inc_rpm_image_gen == "1": self._create_incremental(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.update() pkgs = [] @@ -433,22 +462,34 @@ class RpmRootfs(Rootfs): else: pkgs += pkgs_to_install[pkg_type] + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs_attempt, True) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() - self._setup_dbg_rootfs(['/etc/rpm', '/var/lib/rpm', '/var/lib/smart']) + if self.progress_reporter: + self.progress_reporter.next_stage() - execute_pre_post_process(self.d, rpm_post_process_cmds) + self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) - self._log_check() + execute_pre_post_process(self.d, rpm_post_process_cmds) if self.inc_rpm_image_gen == "1": self.pm.backup_packaging_data() - self.pm.rpm_setup_smart_target_config() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): @@ -483,19 +524,11 @@ class RpmRootfs(Rootfs): self.pm.save_rpmpostinst(pkg) def _cleanup(self): - # during the execution of postprocess commands, rpm is called several - # times to get the files installed, dependencies, etc. This creates the - # __db.00* (Berkeley DB files that hold locks, rpm specific environment - # settings, etc.), that should not get into the final rootfs - self.pm.unlock_rpm_db() - if os.path.isdir(self.pm.install_dir_path + "/tmp") and not os.listdir(self.pm.install_dir_path + "/tmp"): - bb.utils.remove(self.pm.install_dir_path + "/tmp", True) - if os.path.isdir(self.pm.install_dir_path) and not os.listdir(self.pm.install_dir_path): - bb.utils.remove(self.pm.install_dir_path, True) + pass class DpkgOpkgRootfs(Rootfs): - def __init__(self, d): - super(DpkgOpkgRootfs, self).__init__(d) + def __init__(self, d, progress_reporter=None, logcatcher=None): + super(DpkgOpkgRootfs, self).__init__(d, progress_reporter, logcatcher) def _get_pkgs_postinsts(self, status_file): def _get_pkg_depends_list(pkg_depends): @@ -562,7 +595,7 @@ class DpkgOpkgRootfs(Rootfs): pkg_list = [] pkgs = None - if not self.d.getVar('PACKAGE_INSTALL', True).strip(): + if not self.d.getVar('PACKAGE_INSTALL').strip(): bb.note("Building empty image") else: pkgs = self._get_pkgs_postinsts(status_file) @@ -589,8 +622,8 @@ class DpkgOpkgRootfs(Rootfs): num += 1 class DpkgRootfs(DpkgOpkgRootfs): - def __init__(self, d, manifest_dir): - super(DpkgRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(DpkgRootfs, self).__init__(d, progress_reporter, logcatcher) self.log_check_regex = '^E:' self.log_check_expected_regexes = \ [ @@ -598,17 +631,17 @@ class DpkgRootfs(DpkgOpkgRootfs): ] bb.utils.remove(self.image_rootfs, True) - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) self.manifest = DpkgManifest(d, manifest_dir) - self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS', True), - d.getVar('PACKAGE_ARCHS', True), - d.getVar('DPKG_ARCH', True)) + self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS'), + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH')) def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() - deb_pre_process_cmds = self.d.getVar('DEB_PREPROCESS_COMMANDS', True) - deb_post_process_cmds = self.d.getVar('DEB_POSTPROCESS_COMMANDS', True) + deb_pre_process_cmds = self.d.getVar('DEB_PREPROCESS_COMMANDS') + deb_post_process_cmds = self.d.getVar('DEB_POSTPROCESS_COMMANDS') alt_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/alternatives") bb.utils.mkdirhier(alt_dir) @@ -618,15 +651,31 @@ class DpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Don't support incremental, so skip that + self.progress_reporter.next_stage() + self.pm.update() + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + # Don't support attemptonly, so skip that + self.progress_reporter.next_stage() + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._setup_dbg_rootfs(['/var/lib/dpkg']) self.pm.fix_broken_dependencies() @@ -637,6 +686,9 @@ class DpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS'] @@ -662,15 +714,15 @@ class DpkgRootfs(DpkgOpkgRootfs): class OpkgRootfs(DpkgOpkgRootfs): - def __init__(self, d, manifest_dir): - super(OpkgRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(OpkgRootfs, self).__init__(d, progress_reporter, logcatcher) self.log_check_regex = '(exit 1|Collected errors)' self.manifest = OpkgManifest(d, manifest_dir) - self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True) - self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True) + self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") + self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS") - self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or "" + self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN') or "" if self._remove_old_rootfs(): bb.utils.remove(self.image_rootfs, True) self.pm = OpkgPM(d, @@ -684,7 +736,7 @@ class OpkgRootfs(DpkgOpkgRootfs): self.pkg_archs) self.pm.recover_packaging_data() - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) def _prelink_file(self, root_dir, filename): bb.note('prelink %s in %s' % (filename, root_dir)) @@ -739,7 +791,7 @@ class OpkgRootfs(DpkgOpkgRootfs): """ def _multilib_sanity_test(self, dirs): - allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP", True) + allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") if allow_replace is None: allow_replace = "" @@ -771,12 +823,12 @@ class OpkgRootfs(DpkgOpkgRootfs): files[key] = item def _multilib_test_install(self, pkgs): - ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS", True) + ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") bb.utils.mkdirhier(ml_temp) dirs = [self.image_rootfs] - for variant in self.d.getVar("MULTILIB_VARIANTS", True).split(): + for variant in self.d.getVar("MULTILIB_VARIANTS").split(): ml_target_rootfs = os.path.join(ml_temp, variant) bb.utils.remove(ml_target_rootfs, True) @@ -836,9 +888,9 @@ class OpkgRootfs(DpkgOpkgRootfs): old_vars_list = open(vars_list_file, 'r+').read() new_vars_list = '%s:%s:%s\n' % \ - ((self.d.getVar('BAD_RECOMMENDATIONS', True) or '').strip(), - (self.d.getVar('NO_RECOMMENDATIONS', True) or '').strip(), - (self.d.getVar('PACKAGE_EXCLUDE', True) or '').strip()) + ((self.d.getVar('BAD_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('NO_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('PACKAGE_EXCLUDE') or '').strip()) open(vars_list_file, 'w+').write(new_vars_list) if old_vars_list != new_vars_list: @@ -848,22 +900,33 @@ class OpkgRootfs(DpkgOpkgRootfs): def _create(self): pkgs_to_install = self.manifest.parse_initial_manifest() - opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True) - opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS', True) + opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') + opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') # update PM index files, unless users provide their own feeds - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": self.pm.write_index() execute_pre_post_process(self.d, opkg_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Steps are a bit different in order, skip next + self.progress_reporter.next_stage() + self.pm.update() self.pm.handle_bad_recommendations() + if self.progress_reporter: + self.progress_reporter.next_stage() + if self.inc_opkg_image_gen == "1": self._remove_extra_packages(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: # For multilib, we perform a sanity test before final install @@ -875,22 +938,33 @@ class OpkgRootfs(DpkgOpkgRootfs): self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() - self._setup_dbg_rootfs(['/etc', '/var/lib/opkg', '/usr/lib/ssl']) + if self.progress_reporter: + self.progress_reporter.next_stage() + + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') + self._setup_dbg_rootfs([opkg_dir]) execute_pre_post_process(self.d, opkg_post_process_cmds) if self.inc_opkg_image_gen == "1": self.pm.backup_packaging_data() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] def _get_delayed_postinsts(self): status_file = os.path.join(self.image_rootfs, - self.d.getVar('OPKGLIBDIR', True).strip('/'), + self.d.getVar('OPKGLIBDIR').strip('/'), "opkg", "status") return self._get_delayed_postinsts_common(status_file) @@ -915,20 +989,20 @@ def get_class_for_type(imgtype): "deb": DpkgRootfs}[imgtype] def variable_depends(d, manifest_dir=None): - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') cls = get_class_for_type(img_type) return cls._depends_list() -def create_rootfs(d, manifest_dir=None): +def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): env_bkp = os.environ.copy() - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": - RpmRootfs(d, manifest_dir).create() + RpmRootfs(d, manifest_dir, progress_reporter, logcatcher).create() elif img_type == "ipk": - OpkgRootfs(d, manifest_dir).create() + OpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() elif img_type == "deb": - DpkgRootfs(d, manifest_dir).create() + DpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() os.environ.clear() os.environ.update(env_bkp) @@ -936,13 +1010,13 @@ def create_rootfs(d, manifest_dir=None): def image_list_installed_packages(d, rootfs_dir=None): if not rootfs_dir: - rootfs_dir = d.getVar('IMAGE_ROOTFS', True) + rootfs_dir = d.getVar('IMAGE_ROOTFS') - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": return RpmPkgsList(d, rootfs_dir).list_pkgs() elif img_type == "ipk": - return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET", True)).list_pkgs() + return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET")).list_pkgs() elif img_type == "deb": return DpkgPkgsList(d, rootfs_dir).list_pkgs() diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py index c74525f929..deb823b6ec 100644 --- a/meta/lib/oe/sdk.py +++ b/meta/lib/oe/sdk.py @@ -11,16 +11,16 @@ import traceback class Sdk(object, metaclass=ABCMeta): def __init__(self, d, manifest_dir): self.d = d - self.sdk_output = self.d.getVar('SDK_OUTPUT', True) - self.sdk_native_path = self.d.getVar('SDKPATHNATIVE', True).strip('/') - self.target_path = self.d.getVar('SDKTARGETSYSROOT', True).strip('/') - self.sysconfdir = self.d.getVar('sysconfdir', True).strip('/') + self.sdk_output = self.d.getVar('SDK_OUTPUT') + self.sdk_native_path = self.d.getVar('SDKPATHNATIVE').strip('/') + self.target_path = self.d.getVar('SDKTARGETSYSROOT').strip('/') + self.sysconfdir = self.d.getVar('sysconfdir').strip('/') self.sdk_target_sysroot = os.path.join(self.sdk_output, self.target_path) self.sdk_host_sysroot = self.sdk_output if manifest_dir is None: - self.manifest_dir = self.d.getVar("SDK_DIR", True) + self.manifest_dir = self.d.getVar("SDK_DIR") else: self.manifest_dir = manifest_dir @@ -40,12 +40,12 @@ class Sdk(object, metaclass=ABCMeta): # Don't ship any libGL in the SDK self.remove(os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('libdir_nativesdk', True).strip('/'), + self.d.getVar('libdir_nativesdk').strip('/'), "libGL*")) # Fix or remove broken .la files self.remove(os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('libdir_nativesdk', True).strip('/'), + self.d.getVar('libdir_nativesdk').strip('/'), "*.la")) # Link the ld.so.cache file into the hosts filesystem @@ -54,7 +54,7 @@ class Sdk(object, metaclass=ABCMeta): self.mkdirhier(os.path.dirname(link_name)) os.symlink("/etc/ld.so.cache", link_name) - execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND', True)) + execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND')) def movefile(self, sourcefile, destdir): try: @@ -102,7 +102,7 @@ class RpmSdk(Sdk): self.target_pm = RpmPM(d, self.sdk_target_sysroot, - self.d.getVar('TARGET_VENDOR', True), + self.d.getVar('TARGET_VENDOR'), 'target', target_providename ) @@ -118,7 +118,7 @@ class RpmSdk(Sdk): self.host_pm = RpmPM(d, self.sdk_host_sysroot, - self.d.getVar('SDK_VENDOR', True), + self.d.getVar('SDK_VENDOR'), 'host', sdk_providename, "SDK_PACKAGE_ARCHS", @@ -130,7 +130,6 @@ class RpmSdk(Sdk): pm.create_configs() pm.write_index() - pm.dump_all_available_pkgs() pm.update() pkgs = [] @@ -149,9 +148,9 @@ class RpmSdk(Sdk): bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) - self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY', True)) + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): self.target_pm.remove_packaging_data() @@ -159,7 +158,7 @@ class RpmSdk(Sdk): bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): self.host_pm.remove_packaging_data() @@ -167,7 +166,7 @@ class RpmSdk(Sdk): # Move host RPM library data native_rpm_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('localstatedir_nativesdk', True).strip('/'), + self.d.getVar('localstatedir_nativesdk').strip('/'), "lib", "rpm" ) @@ -188,7 +187,9 @@ class RpmSdk(Sdk): True).strip('/'), ) self.mkdirhier(native_sysconf_dir) - for f in glob.glob(os.path.join(self.sdk_output, "etc", "*")): + for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")): + self.movefile(f, native_sysconf_dir) + for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", "*")): self.movefile(f, native_sysconf_dir) self.remove(os.path.join(self.sdk_output, "etc"), True) @@ -197,8 +198,8 @@ class OpkgSdk(Sdk): def __init__(self, d, manifest_dir=None): super(OpkgSdk, self).__init__(d, manifest_dir) - self.target_conf = self.d.getVar("IPKGCONF_TARGET", True) - self.host_conf = self.d.getVar("IPKGCONF_SDK", True) + self.target_conf = self.d.getVar("IPKGCONF_TARGET") + self.host_conf = self.d.getVar("IPKGCONF_SDK") self.target_manifest = OpkgManifest(d, self.manifest_dir, Manifest.MANIFEST_TYPE_SDK_TARGET) @@ -206,15 +207,15 @@ class OpkgSdk(Sdk): Manifest.MANIFEST_TYPE_SDK_HOST) self.target_pm = OpkgPM(d, self.sdk_target_sysroot, self.target_conf, - self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)) + self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) self.host_pm = OpkgPM(d, self.sdk_host_sysroot, self.host_conf, - self.d.getVar("SDK_PACKAGE_ARCHS", True)) + self.d.getVar("SDK_PACKAGE_ARCHS")) def _populate_sysroot(self, pm, manifest): pkgs_to_install = manifest.parse_initial_manifest() - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": pm.write_index() pm.update() @@ -228,9 +229,9 @@ class OpkgSdk(Sdk): bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) - self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY', True)) + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): self.target_pm.remove_packaging_data() @@ -238,7 +239,7 @@ class OpkgSdk(Sdk): bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): self.host_pm.remove_packaging_data() @@ -257,7 +258,7 @@ class OpkgSdk(Sdk): os.path.basename(self.host_conf)), 0o644) native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('localstatedir_nativesdk', True).strip('/'), + self.d.getVar('localstatedir_nativesdk').strip('/'), "lib", "opkg") self.mkdirhier(native_opkg_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "opkg", "*")): @@ -270,8 +271,8 @@ class DpkgSdk(Sdk): def __init__(self, d, manifest_dir=None): super(DpkgSdk, self).__init__(d, manifest_dir) - self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET", True), "apt") - self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET", True), "apt-sdk") + self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt") + self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt-sdk") self.target_manifest = DpkgManifest(d, self.manifest_dir, Manifest.MANIFEST_TYPE_SDK_TARGET) @@ -279,17 +280,17 @@ class DpkgSdk(Sdk): Manifest.MANIFEST_TYPE_SDK_HOST) self.target_pm = DpkgPM(d, self.sdk_target_sysroot, - self.d.getVar("PACKAGE_ARCHS", True), - self.d.getVar("DPKG_ARCH", True), + self.d.getVar("PACKAGE_ARCHS"), + self.d.getVar("DPKG_ARCH"), self.target_conf_dir) self.host_pm = DpkgPM(d, self.sdk_host_sysroot, - self.d.getVar("SDK_PACKAGE_ARCHS", True), - self.d.getVar("DEB_SDK_ARCH", True), + self.d.getVar("SDK_PACKAGE_ARCHS"), + self.d.getVar("DEB_SDK_ARCH"), self.host_conf_dir) def _copy_apt_dir_to(self, dst_dir): - staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE", True) + staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE") self.remove(dst_dir, True) @@ -310,9 +311,9 @@ class DpkgSdk(Sdk): bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) - self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY', True)) + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) self._copy_apt_dir_to(os.path.join(self.sdk_target_sysroot, "etc", "apt")) @@ -322,7 +323,7 @@ class DpkgSdk(Sdk): bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) - execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) self._copy_apt_dir_to(os.path.join(self.sdk_output, self.sdk_native_path, "etc", "apt")) @@ -341,26 +342,26 @@ class DpkgSdk(Sdk): def sdk_list_installed_packages(d, target, rootfs_dir=None): if rootfs_dir is None: - sdk_output = d.getVar('SDK_OUTPUT', True) - target_path = d.getVar('SDKTARGETSYSROOT', True).strip('/') + sdk_output = d.getVar('SDK_OUTPUT') + target_path = d.getVar('SDKTARGETSYSROOT').strip('/') rootfs_dir = [sdk_output, os.path.join(sdk_output, target_path)][target is True] - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": arch_var = ["SDK_PACKAGE_ARCHS", None][target is True] os_var = ["SDK_OS", None][target is True] - return RpmPkgsList(d, rootfs_dir, arch_var, os_var).list_pkgs() + return RpmPkgsList(d, rootfs_dir).list_pkgs() elif img_type == "ipk": conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True] - return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var, True)).list_pkgs() + return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs() elif img_type == "deb": return DpkgPkgsList(d, rootfs_dir).list_pkgs() def populate_sdk(d, manifest_dir=None): env_bkp = os.environ.copy() - img_type = d.getVar('IMAGE_PKGTYPE', True) + img_type = d.getVar('IMAGE_PKGTYPE') if img_type == "rpm": RpmSdk(d, manifest_dir).populate() elif img_type == "ipk": diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index a58f03a342..f087a019e1 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py @@ -63,10 +63,10 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): def sstate_lockedsigs(d): sigs = {} - types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split() + types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split() for t in types: siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t - lockedsigs = (d.getVar(siggen_lockedsigs_var, True) or "").split() + lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split() for ls in lockedsigs: pn, task, h = ls.split(":", 2) if pn not in sigs: @@ -77,8 +77,8 @@ def sstate_lockedsigs(d): class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): name = "OEBasic" def init_rundepcheck(self, data): - self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split() - self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split() + self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() pass def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) @@ -86,15 +86,15 @@ class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): name = "OEBasicHash" def init_rundepcheck(self, data): - self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split() - self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split() + self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() self.lockedsigs = sstate_lockedsigs(data) self.lockedhashes = {} self.lockedpnmap = {} self.lockedhashfn = {} - self.machine = data.getVar("MACHINE", True) + self.machine = data.getVar("MACHINE") self.mismatch_msgs = [] - self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES", True) or + self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or "").split() self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } pass @@ -130,7 +130,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata) def dump_sigs(self, dataCache, options): - self.dump_lockedsigs() + sigfile = os.getcwd() + "/locked-sigs.inc" + bb.plain("Writing locked sigs to %s" % sigfile) + self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) def get_taskhash(self, fn, task, deps, dataCache): @@ -181,11 +183,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) - def dump_lockedsigs(self, sigfile=None, taskfilter=None): - if not sigfile: - sigfile = os.getcwd() + "/locked-sigs.inc" - - bb.plain("Writing locked sigs to %s" % sigfile) + def dump_lockedsigs(self, sigfile, taskfilter=None): types = {} for k in self.runtaskdeps: if taskfilter: @@ -199,7 +197,8 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): types[t].append(k) with open(sigfile, "w") as f: - for t in types: + l = sorted(types) + for t in l: f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) types[t].sort() sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]]) @@ -210,7 +209,17 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): continue f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n") f.write(' "\n') - f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(list(types.keys())))) + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(l))) + + def dump_siglist(self, sigfile): + with open(sigfile, "w") as f: + tasks = [] + for taskitem in self.taskhash: + (fn, task) = taskitem.rsplit(".", 1) + pn = self.lockedpnmap[fn] + tasks.append((pn, task, fn, self.taskhash[taskitem])) + for (pn, task, fn, taskhash) in sorted(tasks): + f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash)) def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): warn_msgs = [] @@ -226,13 +235,13 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task])) - checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK", True) + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") if checklevel == 'warn': warn_msgs += self.mismatch_msgs elif checklevel == 'error': error_msgs += self.mismatch_msgs - checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK", True) + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK") if checklevel == 'warn': warn_msgs += sstate_missing_msgs elif checklevel == 'error': @@ -255,9 +264,6 @@ def find_siginfo(pn, taskname, taskhashlist, d): import fnmatch import glob - if taskhashlist: - hashfiles = {} - if not taskname: # We have to derive pn and taskname key = pn @@ -267,8 +273,15 @@ def find_siginfo(pn, taskname, taskhashlist, d): if key.startswith('virtual:native:'): pn = pn + '-native' + hashfiles = {} filedates = {} + def get_hashval(siginfo): + if siginfo.endswith('.siginfo'): + return siginfo.rpartition(':')[2].partition('_')[0] + else: + return siginfo.rpartition('.')[2] + # First search in stamps dir localdata = d.createCopy() localdata.setVar('MULTIMACH_TARGET_SYS', '*') @@ -276,7 +289,7 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('PV', '*') localdata.setVar('PR', '*') localdata.setVar('EXTENDPE', '') - stamp = localdata.getVar('STAMP', True) + stamp = localdata.getVar('STAMP') if pn.startswith("gcc-source"): # gcc-source shared workdir is a special case :( stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") @@ -298,6 +311,8 @@ def find_siginfo(pn, taskname, taskhashlist, d): filedates[fullpath] = os.stat(fullpath).st_mtime except OSError: continue + hashval = get_hashval(fullpath) + hashfiles[hashval] = fullpath if not taskhashlist or (len(filedates) < 2 and not foundall): # That didn't work, look in sstate-cache @@ -311,30 +326,25 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('PV', '*') localdata.setVar('PR', '*') localdata.setVar('BB_TASKHASH', hashval) - swspec = localdata.getVar('SSTATE_SWSPEC', True) + swspec = localdata.getVar('SSTATE_SWSPEC') if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec: localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}') elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") sstatename = taskname[3:] - filespec = '%s_%s.*.siginfo' % (localdata.getVar('SSTATE_PKG', True), sstatename) + filespec = '%s_%s.*.siginfo' % (localdata.getVar('SSTATE_PKG'), sstatename) - if hashval != '*': - sstatedir = "%s/%s" % (d.getVar('SSTATE_DIR', True), hashval[:2]) - else: - sstatedir = d.getVar('SSTATE_DIR', True) - - for root, dirs, files in os.walk(sstatedir): - for fn in files: - fullpath = os.path.join(root, fn) - if fnmatch.fnmatch(fullpath, filespec): - if taskhashlist: - hashfiles[hashval] = fullpath - else: - try: - filedates[fullpath] = os.stat(fullpath).st_mtime - except: - continue + matchedfiles = glob.glob(filespec) + for fullpath in matchedfiles: + actual_hashval = get_hashval(fullpath) + if actual_hashval in hashfiles: + continue + hashfiles[hashval] = fullpath + if not taskhashlist: + try: + filedates[fullpath] = os.stat(fullpath).st_mtime + except: + continue if taskhashlist: return hashfiles @@ -350,7 +360,7 @@ def sstate_get_manifest_filename(task, d): Also returns the datastore that can be used to query related variables. """ d2 = d.createCopy() - extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info', True) + extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info') if extrainf: d2.setVar("SSTATE_MANMACH", extrainf) return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2) diff --git a/meta/lib/oe/terminal.py b/meta/lib/oe/terminal.py index dc25d14ff6..0426e15834 100644 --- a/meta/lib/oe/terminal.py +++ b/meta/lib/oe/terminal.py @@ -11,7 +11,8 @@ class UnsupportedTerminal(Exception): pass class NoSupportedTerminals(Exception): - pass + def __init__(self, terms): + self.terms = terms class Registry(oe.classutils.ClassRegistry): @@ -51,7 +52,7 @@ class XTerminal(Terminal): raise UnsupportedTerminal(self.name) class Gnome(XTerminal): - command = 'gnome-terminal -t "{title}" --disable-factory -x {command}' + command = 'gnome-terminal -t "{title}" -x {command}' priority = 2 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -61,12 +62,28 @@ class Gnome(XTerminal): # Once fixed on the gnome-terminal project, this should be removed. if os.getenv('LC_ALL'): os.putenv('LC_ALL','') - # Check version - vernum = check_terminal_version("gnome-terminal") - if vernum and LooseVersion(vernum) >= '3.10': - logger.debug(1, 'Gnome-Terminal 3.10 or later does not support --disable-factory') - self.command = 'gnome-terminal -t "{title}" -x {command}' - XTerminal.__init__(self, sh_cmd, title, env, d) + # We need to know when the command completes but gnome-terminal gives us no way + # to do this. We therefore write the pid to a file using a "phonehome" wrapper + # script, then monitor the pid until it exits. Thanks gnome! + import tempfile + pidfile = tempfile.NamedTemporaryFile(delete = False).name + try: + sh_cmd = "oe-gnome-terminal-phonehome " + pidfile + " " + sh_cmd + XTerminal.__init__(self, sh_cmd, title, env, d) + while os.stat(pidfile).st_size <= 0: + continue + with open(pidfile, "r") as f: + pid = int(f.readline()) + finally: + os.unlink(pidfile) + + import time + while True: + try: + os.kill(pid, 0) + time.sleep(0.1) + except OSError: + return class Mate(XTerminal): command = 'mate-terminal -t "{title}" -x {command}' @@ -81,7 +98,7 @@ class Terminology(XTerminal): priority = 2 class Konsole(XTerminal): - command = 'konsole --nofork --workdir . -p tabtitle="{title}" -e {command}' + command = 'konsole --separate --workdir . -p tabtitle="{title}" -e {command}' priority = 2 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -90,6 +107,9 @@ class Konsole(XTerminal): if vernum and LooseVersion(vernum) < '2.0.0': # Konsole from KDE 3.x self.command = 'konsole -T "{title}" -e {command}' + elif vernum and LooseVersion(vernum) < '16.08.1': + # Konsole pre 16.08.01 Has nofork + self.command = 'konsole --nofork --workdir . -p tabtitle="{title}" -e {command}' XTerminal.__init__(self, sh_cmd, title, env, d) class XTerm(XTerminal): @@ -176,7 +196,7 @@ class Custom(Terminal): priority = 3 def __init__(self, sh_cmd, title=None, env=None, d=None): - self.command = d and d.getVar('OE_TERMINAL_CUSTOMCMD', True) + self.command = d and d.getVar('OE_TERMINAL_CUSTOMCMD') if self.command: if not '{command}' in self.command: self.command += ' {command}' @@ -190,6 +210,14 @@ class Custom(Terminal): def prioritized(): return Registry.prioritized() +def get_cmd_list(): + terms = Registry.prioritized() + cmds = [] + for term in terms: + if term.command: + cmds.append(term.command) + return cmds + def spawn_preferred(sh_cmd, title=None, env=None, d=None): """Spawn the first supported terminal, by priority""" for terminal in prioritized(): @@ -199,7 +227,7 @@ def spawn_preferred(sh_cmd, title=None, env=None, d=None): except UnsupportedTerminal: continue else: - raise NoSupportedTerminals() + raise NoSupportedTerminals(get_cmd_list()) def spawn(name, sh_cmd, title=None, env=None, d=None): """Spawn the specified terminal, by name""" @@ -211,6 +239,8 @@ def spawn(name, sh_cmd, title=None, env=None, d=None): pipe = terminal(sh_cmd, title, env, d) output = pipe.communicate()[0] + if output: + output = output.decode("utf-8") if pipe.returncode != 0: raise ExecutionError(sh_cmd, pipe.returncode, output) @@ -246,7 +276,7 @@ def check_terminal_version(terminalName): newenv["LANG"] = "C" p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv) out, err = p.communicate() - ver_info = out.rstrip().split('\n') + ver_info = out.decode().rstrip().split('\n') except OSError as exc: import errno if exc.errno == errno.ENOENT: diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index cecddc657f..330a5ff94a 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -1,9 +1,4 @@ -try: - # Python 2 - import commands as cmdstatus -except ImportError: - # Python 3 - import subprocess as cmdstatus +import subprocess def read_file(filename): try: @@ -23,27 +18,27 @@ def ifelse(condition, iftrue = True, iffalse = False): return iffalse def conditional(variable, checkvalue, truevalue, falsevalue, d): - if d.getVar(variable,1) == checkvalue: + if d.getVar(variable) == checkvalue: return truevalue else: return falsevalue def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - if float(d.getVar(variable,1)) <= float(checkvalue): + if float(d.getVar(variable)) <= float(checkvalue): return truevalue else: return falsevalue def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - result = bb.utils.vercmp_string(d.getVar(variable,True), checkvalue) + result = bb.utils.vercmp_string(d.getVar(variable), checkvalue) if result <= 0: return truevalue else: return falsevalue def both_contain(variable1, variable2, checkvalue, d): - val1 = d.getVar(variable1, True) - val2 = d.getVar(variable2, True) + val1 = d.getVar(variable1) + val2 = d.getVar(variable2) val1 = set(val1.split()) val2 = set(val2.split()) if isinstance(checkvalue, str): @@ -66,8 +61,8 @@ def set_intersect(variable1, variable2, d): s3 = set_intersect(s1, s2) => s3 = "b c" """ - val1 = set(d.getVar(variable1, True).split()) - val2 = set(d.getVar(variable2, True).split()) + val1 = set(d.getVar(variable1).split()) + val2 = set(d.getVar(variable2).split()) return " ".join(val1 & val2) def prune_suffix(var, suffixes, d): @@ -77,7 +72,7 @@ def prune_suffix(var, suffixes, d): if var.endswith(suffix): var = var.replace(suffix, "") - prefix = d.getVar("MLPREFIX", True) + prefix = d.getVar("MLPREFIX") if prefix and var.startswith(prefix): var = var.replace(prefix, "") @@ -102,6 +97,10 @@ def param_bool(cfg, field, dflt = None): return False raise ValueError("invalid value for boolean parameter '%s': '%s'" % (field, value)) +def build_depends_string(depends, task): + """Append a taskname to a string of dependencies as used by the [depends] flag""" + return " ".join(dep + ":" + task for dep in depends.split()) + def inherits(d, *classes): """Return True if the metadata inherits any of the specified classes""" return any(bb.data.inherits_class(cls, d) for cls in classes) @@ -115,9 +114,9 @@ def features_backfill(var,d): # disturbing distributions that have already set DISTRO_FEATURES. # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED - features = (d.getVar(var, True) or "").split() - backfill = (d.getVar(var+"_BACKFILL", True) or "").split() - considered = (d.getVar(var+"_BACKFILL_CONSIDERED", True) or "").split() + features = (d.getVar(var) or "").split() + backfill = (d.getVar(var+"_BACKFILL") or "").split() + considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split() addfeatures = [] for feature in backfill: @@ -133,18 +132,18 @@ def packages_filter_out_system(d): Return a list of packages from PACKAGES with the "system" packages such as PN-dbg PN-doc PN-locale-eb-gb removed. """ - pn = d.getVar('PN', True) + pn = d.getVar('PN') blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] localepkg = pn + "-locale-" pkgs = [] - for pkg in d.getVar('PACKAGES', True).split(): + for pkg in d.getVar('PACKAGES').split(): if pkg not in blacklist and localepkg not in pkg: pkgs.append(pkg) return pkgs def getstatusoutput(cmd): - return cmdstatus.getstatusoutput(cmd) + return subprocess.getstatusoutput(cmd) def trim_version(version, num_parts=2): @@ -230,6 +229,24 @@ def format_pkg_list(pkg_dict, ret_format=None): return '\n'.join(output) +def host_gcc_version(d): + import re, subprocess + + compiler = d.getVar("BUILD_CC") + try: + env = os.environ.copy() + env["PATH"] = d.getVar("PATH") + output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) + + match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + if not match: + bb.fatal("Can't get compiler version from %s --version output" % compiler) + + version = match.group(1) + return "-%s" % version if version in ("4.8", "4.9") else "" + # # Python 2.7 doesn't have threaded pools (just multiprocessing) # so implement a version here @@ -302,5 +319,18 @@ def write_ld_so_conf(d): bb.utils.remove(ldsoconf) bb.utils.mkdirhier(os.path.dirname(ldsoconf)) with open(ldsoconf, "w") as f: - f.write(d.getVar("base_libdir", True) + '\n') - f.write(d.getVar("libdir", True) + '\n') + f.write(d.getVar("base_libdir") + '\n') + f.write(d.getVar("libdir") + '\n') + +class ImageQAFailed(bb.build.FuncFailed): + def __init__(self, description, name=None, logfile=None): + self.description = description + self.name = name + self.logfile=logfile + + def __str__(self): + msg = 'Function failed: %s' % self.name + if self.description: + msg = msg + ' (%s)' % self.description + + return msg diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py new file mode 100644 index 0000000000..605f429ecc --- /dev/null +++ b/meta/lib/oeqa/buildperf/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance tests""" +from .base import (BuildPerfTestCase, + BuildPerfTestLoader, + BuildPerfTestResult, + BuildPerfTestRunner, + KernelDropCaches, + runCmd2) +from .test_basic import * diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py new file mode 100644 index 0000000000..6e62b279c1 --- /dev/null +++ b/meta/lib/oeqa/buildperf/base.py @@ -0,0 +1,510 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance test base classes and functionality""" +import json +import logging +import os +import re +import resource +import socket +import shutil +import time +import unittest +import xml.etree.ElementTree as ET +from collections import OrderedDict +from datetime import datetime, timedelta +from functools import partial +from multiprocessing import Process +from multiprocessing import SimpleQueue +from xml.dom import minidom + +import oe.path +from oeqa.utils.commands import CommandError, runCmd, get_bb_vars +from oeqa.utils.git import GitError, GitRepo + +# Get logger for this module +log = logging.getLogger('build-perf') + +# Our own version of runCmd which does not raise AssertErrors which would cause +# errors to interpreted as failures +runCmd2 = partial(runCmd, assert_error=False, limit_exc_output=40) + + +class KernelDropCaches(object): + """Container of the functions for dropping kernel caches""" + sudo_passwd = None + + @classmethod + def check(cls): + """Check permssions for dropping kernel caches""" + from getpass import getpass + from locale import getdefaultlocale + cmd = ['sudo', '-k', '-n', 'tee', '/proc/sys/vm/drop_caches'] + ret = runCmd2(cmd, ignore_status=True, data=b'0') + if ret.output.startswith('sudo:'): + pass_str = getpass( + "\nThe script requires sudo access to drop caches between " + "builds (echo 3 > /proc/sys/vm/drop_caches).\n" + "Please enter your sudo password: ") + cls.sudo_passwd = bytes(pass_str, getdefaultlocale()[1]) + + @classmethod + def drop(cls): + """Drop kernel caches""" + cmd = ['sudo', '-k'] + if cls.sudo_passwd: + cmd.append('-S') + input_data = cls.sudo_passwd + b'\n' + else: + cmd.append('-n') + input_data = b'' + cmd += ['tee', '/proc/sys/vm/drop_caches'] + input_data += b'3' + runCmd2(cmd, data=input_data) + + +def str_to_fn(string): + """Convert string to a sanitized filename""" + return re.sub(r'(\W+)', '-', string, flags=re.LOCALE) + + +class ResultsJsonEncoder(json.JSONEncoder): + """Extended encoder for build perf test results""" + unix_epoch = datetime.utcfromtimestamp(0) + + def default(self, obj): + """Encoder for our types""" + if isinstance(obj, datetime): + # NOTE: we assume that all timestamps are in UTC time + return (obj - self.unix_epoch).total_seconds() + if isinstance(obj, timedelta): + return obj.total_seconds() + return json.JSONEncoder.default(self, obj) + + +class BuildPerfTestResult(unittest.TextTestResult): + """Runner class for executing the individual tests""" + # List of test cases to run + test_run_queue = [] + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestResult, self).__init__(*args, **kwargs) + + self.out_dir = out_dir + self.hostname = socket.gethostname() + self.product = os.getenv('OE_BUILDPERFTEST_PRODUCT', 'oe-core') + self.start_time = self.elapsed_time = None + self.successes = [] + + def addSuccess(self, test): + """Record results from successful tests""" + super(BuildPerfTestResult, self).addSuccess(test) + self.successes.append(test) + + def addError(self, test, err): + """Record results from crashed test""" + test.err = err + super(BuildPerfTestResult, self).addError(test, err) + + def addFailure(self, test, err): + """Record results from failed test""" + test.err = err + super(BuildPerfTestResult, self).addFailure(test, err) + + def addExpectedFailure(self, test, err): + """Record results from expectedly failed test""" + test.err = err + super(BuildPerfTestResult, self).addExpectedFailure(test, err) + + def startTest(self, test): + """Pre-test hook""" + test.base_dir = self.out_dir + log.info("Executing test %s: %s", test.name, test.shortDescription()) + self.stream.write(datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ")) + super(BuildPerfTestResult, self).startTest(test) + + def startTestRun(self): + """Pre-run hook""" + self.start_time = datetime.utcnow() + + def stopTestRun(self): + """Pre-run hook""" + self.elapsed_time = datetime.utcnow() - self.start_time + + def all_results(self): + compound = [('SUCCESS', t, None) for t in self.successes] + \ + [('FAILURE', t, m) for t, m in self.failures] + \ + [('ERROR', t, m) for t, m in self.errors] + \ + [('EXPECTED_FAILURE', t, m) for t, m in self.expectedFailures] + \ + [('UNEXPECTED_SUCCESS', t, None) for t in self.unexpectedSuccesses] + \ + [('SKIPPED', t, m) for t, m in self.skipped] + return sorted(compound, key=lambda info: info[1].start_time) + + + def write_buildstats_json(self): + """Write buildstats file""" + buildstats = OrderedDict() + for _, test, _ in self.all_results(): + for key, val in test.buildstats.items(): + buildstats[test.name + '.' + key] = val + with open(os.path.join(self.out_dir, 'buildstats.json'), 'w') as fobj: + json.dump(buildstats, fobj, cls=ResultsJsonEncoder) + + + def write_results_json(self): + """Write test results into a json-formatted file""" + results = OrderedDict([('tester_host', self.hostname), + ('start_time', self.start_time), + ('elapsed_time', self.elapsed_time), + ('tests', OrderedDict())]) + + for status, test, reason in self.all_results(): + test_result = OrderedDict([('name', test.name), + ('description', test.shortDescription()), + ('status', status), + ('start_time', test.start_time), + ('elapsed_time', test.elapsed_time), + ('measurements', test.measurements)]) + if status in ('ERROR', 'FAILURE', 'EXPECTED_FAILURE'): + test_result['message'] = str(test.err[1]) + test_result['err_type'] = test.err[0].__name__ + test_result['err_output'] = reason + elif reason: + test_result['message'] = reason + + results['tests'][test.name] = test_result + + with open(os.path.join(self.out_dir, 'results.json'), 'w') as fobj: + json.dump(results, fobj, indent=4, + cls=ResultsJsonEncoder) + + def write_results_xml(self): + """Write test results into a JUnit XML file""" + top = ET.Element('testsuites') + suite = ET.SubElement(top, 'testsuite') + suite.set('name', 'oeqa.buildperf') + suite.set('timestamp', self.start_time.isoformat()) + suite.set('time', str(self.elapsed_time.total_seconds())) + suite.set('hostname', self.hostname) + suite.set('failures', str(len(self.failures) + len(self.expectedFailures))) + suite.set('errors', str(len(self.errors))) + suite.set('skipped', str(len(self.skipped))) + + test_cnt = 0 + for status, test, reason in self.all_results(): + test_cnt += 1 + testcase = ET.SubElement(suite, 'testcase') + testcase.set('classname', test.__module__ + '.' + test.__class__.__name__) + testcase.set('name', test.name) + testcase.set('description', test.shortDescription()) + testcase.set('timestamp', test.start_time.isoformat()) + testcase.set('time', str(test.elapsed_time.total_seconds())) + if status in ('ERROR', 'FAILURE', 'EXP_FAILURE'): + if status in ('FAILURE', 'EXP_FAILURE'): + result = ET.SubElement(testcase, 'failure') + else: + result = ET.SubElement(testcase, 'error') + result.set('message', str(test.err[1])) + result.set('type', test.err[0].__name__) + result.text = reason + elif status == 'SKIPPED': + result = ET.SubElement(testcase, 'skipped') + result.text = reason + elif status not in ('SUCCESS', 'UNEXPECTED_SUCCESS'): + raise TypeError("BUG: invalid test status '%s'" % status) + + for data in test.measurements.values(): + measurement = ET.SubElement(testcase, data['type']) + measurement.set('name', data['name']) + measurement.set('legend', data['legend']) + vals = data['values'] + if data['type'] == BuildPerfTestCase.SYSRES: + ET.SubElement(measurement, 'time', + timestamp=vals['start_time'].isoformat()).text = \ + str(vals['elapsed_time'].total_seconds()) + attrib = dict((k, str(v)) for k, v in vals['iostat'].items()) + ET.SubElement(measurement, 'iostat', attrib=attrib) + attrib = dict((k, str(v)) for k, v in vals['rusage'].items()) + ET.SubElement(measurement, 'rusage', attrib=attrib) + elif data['type'] == BuildPerfTestCase.DISKUSAGE: + ET.SubElement(measurement, 'size').text = str(vals['size']) + else: + raise TypeError('BUG: unsupported measurement type') + + suite.set('tests', str(test_cnt)) + + # Use minidom for pretty-printing + dom_doc = minidom.parseString(ET.tostring(top, 'utf-8')) + with open(os.path.join(self.out_dir, 'results.xml'), 'w') as fobj: + dom_doc.writexml(fobj, addindent=' ', newl='\n', encoding='utf-8') + + +class BuildPerfTestCase(unittest.TestCase): + """Base class for build performance tests""" + SYSRES = 'sysres' + DISKUSAGE = 'diskusage' + build_target = None + + def __init__(self, *args, **kwargs): + super(BuildPerfTestCase, self).__init__(*args, **kwargs) + self.name = self._testMethodName + self.base_dir = None + self.start_time = None + self.elapsed_time = None + self.measurements = OrderedDict() + self.buildstats = OrderedDict() + # self.err is supposed to be a tuple from sys.exc_info() + self.err = None + self.bb_vars = get_bb_vars() + # TODO: remove 'times' and 'sizes' arrays when globalres support is + # removed + self.times = [] + self.sizes = [] + + @property + def tmp_dir(self): + return os.path.join(self.base_dir, self.name + '.tmp') + + def shortDescription(self): + return super(BuildPerfTestCase, self).shortDescription() or "" + + def setUp(self): + """Set-up fixture for each test""" + if not os.path.isdir(self.tmp_dir): + os.mkdir(self.tmp_dir) + if self.build_target: + self.run_cmd(['bitbake', self.build_target, '-c', 'fetchall']) + + def tearDown(self): + """Tear-down fixture for each test""" + if os.path.isdir(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + + def run(self, *args, **kwargs): + """Run test""" + self.start_time = datetime.now() + super(BuildPerfTestCase, self).run(*args, **kwargs) + self.elapsed_time = datetime.now() - self.start_time + + def run_cmd(self, cmd): + """Convenience method for running a command""" + cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) + log.info("Logging command: %s", cmd_str) + try: + runCmd2(cmd) + except CommandError as err: + log.error("Command failed: %s", err.retcode) + raise + + def _append_measurement(self, measurement): + """Simple helper for adding measurements results""" + if measurement['name'] in self.measurements: + raise ValueError('BUG: two measurements with the same name in {}'.format( + self.__class__.__name__)) + self.measurements[measurement['name']] = measurement + + def measure_cmd_resources(self, cmd, name, legend, save_bs=False): + """Measure system resource usage of a command""" + def _worker(data_q, cmd, **kwargs): + """Worker process for measuring resources""" + try: + start_time = datetime.now() + ret = runCmd2(cmd, **kwargs) + etime = datetime.now() - start_time + rusage_struct = resource.getrusage(resource.RUSAGE_CHILDREN) + iostat = OrderedDict() + with open('/proc/{}/io'.format(os.getpid())) as fobj: + for line in fobj.readlines(): + key, val = line.split(':') + iostat[key] = int(val) + rusage = OrderedDict() + # Skip unused fields, (i.e. 'ru_ixrss', 'ru_idrss', 'ru_isrss', + # 'ru_nswap', 'ru_msgsnd', 'ru_msgrcv' and 'ru_nsignals') + for key in ['ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', + 'ru_majflt', 'ru_inblock', 'ru_oublock', + 'ru_nvcsw', 'ru_nivcsw']: + rusage[key] = getattr(rusage_struct, key) + data_q.put({'ret': ret, + 'start_time': start_time, + 'elapsed_time': etime, + 'rusage': rusage, + 'iostat': iostat}) + except Exception as err: + data_q.put(err) + + cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) + log.info("Timing command: %s", cmd_str) + data_q = SimpleQueue() + try: + proc = Process(target=_worker, args=(data_q, cmd,)) + proc.start() + data = data_q.get() + proc.join() + if isinstance(data, Exception): + raise data + except CommandError: + log.error("Command '%s' failed", cmd_str) + raise + etime = data['elapsed_time'] + + measurement = OrderedDict([('type', self.SYSRES), + ('name', name), + ('legend', legend)]) + measurement['values'] = OrderedDict([('start_time', data['start_time']), + ('elapsed_time', etime), + ('rusage', data['rusage']), + ('iostat', data['iostat'])]) + if save_bs: + self.save_buildstats(name) + + self._append_measurement(measurement) + + # Append to 'times' array for globalres log + e_sec = etime.total_seconds() + self.times.append('{:d}:{:02d}:{:05.2f}'.format(int(e_sec / 3600), + int((e_sec % 3600) / 60), + e_sec % 60)) + + def measure_disk_usage(self, path, name, legend, apparent_size=False): + """Estimate disk usage of a file or directory""" + cmd = ['du', '-s', '--block-size', '1024'] + if apparent_size: + cmd.append('--apparent-size') + cmd.append(path) + + ret = runCmd2(cmd) + size = int(ret.output.split()[0]) + log.debug("Size of %s path is %s", path, size) + measurement = OrderedDict([('type', self.DISKUSAGE), + ('name', name), + ('legend', legend)]) + measurement['values'] = OrderedDict([('size', size)]) + self._append_measurement(measurement) + # Append to 'sizes' array for globalres log + self.sizes.append(str(size)) + + def save_buildstats(self, measurement_name): + """Save buildstats""" + def split_nevr(nevr): + """Split name and version information from recipe "nevr" string""" + n_e_v, revision = nevr.rsplit('-', 1) + match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$', + n_e_v) + if not match: + # If we're not able to parse a version starting with a number, just + # take the part after last dash + match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$', + n_e_v) + name = match.group('name') + version = match.group('version') + epoch = match.group('epoch') + return name, epoch, version, revision + + def bs_to_json(filename): + """Convert (task) buildstats file into json format""" + bs_json = OrderedDict() + iostat = OrderedDict() + rusage = OrderedDict() + with open(filename) as fobj: + for line in fobj.readlines(): + key, val = line.split(':', 1) + val = val.strip() + if key == 'Started': + start_time = datetime.utcfromtimestamp(float(val)) + bs_json['start_time'] = start_time + elif key == 'Ended': + end_time = datetime.utcfromtimestamp(float(val)) + elif key.startswith('IO '): + split = key.split() + iostat[split[1]] = int(val) + elif key.find('rusage') >= 0: + split = key.split() + ru_key = split[-1] + if ru_key in ('ru_stime', 'ru_utime'): + val = float(val) + else: + val = int(val) + rusage[ru_key] = rusage.get(ru_key, 0) + val + elif key == 'Status': + bs_json['status'] = val + bs_json['elapsed_time'] = end_time - start_time + bs_json['rusage'] = rusage + bs_json['iostat'] = iostat + return bs_json + + log.info('Saving buildstats in JSON format') + bs_dirs = sorted(os.listdir(self.bb_vars['BUILDSTATS_BASE'])) + if len(bs_dirs) > 1: + log.warning("Multiple buildstats found for test %s, only " + "archiving the last one", self.name) + bs_dir = os.path.join(self.bb_vars['BUILDSTATS_BASE'], bs_dirs[-1]) + + buildstats = [] + for fname in os.listdir(bs_dir): + recipe_dir = os.path.join(bs_dir, fname) + if not os.path.isdir(recipe_dir): + continue + name, epoch, version, revision = split_nevr(fname) + recipe_bs = OrderedDict((('name', name), + ('epoch', epoch), + ('version', version), + ('revision', revision), + ('tasks', OrderedDict()))) + for task in os.listdir(recipe_dir): + recipe_bs['tasks'][task] = bs_to_json(os.path.join(recipe_dir, + task)) + buildstats.append(recipe_bs) + + self.buildstats[measurement_name] = buildstats + + def rm_tmp(self): + """Cleanup temporary/intermediate files and directories""" + log.debug("Removing temporary and cache files") + for name in ['bitbake.lock', 'conf/sanity_info', + self.bb_vars['TMPDIR']]: + oe.path.remove(name, recurse=True) + + def rm_sstate(self): + """Remove sstate directory""" + log.debug("Removing sstate-cache") + oe.path.remove(self.bb_vars['SSTATE_DIR'], recurse=True) + + def rm_cache(self): + """Drop bitbake caches""" + oe.path.remove(self.bb_vars['PERSISTENT_DIR'], recurse=True) + + @staticmethod + def sync(): + """Sync and drop kernel caches""" + log.debug("Syncing and dropping kernel caches""") + KernelDropCaches.drop() + os.sync() + # Wait a bit for all the dirty blocks to be written onto disk + time.sleep(3) + + +class BuildPerfTestLoader(unittest.TestLoader): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + +class BuildPerfTestRunner(unittest.TextTestRunner): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestRunner, self).__init__(*args, **kwargs) + self.out_dir = out_dir + + def _makeResult(self): + return BuildPerfTestResult(self.out_dir, self.stream, self.descriptions, + self.verbosity) diff --git a/meta/lib/oeqa/buildperf/test_basic.py b/meta/lib/oeqa/buildperf/test_basic.py new file mode 100644 index 0000000000..a9e4a5b731 --- /dev/null +++ b/meta/lib/oeqa/buildperf/test_basic.py @@ -0,0 +1,125 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Basic set of build performance tests""" +import os +import shutil + +import oe.path +from oeqa.buildperf import BuildPerfTestCase +from oeqa.utils.commands import get_bb_vars + + +class Test1P1(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test1(self): + """Build core-image-sato""" + self.rm_tmp() + self.rm_sstate() + self.rm_cache() + self.sync() + self.measure_cmd_resources(['bitbake', self.build_target], 'build', + 'bitbake ' + self.build_target, save_bs=True) + self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir') + + +class Test1P2(BuildPerfTestCase): + build_target = 'virtual/kernel' + + def test12(self): + """Build virtual/kernel""" + # Build and cleans state in order to get all dependencies pre-built + self.run_cmd(['bitbake', self.build_target]) + self.run_cmd(['bitbake', self.build_target, '-c', 'cleansstate']) + + self.sync() + self.measure_cmd_resources(['bitbake', self.build_target], 'build', + 'bitbake ' + self.build_target) + + +class Test1P3(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test13(self): + """Build core-image-sato with rm_work enabled""" + postfile = os.path.join(self.tmp_dir, 'postfile.conf') + with open(postfile, 'w') as fobj: + fobj.write('INHERIT += "rm_work"\n') + + self.rm_tmp() + self.rm_sstate() + self.rm_cache() + self.sync() + cmd = ['bitbake', '-R', postfile, self.build_target] + self.measure_cmd_resources(cmd, 'build', + 'bitbake' + self.build_target, + save_bs=True) + self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir') + + +class Test2(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test2(self): + """Run core-image-sato do_rootfs with sstate""" + # Build once in order to populate sstate cache + self.run_cmd(['bitbake', self.build_target]) + + self.rm_tmp() + self.rm_cache() + self.sync() + cmd = ['bitbake', self.build_target, '-c', 'rootfs'] + self.measure_cmd_resources(cmd, 'do_rootfs', 'bitbake do_rootfs') + + +class Test3(BuildPerfTestCase): + + def test3(self): + """Bitbake parsing (bitbake -p)""" + # Drop all caches and parse + self.rm_cache() + oe.path.remove(os.path.join(self.bb_vars['TMPDIR'], 'cache'), True) + self.measure_cmd_resources(['bitbake', '-p'], 'parse_1', + 'bitbake -p (no caches)') + # Drop tmp/cache + oe.path.remove(os.path.join(self.bb_vars['TMPDIR'], 'cache'), True) + self.measure_cmd_resources(['bitbake', '-p'], 'parse_2', + 'bitbake -p (no tmp/cache)') + # Parse with fully cached data + self.measure_cmd_resources(['bitbake', '-p'], 'parse_3', + 'bitbake -p (cached)') + + +class Test4(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test4(self): + """eSDK metrics""" + self.run_cmd(['bitbake', '-c', 'do_populate_sdk_ext', + self.build_target]) + self.bb_vars = get_bb_vars(None, self.build_target) + tmp_dir = self.bb_vars['TMPDIR'] + installer = os.path.join( + self.bb_vars['SDK_DEPLOY'], + self.bb_vars['TOOLCHAINEXT_OUTPUTNAME'] + '.sh') + # Measure installer size + self.measure_disk_usage(installer, 'installer_bin', 'eSDK installer', + apparent_size=True) + # Measure deployment time and deployed size + deploy_dir = os.path.join(tmp_dir, 'esdk-deploy') + if os.path.exists(deploy_dir): + shutil.rmtree(deploy_dir) + self.sync() + self.measure_cmd_resources([installer, '-y', '-d', deploy_dir], + 'deploy', 'eSDK deploy') + self.measure_disk_usage(deploy_dir, 'deploy_dir', 'deploy dir', + apparent_size=True) diff --git a/meta/lib/oeqa/controllers/masterimage.py b/meta/lib/oeqa/controllers/masterimage.py index 4cb75539ee..07418fcda1 100644 --- a/meta/lib/oeqa/controllers/masterimage.py +++ b/meta/lib/oeqa/controllers/masterimage.py @@ -32,14 +32,14 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta super(MasterImageHardwareTarget, self).__init__(d) # target ip - addr = d.getVar("TEST_TARGET_IP", True) or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') + addr = d.getVar("TEST_TARGET_IP") or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') self.ip = addr.split(":")[0] try: self.port = addr.split(":")[1] except IndexError: self.port = None bb.note("Target IP: %s" % self.ip) - self.server_ip = d.getVar("TEST_SERVER_IP", True) + self.server_ip = d.getVar("TEST_SERVER_IP") if not self.server_ip: try: self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split("\n")[0].split()[-1] @@ -49,8 +49,8 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta # test rootfs + kernel self.image_fstype = self.get_image_fstype(d) - self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) - self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') + self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("IMAGE_LINK_NAME") + '.' + self.image_fstype) + self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') if not os.path.isfile(self.rootfs): # we could've checked that IMAGE_FSTYPES contains tar.gz but the config for running testimage might not be # the same as the config with which the image was build, ie @@ -64,16 +64,16 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta # master ssh connection self.master = None # if the user knows what they are doing, then by all means... - self.user_cmds = d.getVar("TEST_DEPLOY_CMDS", True) + self.user_cmds = d.getVar("TEST_DEPLOY_CMDS") self.deploy_cmds = None # this is the name of the command that controls the power for a board # e.g: TEST_POWERCONTROL_CMD = "/home/user/myscripts/powercontrol.py ${MACHINE} what-ever-other-args-the-script-wants" # the command should take as the last argument "off" and "on" and "cycle" (off, on) - self.powercontrol_cmd = d.getVar("TEST_POWERCONTROL_CMD", True) or None + self.powercontrol_cmd = d.getVar("TEST_POWERCONTROL_CMD") or None self.powercontrol_args = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS", False) or "" - self.serialcontrol_cmd = d.getVar("TEST_SERIALCONTROL_CMD", True) or None + self.serialcontrol_cmd = d.getVar("TEST_SERIALCONTROL_CMD") or None self.serialcontrol_args = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS", False) or "" self.origenv = os.environ @@ -82,7 +82,7 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta # ssh + keys means we need the original user env bborigenv = d.getVar("BB_ORIGENV", False) or {} for key in bborigenv: - val = bborigenv.getVar(key, True) + val = bborigenv.getVar(key) if val is not None: self.origenv[key] = str(val) @@ -159,10 +159,50 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta self.power_cycle(self.connection) -class GummibootTarget(MasterImageHardwareTarget): +class SystemdbootTarget(MasterImageHardwareTarget): def __init__(self, d): - super(GummibootTarget, self).__init__(d) + super(SystemdbootTarget, self).__init__(d) + # this the value we need to set in the LoaderEntryOneShot EFI variable + # so the system boots the 'test' bootloader label and not the default + # The first four bytes are EFI bits, and the rest is an utf-16le string + # (EFI vars values need to be utf-16) + # $ echo -en "test\0" | iconv -f ascii -t utf-16le | hexdump -C + # 00000000 74 00 65 00 73 00 74 00 00 00 |t.e.s.t...| + self.efivarvalue = r'\x07\x00\x00\x00\x74\x00\x65\x00\x73\x00\x74\x00\x00\x00' + self.deploy_cmds = [ + 'mount -L boot /boot', + 'mkdir -p /mnt/testrootfs', + 'mount -L testrootfs /mnt/testrootfs', + 'modprobe efivarfs', + 'mount -t efivarfs efivarfs /sys/firmware/efi/efivars', + 'cp ~/test-kernel /boot', + 'rm -rf /mnt/testrootfs/*', + 'tar xvf ~/test-rootfs.%s -C /mnt/testrootfs' % self.image_fstype, + 'printf "%s" > /sys/firmware/efi/efivars/LoaderEntryOneShot-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f' % self.efivarvalue + ] + + def _deploy(self): + # make sure these aren't mounted + self.master.run("umount /boot; umount /mnt/testrootfs; umount /sys/firmware/efi/efivars;") + # from now on, every deploy cmd should return 0 + # else an exception will be thrown by sshcontrol + self.master.ignore_status = False + self.master.copy_to(self.rootfs, "~/test-rootfs." + self.image_fstype) + self.master.copy_to(self.kernel, "~/test-kernel") + for cmd in self.deploy_cmds: + self.master.run(cmd) + + def _start(self, params=None): + self.power_cycle(self.master) + # there are better ways than a timeout but this should work for now + time.sleep(120) + + +class SystemdbootTarget(MasterImageHardwareTarget): + + def __init__(self, d): + super(SystemdbootTarget, self).__init__(d) # this the value we need to set in the LoaderEntryOneShot EFI variable # so the system boots the 'test' bootloader label and not the default # The first four bytes are EFI bits, and the rest is an utf-16le string diff --git a/meta/lib/oeqa/controllers/testtargetloader.py b/meta/lib/oeqa/controllers/testtargetloader.py index a1b7b1d92b..b51d04b213 100644 --- a/meta/lib/oeqa/controllers/testtargetloader.py +++ b/meta/lib/oeqa/controllers/testtargetloader.py @@ -61,8 +61,6 @@ class TestTargetLoader: obj = getattr(module, target) if obj: from oeqa.targetcontrol import BaseTarget - if (not isinstance(obj, (type, types.ClassType))): - bb.warn("Target {0} found, but not of type Class".format(target)) if( not issubclass(obj, BaseTarget)): bb.warn("Target {0} found, but subclass is not BaseTarget".format(target)) except: diff --git a/meta/lib/oeqa/core/README b/meta/lib/oeqa/core/README new file mode 100644 index 0000000000..0c859fd788 --- /dev/null +++ b/meta/lib/oeqa/core/README @@ -0,0 +1,38 @@ += OEQA Framework = + +== Introduction == + +This is the new OEQA framework the base clases of the framework +are in this module oeqa/core the subsequent components needs to +extend this classes. + +A new/unique runner was created called oe-test and is under scripts/ +oe-test, this new runner scans over oeqa module searching for test +components that supports OETestContextExecutor implemented in context +module (i.e. oeqa/core/context.py). + +For execute an example: + +$ source oe-init-build-env +$ oe-test core + +For list supported components: + +$ oe-test -h + +== Create new Test component == + +Usally for add a new Test component the developer needs to extend +OETestContext/OETestContextExecutor in context.py and OETestCase in +case.py. + +== How to run the testing of the OEQA framework == + +Run all tests: + +$ PATH=$PATH:../../ python3 -m unittest discover -s tests + +Run some test: + +$ cd tests/ +$ ./test_data.py diff --git a/meta/lib/oe/tests/__init__.py b/meta/lib/oeqa/core/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/meta/lib/oe/tests/__init__.py +++ b/meta/lib/oeqa/core/__init__.py diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py new file mode 100644 index 0000000000..d2dbf20f9e --- /dev/null +++ b/meta/lib/oeqa/core/case.py @@ -0,0 +1,46 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import unittest + +from oeqa.core.exception import OEQAMissingVariable + +def _validate_td_vars(td, td_vars, type_msg): + if td_vars: + for v in td_vars: + if not v in td: + raise OEQAMissingVariable("Test %s need %s variable but"\ + " isn't into td" % (type_msg, v)) + +class OETestCase(unittest.TestCase): + # TestContext and Logger instance set by OETestLoader. + tc = None + logger = None + + # td has all the variables needed by the test cases + # is the same across all the test cases. + td = None + + # td_vars has the variables needed by a test class + # or test case instance, if some var isn't into td a + # OEMissingVariable exception is raised + td_vars = None + + @classmethod + def _oeSetUpClass(clss): + _validate_td_vars(clss.td, clss.td_vars, "class") + clss.setUpClassMethod() + + @classmethod + def _oeTearDownClass(clss): + clss.tearDownClassMethod() + + def _oeSetUp(self): + for d in self.decorators: + d.setUpDecorator() + self.setUpMethod() + + def _oeTearDown(self): + for d in self.decorators: + d.tearDownDecorator() + self.tearDownMethod() diff --git a/meta/lib/oeqa/__init__.py b/meta/lib/oeqa/core/cases/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/meta/lib/oeqa/__init__.py +++ b/meta/lib/oeqa/core/cases/__init__.py diff --git a/meta/lib/oeqa/core/cases/example/data.json b/meta/lib/oeqa/core/cases/example/data.json new file mode 100644 index 0000000000..21d6b16d17 --- /dev/null +++ b/meta/lib/oeqa/core/cases/example/data.json @@ -0,0 +1 @@ +{"ARCH": "x86", "IMAGE": "core-image-minimal"}
\ No newline at end of file diff --git a/meta/lib/oeqa/core/cases/example/test_basic.py b/meta/lib/oeqa/core/cases/example/test_basic.py new file mode 100644 index 0000000000..11cf3800cc --- /dev/null +++ b/meta/lib/oeqa/core/cases/example/test_basic.py @@ -0,0 +1,20 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.depends import OETestDepends + +class OETestExample(OETestCase): + def test_example(self): + self.logger.info('IMAGE: %s' % self.td.get('IMAGE')) + self.assertEqual('core-image-minimal', self.td.get('IMAGE')) + self.logger.info('ARCH: %s' % self.td.get('ARCH')) + self.assertEqual('x86', self.td.get('ARCH')) + +class OETestExampleDepend(OETestCase): + @OETestDepends(['OETestExample.test_example']) + def test_example_depends(self): + pass + + def test_example_no_depends(self): + pass diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py new file mode 100644 index 0000000000..4476750a3c --- /dev/null +++ b/meta/lib/oeqa/core/context.py @@ -0,0 +1,243 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import json +import time +import logging +import collections +import re + +from oeqa.core.loader import OETestLoader +from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled + +class OETestContext(object): + loaderClass = OETestLoader + runnerClass = OETestRunner + streamLoggerClass = OEStreamLogger + + files_dir = os.path.abspath(os.path.join(os.path.dirname( + os.path.abspath(__file__)), "../files")) + + def __init__(self, td=None, logger=None): + if not type(td) is dict: + raise TypeError("td isn't dictionary type") + + self.td = td + self.logger = logger + self._registry = {} + self._registry['cases'] = collections.OrderedDict() + self._results = {} + + def _read_modules_from_manifest(self, manifest): + if not os.path.exists(manifest): + raise + + modules = [] + for line in open(manifest).readlines(): + line = line.strip() + if line and not line.startswith("#"): + modules.append(line) + + return modules + + def loadTests(self, module_paths, modules=[], tests=[], + modules_manifest="", modules_required=[], filters={}): + if modules_manifest: + modules = self._read_modules_from_manifest(modules_manifest) + + self.loader = self.loaderClass(self, module_paths, modules, tests, + modules_required, filters) + self.suites = self.loader.discover() + + def runTests(self): + streamLogger = self.streamLoggerClass(self.logger) + self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2) + + self._run_start_time = time.time() + result = self.runner.run(self.suites) + self._run_end_time = time.time() + + return result + + def logSummary(self, result, component, context_msg=''): + self.logger.info("SUMMARY:") + self.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component, + context_msg, result.testsRun, result.testsRun != 1 and "s" or "", + (self._run_end_time - self._run_start_time))) + + if result.wasSuccessful(): + msg = "%s - OK - All required tests passed" % component + else: + msg = "%s - FAIL - Required tests failed" % component + skipped = len(self._results['skipped']) + if skipped: + msg += " (skipped=%d)" % skipped + self.logger.info(msg) + + def _getDetailsNotPassed(self, case, type, desc): + found = False + + for (scase, msg) in self._results[type]: + # XXX: When XML reporting is enabled scase is + # xmlrunner.result._TestInfo instance instead of + # string. + if xmlEnabled: + if case.id() == scase.test_id: + found = True + break + scase_str = scase.test_id + else: + if case == scase: + found = True + break + scase_str = str(scase) + + # When fails at module or class level the class name is passed as string + # so figure out to see if match + m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str) + if m: + if case.__class__.__module__ == m.group('module_name'): + found = True + break + + m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str) + if m: + class_name = "%s.%s" % (case.__class__.__module__, + case.__class__.__name__) + + if class_name == m.group('class_name'): + found = True + break + + if found: + return (found, msg) + + return (found, None) + + def logDetails(self): + self.logger.info("RESULTS:") + for case_name in self._registry['cases']: + case = self._registry['cases'][case_name] + + result_types = ['failures', 'errors', 'skipped', 'expectedFailures'] + result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL'] + + fail = False + desc = None + for idx, name in enumerate(result_types): + (fail, msg) = self._getDetailsNotPassed(case, result_types[idx], + result_desc[idx]) + if fail: + desc = result_desc[idx] + break + + oeid = -1 + for d in case.decorators: + if hasattr(d, 'oeid'): + oeid = d.oeid + + if fail: + self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), + oeid, desc)) + if msg: + self.logger.info(msg) + else: + self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), + oeid, 'PASSED')) + +class OETestContextExecutor(object): + _context_class = OETestContext + + name = 'core' + help = 'core test component example' + description = 'executes core test suite example' + + default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'cases/example')] + default_test_data = os.path.join(default_cases[0], 'data.json') + default_tests = None + + def register_commands(self, logger, subparsers): + self.parser = subparsers.add_parser(self.name, help=self.help, + description=self.description, group='components') + + self.default_output_log = '%s-results-%s.log' % (self.name, + time.strftime("%Y%m%d%H%M%S")) + self.parser.add_argument('--output-log', action='store', + default=self.default_output_log, + help="results output log, default: %s" % self.default_output_log) + self.parser.add_argument('--run-tests', action='store', + default=self.default_tests, + help="tests to run in <module>[.<class>[.<name>]] format. Just works for modules now") + + if self.default_test_data: + self.parser.add_argument('--test-data-file', action='store', + default=self.default_test_data, + help="data file to load, default: %s" % self.default_test_data) + else: + self.parser.add_argument('--test-data-file', action='store', + help="data file to load") + + if self.default_cases: + self.parser.add_argument('CASES_PATHS', action='store', + default=self.default_cases, nargs='*', + help="paths to directories with test cases, default: %s"\ + % self.default_cases) + else: + self.parser.add_argument('CASES_PATHS', action='store', + nargs='+', help="paths to directories with test cases") + + self.parser.set_defaults(func=self.run) + + def _setup_logger(self, logger, args): + formatter = logging.Formatter('%(asctime)s - ' + self.name + \ + ' - %(levelname)s - %(message)s') + sh = logger.handlers[0] + sh.setFormatter(formatter) + fh = logging.FileHandler(args.output_log) + fh.setFormatter(formatter) + logger.addHandler(fh) + + return logger + + def _process_args(self, logger, args): + self.tc_kwargs = {} + self.tc_kwargs['init'] = {} + self.tc_kwargs['load'] = {} + self.tc_kwargs['run'] = {} + + self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args) + if args.test_data_file: + self.tc_kwargs['init']['td'] = json.load( + open(args.test_data_file, "r")) + else: + self.tc_kwargs['init']['td'] = {} + + + if args.run_tests: + self.tc_kwargs['load']['modules'] = args.run_tests.split() + else: + self.tc_kwargs['load']['modules'] = None + + self.module_paths = args.CASES_PATHS + + def run(self, logger, args): + self._process_args(logger, args) + + self.tc = self._context_class(**self.tc_kwargs['init']) + self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) + rc = self.tc.runTests(**self.tc_kwargs['run']) + self.tc.logSummary(rc, self.name) + self.tc.logDetails() + + output_link = os.path.join(os.path.dirname(args.output_log), + "%s-results.log" % self.name) + if os.path.exists(output_link): + os.remove(output_link) + os.symlink(args.output_log, output_link) + + return rc + +_executor_class = OETestContextExecutor diff --git a/meta/lib/oeqa/core/decorator/__init__.py b/meta/lib/oeqa/core/decorator/__init__.py new file mode 100644 index 0000000000..855b6b9d28 --- /dev/null +++ b/meta/lib/oeqa/core/decorator/__init__.py @@ -0,0 +1,71 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from functools import wraps +from abc import abstractmethod + +decoratorClasses = set() + +def registerDecorator(obj): + decoratorClasses.add(obj) + return obj + +class OETestDecorator(object): + case = None # Reference of OETestCase decorated + attrs = None # Attributes to be loaded by decorator implementation + + def __init__(self, *args, **kwargs): + if not self.attrs: + return + + for idx, attr in enumerate(self.attrs): + if attr in kwargs: + value = kwargs[attr] + else: + value = args[idx] + setattr(self, attr, value) + + def __call__(self, func): + @wraps(func) + def wrapped_f(*args, **kwargs): + self.attrs = self.attrs # XXX: Enables OETestLoader discover + return func(*args, **kwargs) + return wrapped_f + + # OETestLoader call it when is loading test cases. + # XXX: Most methods would change the registry for later + # processing; be aware that filtrate method needs to + # run later than bind, so there could be data (in the + # registry) of a cases that were filtered. + def bind(self, registry, case): + self.case = case + self.logger = case.tc.logger + self.case.decorators.append(self) + + # OETestRunner call this method when tries to run + # the test case. + def setUpDecorator(self): + pass + + # OETestRunner call it after a test method has been + # called even if the method raised an exception. + def tearDownDecorator(self): + pass + +class OETestDiscover(OETestDecorator): + + # OETestLoader call it after discover test cases + # needs to return the cases to be run. + @staticmethod + def discover(registry): + return registry['cases'] + +class OETestFilter(OETestDecorator): + + # OETestLoader call it while loading the tests + # in loadTestsFromTestCase method, it needs to + # return a bool, True if needs to be filtered. + # This method must consume the filter used. + @abstractmethod + def filtrate(self, filters): + return False diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py new file mode 100644 index 0000000000..ff7bdd98b7 --- /dev/null +++ b/meta/lib/oeqa/core/decorator/data.py @@ -0,0 +1,98 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.exception import OEQAMissingVariable + +from . import OETestDecorator, registerDecorator + +def has_feature(td, feature): + """ + Checks for feature in DISTRO_FEATURES or IMAGE_FEATURES. + """ + + if (feature in td.get('DISTRO_FEATURES', '') or + feature in td.get('IMAGE_FEATURES', '')): + return True + return False + +@registerDecorator +class skipIfDataVar(OETestDecorator): + """ + Skip test based on value of a data store's variable. + + It will get the info of var from the data store and will + check it against value; if are equal it will skip the test + with msg as the reason. + """ + + attrs = ('var', 'value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %r value is %r to skip test' % + (self.var, self.value)) + self.logger.debug(msg) + if self.case.td.get(self.var) == self.value: + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfNotDataVar(OETestDecorator): + """ + Skip test based on value of a data store's variable. + + It will get the info of var from the data store and will + check it against value; if are not equal it will skip the + test with msg as the reason. + """ + + attrs = ('var', 'value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %r value is not %r to skip test' % + (self.var, self.value)) + self.logger.debug(msg) + if not self.case.td.get(self.var) == self.value: + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfNotInDataVar(OETestDecorator): + """ + Skip test if value is not in data store's variable. + """ + + attrs = ('var', 'value', 'msg') + def setUpDecorator(self): + msg = ('Checking if %r value is in %r to run ' + 'the test' % (self.var, self.value)) + self.logger.debug(msg) + if not self.value in self.case.td.get(self.var): + self.case.skipTest(self.msg) + +@registerDecorator +class OETestDataDepends(OETestDecorator): + attrs = ('td_depends',) + + def setUpDecorator(self): + for v in self.td_depends: + try: + value = self.case.td[v] + except KeyError: + raise OEQAMissingVariable("Test case need %s variable but"\ + " isn't into td" % v) + +@registerDecorator +class skipIfNotFeature(OETestDecorator): + """ + Skip test based on DISTRO_FEATURES. + + value must be in distro features or it will skip the test + with msg as the reason. + """ + + attrs = ('value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %s is in DISTRO_FEATURES ' + 'or IMAGE_FEATURES' % (self.value)) + self.logger.debug(msg) + if not has_feature(self.case.td, self.value): + self.case.skipTest(self.msg) diff --git a/meta/lib/oeqa/core/decorator/depends.py b/meta/lib/oeqa/core/decorator/depends.py new file mode 100644 index 0000000000..195711cf1e --- /dev/null +++ b/meta/lib/oeqa/core/decorator/depends.py @@ -0,0 +1,94 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from unittest import SkipTest + +from oeqa.core.exception import OEQADependency + +from . import OETestDiscover, registerDecorator + +def _add_depends(registry, case, depends): + module_name = case.__module__ + class_name = case.__class__.__name__ + + case_id = case.id() + + for depend in depends: + dparts = depend.split('.') + + if len(dparts) == 1: + depend_id = ".".join((module_name, class_name, dparts[0])) + elif len(dparts) == 2: + depend_id = ".".join((module_name, dparts[0], dparts[1])) + else: + depend_id = depend + + if not case_id in registry: + registry[case_id] = [] + if not depend_id in registry[case_id]: + registry[case_id].append(depend_id) + +def _validate_test_case_depends(cases, depends): + for case in depends: + if not case in cases: + continue + for dep in depends[case]: + if not dep in cases: + raise OEQADependency("TestCase %s depends on %s and isn't available"\ + ", cases available %s." % (case, dep, str(cases.keys()))) + +def _order_test_case_by_depends(cases, depends): + def _dep_resolve(graph, node, resolved, seen): + seen.append(node) + for edge in graph[node]: + if edge not in resolved: + if edge in seen: + raise OEQADependency("Test cases %s and %s have a circular" \ + " dependency." % (node, edge)) + _dep_resolve(graph, edge, resolved, seen) + resolved.append(node) + + dep_graph = {} + dep_graph['__root__'] = cases.keys() + for case in cases: + if case in depends: + dep_graph[case] = depends[case] + else: + dep_graph[case] = [] + + cases_ordered = [] + _dep_resolve(dep_graph, '__root__', cases_ordered, []) + cases_ordered.remove('__root__') + + return [cases[case_id] for case_id in cases_ordered] + +def _skipTestDependency(case, depends): + results = case.tc._results + skipReasons = ['errors', 'failures', 'skipped'] + + for reason in skipReasons: + for test, _ in results[reason]: + if test.id() in depends: + raise SkipTest("Test case %s depends on %s and was in %s." \ + % (case.id(), test.id(), reason)) + +@registerDecorator +class OETestDepends(OETestDiscover): + attrs = ('depends',) + + def bind(self, registry, case): + super(OETestDepends, self).bind(registry, case) + if not registry.get('depends'): + registry['depends'] = {} + _add_depends(registry['depends'], case, self.depends) + + @staticmethod + def discover(registry): + if registry.get('depends'): + _validate_test_case_depends(registry['cases'], registry['depends']) + return _order_test_case_by_depends(registry['cases'], registry['depends']) + else: + return [registry['cases'][case_id] for case_id in registry['cases']] + + def setUpDecorator(self): + _skipTestDependency(self.case, self.depends) diff --git a/meta/lib/oeqa/core/decorator/oeid.py b/meta/lib/oeqa/core/decorator/oeid.py new file mode 100644 index 0000000000..ea8017a55a --- /dev/null +++ b/meta/lib/oeqa/core/decorator/oeid.py @@ -0,0 +1,23 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from . import OETestFilter, registerDecorator +from oeqa.core.utils.misc import intToList + +def _idFilter(oeid, filters): + return False if oeid in filters else True + +@registerDecorator +class OETestID(OETestFilter): + attrs = ('oeid',) + + def bind(self, registry, case): + super(OETestID, self).bind(registry, case) + + def filtrate(self, filters): + if filters.get('oeid'): + filterx = intToList(filters['oeid'], 'oeid') + del filters['oeid'] + if _idFilter(self.oeid, filterx): + return True + return False diff --git a/meta/lib/oeqa/core/decorator/oetag.py b/meta/lib/oeqa/core/decorator/oetag.py new file mode 100644 index 0000000000..ad38ab78a5 --- /dev/null +++ b/meta/lib/oeqa/core/decorator/oetag.py @@ -0,0 +1,24 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from . import OETestFilter, registerDecorator +from oeqa.core.utils.misc import strToList + +def _tagFilter(tags, filters): + return False if set(tags) & set(filters) else True + +@registerDecorator +class OETestTag(OETestFilter): + attrs = ('oetag',) + + def bind(self, registry, case): + super(OETestTag, self).bind(registry, case) + self.oetag = strToList(self.oetag, 'oetag') + + def filtrate(self, filters): + if filters.get('oetag'): + filterx = strToList(filters['oetag'], 'oetag') + del filters['oetag'] + if _tagFilter(self.oetag, filterx): + return True + return False diff --git a/meta/lib/oeqa/core/decorator/oetimeout.py b/meta/lib/oeqa/core/decorator/oetimeout.py new file mode 100644 index 0000000000..a247583f7f --- /dev/null +++ b/meta/lib/oeqa/core/decorator/oetimeout.py @@ -0,0 +1,25 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import signal +from . import OETestDecorator, registerDecorator +from oeqa.core.exception import OEQATimeoutError + +@registerDecorator +class OETimeout(OETestDecorator): + attrs = ('oetimeout',) + + def setUpDecorator(self): + timeout = self.oetimeout + def _timeoutHandler(signum, frame): + raise OEQATimeoutError("Timed out after %s " + "seconds of execution" % timeout) + + self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) + self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) + signal.alarm(self.oetimeout) + + def tearDownDecorator(self): + signal.alarm(0) + signal.signal(signal.SIGALRM, self.alarmSignal) + self.logger.debug("Removed SIGALRM handler") diff --git a/meta/lib/oeqa/core/exception.py b/meta/lib/oeqa/core/exception.py new file mode 100644 index 0000000000..2dfd8402cf --- /dev/null +++ b/meta/lib/oeqa/core/exception.py @@ -0,0 +1,14 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +class OEQAException(Exception): + pass + +class OEQATimeoutError(OEQAException): + pass + +class OEQAMissingVariable(OEQAException): + pass + +class OEQADependency(OEQAException): + pass diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py new file mode 100644 index 0000000000..63a1703536 --- /dev/null +++ b/meta/lib/oeqa/core/loader.py @@ -0,0 +1,272 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import unittest + +from oeqa.core.utils.path import findFile +from oeqa.core.utils.test import getSuiteModules, getCaseID + +from oeqa.core.case import OETestCase +from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ + OETestFilter, OETestDiscover + +def _make_failed_test(classname, methodname, exception, suiteClass): + """ + When loading tests unittest framework stores the exception in a new + class created for be displayed into run(). + + For our purposes will be better to raise the exception in loading + step instead of wait to run the test suite. + """ + raise exception +unittest.loader._make_failed_test = _make_failed_test + +def _find_duplicated_modules(suite, directory): + for module in getSuiteModules(suite): + path = findFile('%s.py' % module, directory) + if path: + raise ImportError("Duplicated %s module found in %s" % (module, path)) + +class OETestLoader(unittest.TestLoader): + caseClass = OETestCase + + kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass', + '_top_level_dir'] + + def __init__(self, tc, module_paths, modules, tests, modules_required, + filters, *args, **kwargs): + self.tc = tc + + self.modules = modules + self.tests = tests + self.modules_required = modules_required + + self.filters = filters + self.decorator_filters = [d for d in decoratorClasses if \ + issubclass(d, OETestFilter)] + self._validateFilters(self.filters, self.decorator_filters) + self.used_filters = [d for d in self.decorator_filters + for f in self.filters + if f in d.attrs] + + if isinstance(module_paths, str): + module_paths = [module_paths] + elif not isinstance(module_paths, list): + raise TypeError('module_paths must be a str or a list of str') + self.module_paths = module_paths + + for kwname in self.kwargs_names: + if kwname in kwargs: + setattr(self, kwname, kwargs[kwname]) + + self._patchCaseClass(self.caseClass) + + def _patchCaseClass(self, testCaseClass): + # Adds custom attributes to the OETestCase class + setattr(testCaseClass, 'tc', self.tc) + setattr(testCaseClass, 'td', self.tc.td) + setattr(testCaseClass, 'logger', self.tc.logger) + + def _validateFilters(self, filters, decorator_filters): + # Validate if filter isn't empty + for key,value in filters.items(): + if not value: + raise TypeError("Filter %s specified is empty" % key) + + # Validate unique attributes + attr_filters = [attr for clss in decorator_filters \ + for attr in clss.attrs] + dup_attr = [attr for attr in attr_filters + if attr_filters.count(attr) > 1] + if dup_attr: + raise TypeError('Detected duplicated attribute(s) %s in filter' + ' decorators' % ' ,'.join(dup_attr)) + + # Validate if filter is supported + for f in filters: + if f not in attr_filters: + classes = ', '.join([d.__name__ for d in decorator_filters]) + raise TypeError('Found "%s" filter but not declared in any of ' + '%s decorators' % (f, classes)) + + def _registerTestCase(self, case): + case_id = case.id() + self.tc._registry['cases'][case_id] = case + + def _handleTestCaseDecorators(self, case): + def _handle(obj): + if isinstance(obj, OETestDecorator): + if not obj.__class__ in decoratorClasses: + raise Exception("Decorator %s isn't registered" \ + " in decoratorClasses." % obj.__name__) + obj.bind(self.tc._registry, case) + + def _walk_closure(obj): + if hasattr(obj, '__closure__') and obj.__closure__: + for f in obj.__closure__: + obj = f.cell_contents + _handle(obj) + _walk_closure(obj) + method = getattr(case, case._testMethodName, None) + _walk_closure(method) + + def _filterTest(self, case): + """ + Returns True if test case must be filtered, False otherwise. + """ + if self.filters: + filters = self.filters.copy() + case_decorators = [cd for cd in case.decorators + if cd.__class__ in self.used_filters] + + # Iterate over case decorators to check if needs to be filtered. + for cd in case_decorators: + if cd.filtrate(filters): + return True + + # Case is missing one or more decorators for all the filters + # being used, so filter test case. + if filters: + return True + + return False + + def _getTestCase(self, testCaseClass, tcName): + if not hasattr(testCaseClass, '__oeqa_loader'): + # In order to support data_vars validation + # monkey patch the default setUp/tearDown{Class} to use + # the ones provided by OETestCase + setattr(testCaseClass, 'setUpClassMethod', + getattr(testCaseClass, 'setUpClass')) + setattr(testCaseClass, 'tearDownClassMethod', + getattr(testCaseClass, 'tearDownClass')) + setattr(testCaseClass, 'setUpClass', + testCaseClass._oeSetUpClass) + setattr(testCaseClass, 'tearDownClass', + testCaseClass._oeTearDownClass) + + # In order to support decorators initialization + # monkey patch the default setUp/tearDown to use + # a setUpDecorators/tearDownDecorators that methods + # will call setUp/tearDown original methods. + setattr(testCaseClass, 'setUpMethod', + getattr(testCaseClass, 'setUp')) + setattr(testCaseClass, 'tearDownMethod', + getattr(testCaseClass, 'tearDown')) + setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp) + setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown) + + setattr(testCaseClass, '__oeqa_loader', True) + + case = testCaseClass(tcName) + setattr(case, 'decorators', []) + + return case + + def loadTestsFromTestCase(self, testCaseClass): + """ + Returns a suite of all tests cases contained in testCaseClass. + """ + if issubclass(testCaseClass, unittest.suite.TestSuite): + raise TypeError("Test cases should not be derived from TestSuite." \ + " Maybe you meant to derive %s from TestCase?" \ + % testCaseClass.__name__) + if not issubclass(testCaseClass, self.caseClass): + raise TypeError("Test %s is not derived from %s" % \ + (testCaseClass.__name__, self.caseClass.__name__)) + + testCaseNames = self.getTestCaseNames(testCaseClass) + if not testCaseNames and hasattr(testCaseClass, 'runTest'): + testCaseNames = ['runTest'] + + suite = [] + for tcName in testCaseNames: + case = self._getTestCase(testCaseClass, tcName) + # Filer by case id + if not (self.tests and not 'all' in self.tests + and not getCaseID(case) in self.tests): + self._handleTestCaseDecorators(case) + + # Filter by decorators + if not self._filterTest(case): + self._registerTestCase(case) + suite.append(case) + + return self.suiteClass(suite) + + def discover(self): + big_suite = self.suiteClass() + for path in self.module_paths: + _find_duplicated_modules(big_suite, path) + suite = super(OETestLoader, self).discover(path, + pattern='*.py', top_level_dir=path) + big_suite.addTests(suite) + + cases = None + discover_classes = [clss for clss in decoratorClasses + if issubclass(clss, OETestDiscover)] + for clss in discover_classes: + cases = clss.discover(self.tc._registry) + + return self.suiteClass(cases) if cases else big_suite + + # XXX After Python 3.5, remove backward compatibility hacks for + # use_load_tests deprecation via *args and **kws. See issue 16662. + if sys.version_info >= (3,5): + def loadTestsFromModule(self, module, *args, pattern=None, **kws): + """ + Returns a suite of all tests cases contained in module. + """ + if module.__name__ in sys.builtin_module_names: + msg = 'Tried to import %s test module but is a built-in' + raise ImportError(msg % module.__name__) + + # Normal test modules are loaded if no modules were specified, + # if module is in the specified module list or if 'all' is in + # module list. + # Underscore modules are loaded only if specified in module list. + load_module = True if not module.__name__.startswith('_') \ + and (not self.modules \ + or module.__name__ in self.modules \ + or 'all' in self.modules) \ + else False + + load_underscore = True if module.__name__.startswith('_') \ + and module.__name__ in self.modules \ + else False + + if load_module or load_underscore: + return super(OETestLoader, self).loadTestsFromModule( + module, *args, pattern=pattern, **kws) + else: + return self.suiteClass() + else: + def loadTestsFromModule(self, module, use_load_tests=True): + """ + Returns a suite of all tests cases contained in module. + """ + if module.__name__ in sys.builtin_module_names: + msg = 'Tried to import %s test module but is a built-in' + raise ImportError(msg % module.__name__) + + # Normal test modules are loaded if no modules were specified, + # if module is in the specified module list or if 'all' is in + # module list. + # Underscore modules are loaded only if specified in module list. + load_module = True if not module.__name__.startswith('_') \ + and (not self.modules \ + or module.__name__ in self.modules \ + or 'all' in self.modules) \ + else False + + load_underscore = True if module.__name__.startswith('_') \ + and module.__name__ in self.modules \ + else False + + if load_module or load_underscore: + return super(OETestLoader, self).loadTestsFromModule( + module, use_load_tests) + else: + return self.suiteClass() diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py new file mode 100644 index 0000000000..44ffecb0cd --- /dev/null +++ b/meta/lib/oeqa/core/runner.py @@ -0,0 +1,76 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import time +import unittest +import logging + +xmlEnabled = False +try: + import xmlrunner + from xmlrunner.result import _XMLTestResult as _TestResult + from xmlrunner.runner import XMLTestRunner as _TestRunner + xmlEnabled = True +except ImportError: + # use the base runner instead + from unittest import TextTestResult as _TestResult + from unittest import TextTestRunner as _TestRunner + +class OEStreamLogger(object): + def __init__(self, logger): + self.logger = logger + self.buffer = "" + + def write(self, msg): + if len(msg) > 1 and msg[0] != '\n': + self.buffer += msg + else: + self.logger.log(logging.INFO, self.buffer.rstrip("\n")) + self.buffer = "" + + def flush(self): + for handler in self.logger.handlers: + handler.flush() + +class OETestResult(_TestResult): + def __init__(self, tc, *args, **kwargs): + super(OETestResult, self).__init__(*args, **kwargs) + + self.tc = tc + + self.tc._results['failures'] = self.failures + self.tc._results['errors'] = self.errors + self.tc._results['skipped'] = self.skipped + self.tc._results['expectedFailures'] = self.expectedFailures + + def startTest(self, test): + super(OETestResult, self).startTest(test) + +class OETestRunner(_TestRunner): + def __init__(self, tc, *args, **kwargs): + if xmlEnabled: + if not kwargs.get('output'): + kwargs['output'] = os.path.join(os.getcwd(), + 'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid())) + + super(OETestRunner, self).__init__(*args, **kwargs) + self.tc = tc + self.resultclass = OETestResult + + # XXX: The unittest-xml-reporting package defines _make_result method instead + # of _makeResult standard on unittest. + if xmlEnabled: + def _make_result(self): + """ + Creates a TestResult object which will be used to store + information about the executed tests. + """ + # override in subclasses if necessary. + return self.resultclass(self.tc, + self.stream, self.descriptions, self.verbosity, self.elapsed_times + ) + else: + def _makeResult(self): + return self.resultclass(self.tc, self.stream, self.descriptions, + self.verbosity) diff --git a/meta/lib/oeqa/core/target/__init__.py b/meta/lib/oeqa/core/target/__init__.py new file mode 100644 index 0000000000..d2468bc257 --- /dev/null +++ b/meta/lib/oeqa/core/target/__init__.py @@ -0,0 +1,33 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from abc import abstractmethod + +class OETarget(object): + + def __init__(self, logger, *args, **kwargs): + self.logger = logger + + @abstractmethod + def start(self): + pass + + @abstractmethod + def stop(self): + pass + + @abstractmethod + def run(self, cmd, timeout=None): + pass + + @abstractmethod + def copyTo(self, localSrc, remoteDst): + pass + + @abstractmethod + def copyFrom(self, remoteSrc, localDst): + pass + + @abstractmethod + def copyDirTo(self, localSrc, remoteDst): + pass diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py new file mode 100644 index 0000000000..2dc521c216 --- /dev/null +++ b/meta/lib/oeqa/core/target/qemu.py @@ -0,0 +1,45 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import signal +import time + +from .ssh import OESSHTarget +from oeqa.utils.qemurunner import QemuRunner + +supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic', 'elf'] + +class OEQemuTarget(OESSHTarget): + def __init__(self, logger, ip, server_ip, timeout=300, user='root', + port=None, machine='', rootfs='', kernel='', kvm=False, + dump_dir='', dump_host_cmds='', display='', bootlog='', + tmpdir='', dir_image='', boottime=60, **kwargs): + + super(OEQemuTarget, self).__init__(logger, ip, server_ip, timeout, + user, port) + + self.ip = ip + self.server_ip = server_ip + self.machine = machine + self.rootfs = rootfs + self.kernel = kernel + self.kvm = kvm + + self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, + deploy_dir_image=dir_image, display=display, + logfile=bootlog, boottime=boottime, + use_kvm=kvm, dump_dir=dump_dir, + dump_host_cmds=dump_host_cmds) + + def start(self, params=None, extra_bootparams=None): + if self.runner.start(params, extra_bootparams=extra_bootparams): + self.ip = self.runner.ip + self.server_ip = self.runner.server_ip + else: + self.stop() + raise RuntimeError("FAILED to start qemu - check the task log and the boot log") + + def stop(self): + self.runner.stop() diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py new file mode 100644 index 0000000000..b80939c0e5 --- /dev/null +++ b/meta/lib/oeqa/core/target/ssh.py @@ -0,0 +1,266 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import time +import select +import logging +import subprocess + +from . import OETarget + +class OESSHTarget(OETarget): + def __init__(self, logger, ip, server_ip, timeout=300, user='root', + port=None, **kwargs): + if not logger: + logger = logging.getLogger('target') + logger.setLevel(logging.INFO) + filePath = os.path.join(os.getcwd(), 'remoteTarget.log') + fileHandler = logging.FileHandler(filePath, 'w', 'utf-8') + formatter = logging.Formatter( + '%(asctime)s.%(msecs)03d %(levelname)s: %(message)s', + '%H:%M:%S') + fileHandler.setFormatter(formatter) + logger.addHandler(fileHandler) + + super(OESSHTarget, self).__init__(logger) + self.ip = ip + self.server_ip = server_ip + self.timeout = timeout + self.user = user + ssh_options = [ + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + '-o', 'LogLevel=ERROR' + ] + self.ssh = ['ssh', '-l', self.user ] + ssh_options + self.scp = ['scp'] + ssh_options + if port: + self.ssh = self.ssh + [ '-p', port ] + self.scp = self.scp + [ '-P', port ] + + def start(self, **kwargs): + pass + + def stop(self, **kwargs): + pass + + def _run(self, command, timeout=None, ignore_status=True): + """ + Runs command in target using SSHProcess. + """ + self.logger.debug("[Running]$ %s" % " ".join(command)) + + starttime = time.time() + status, output = SSHCall(command, self.logger, timeout) + self.logger.debug("[Command returned '%d' after %.2f seconds]" + "" % (status, time.time() - starttime)) + + if status and not ignore_status: + raise AssertionError("Command '%s' returned non-zero exit " + "status %d:\n%s" % (command, status, output)) + + return (status, output) + + def run(self, command, timeout=None): + """ + Runs command in target. + + command: Command to run on target. + timeout: <value>: Kill command after <val> seconds. + None: Kill command default value seconds. + 0: No timeout, runs until return. + """ + targetCmd = 'export PATH=/usr/sbin:/sbin:/usr/bin:/bin; %s' % command + sshCmd = self.ssh + [self.ip, targetCmd] + + if timeout: + processTimeout = timeout + elif timeout==0: + processTimeout = None + else: + processTimeout = self.timeout + + status, output = self._run(sshCmd, processTimeout, True) + self.logger.info('\nCommand: %s\nOutput: %s\n' % (command, output)) + return (status, output) + + def copyTo(self, localSrc, remoteDst): + """ + Copy file to target. + + If local file is symlink, recreate symlink in target. + """ + if os.path.islink(localSrc): + link = os.readlink(localSrc) + dstDir, dstBase = os.path.split(remoteDst) + sshCmd = 'cd %s; ln -s %s %s' % (dstDir, link, dstBase) + return self.run(sshCmd) + + else: + remotePath = '%s@%s:%s' % (self.user, self.ip, remoteDst) + scpCmd = self.scp + [localSrc, remotePath] + return self._run(scpCmd, ignore_status=False) + + def copyFrom(self, remoteSrc, localDst): + """ + Copy file from target. + """ + remotePath = '%s@%s:%s' % (self.user, self.ip, remoteSrc) + scpCmd = self.scp + [remotePath, localDst] + return self._run(scpCmd, ignore_status=False) + + def copyDirTo(self, localSrc, remoteDst): + """ + Copy recursively localSrc directory to remoteDst in target. + """ + + for root, dirs, files in os.walk(localSrc): + # Create directories in the target as needed + for d in dirs: + tmpDir = os.path.join(root, d).replace(localSrc, "") + newDir = os.path.join(remoteDst, tmpDir.lstrip("/")) + cmd = "mkdir -p %s" % newDir + self.run(cmd) + + # Copy files into the target + for f in files: + tmpFile = os.path.join(root, f).replace(localSrc, "") + dstFile = os.path.join(remoteDst, tmpFile.lstrip("/")) + srcFile = os.path.join(root, f) + self.copyTo(srcFile, dstFile) + + def deleteFiles(self, remotePath, files): + """ + Deletes files in target's remotePath. + """ + + cmd = "rm" + if not isinstance(files, list): + files = [files] + + for f in files: + cmd = "%s %s" % (cmd, os.path.join(remotePath, f)) + + self.run(cmd) + + + def deleteDir(self, remotePath): + """ + Deletes target's remotePath directory. + """ + + cmd = "rmdir %s" % remotePath + self.run(cmd) + + + def deleteDirStructure(self, localPath, remotePath): + """ + Delete recursively localPath structure directory in target's remotePath. + + This function is very usefult to delete a package that is installed in + the DUT and the host running the test has such package extracted in tmp + directory. + + Example: + pwd: /home/user/tmp + tree: . + └── work + ├── dir1 + │  └── file1 + └── dir2 + + localpath = "/home/user/tmp" and remotepath = "/home/user" + + With the above variables this function will try to delete the + directory in the DUT in this order: + /home/user/work/dir1/file1 + /home/user/work/dir1 (if dir is empty) + /home/user/work/dir2 (if dir is empty) + /home/user/work (if dir is empty) + """ + + for root, dirs, files in os.walk(localPath, topdown=False): + # Delete files first + tmpDir = os.path.join(root).replace(localPath, "") + remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) + self.deleteFiles(remoteDir, files) + + # Remove dirs if empty + for d in dirs: + tmpDir = os.path.join(root, d).replace(localPath, "") + remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) + self.deleteDir(remoteDir) + +def SSHCall(command, logger, timeout=None, **opts): + + def run(): + nonlocal output + nonlocal process + starttime = time.time() + process = subprocess.Popen(command, **options) + if timeout: + endtime = starttime + timeout + eof = False + while time.time() < endtime and not eof: + logger.debug('time: %s, endtime: %s' % (time.time(), endtime)) + try: + if select.select([process.stdout], [], [], 5)[0] != []: + data = os.read(process.stdout.fileno(), 1024) + if not data: + process.stdout.close() + eof = True + else: + data = data.decode("utf-8") + output += data + logger.debug('Partial data from SSH call: %s' % data) + endtime = time.time() + timeout + except InterruptedError: + continue + + # process hasn't returned yet + if not eof: + process.terminate() + time.sleep(5) + try: + process.kill() + except OSError: + pass + endtime = time.time() - starttime + lastline = ("\nProcess killed - no output for %d seconds. Total" + " running time: %d seconds." % (timeout, endtime)) + logger.debug('Received data from SSH call %s ' % lastline) + output += lastline + + else: + output = process.communicate()[0].decode("utf-8") + logger.debug('Data from SSH call: %s' % output.rstrip()) + + options = { + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "stdin": None, + "shell": False, + "bufsize": -1, + "preexec_fn": os.setsid, + } + options.update(opts) + output = '' + process = None + + # Unset DISPLAY which means we won't trigger SSH_ASKPASS + env = os.environ.copy() + if "DISPLAY" in env: + del env['DISPLAY'] + options['env'] = env + + try: + run() + except: + # Need to guard against a SystemExit or other exception ocurring + # whilst running and ensure we don't leave a process behind. + if process.poll() is None: + process.kill() + logger.debug('Something went wrong, killing SSH process') + raise + return (process.wait(), output.rstrip()) diff --git a/meta/lib/oeqa/core/tests/__init__.py b/meta/lib/oeqa/core/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/core/tests/__init__.py diff --git a/meta/lib/oeqa/core/tests/cases/data.py b/meta/lib/oeqa/core/tests/cases/data.py new file mode 100644 index 0000000000..88003a6adc --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/data.py @@ -0,0 +1,20 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oetag import OETestTag +from oeqa.core.decorator.data import OETestDataDepends + +class DataTest(OETestCase): + data_vars = ['IMAGE', 'ARCH'] + + @OETestDataDepends(['MACHINE',]) + @OETestTag('dataTestOk') + def testDataOk(self): + self.assertEqual(self.td.get('IMAGE'), 'core-image-minimal') + self.assertEqual(self.td.get('ARCH'), 'x86') + self.assertEqual(self.td.get('MACHINE'), 'qemuarm') + + @OETestTag('dataTestFail') + def testDataFail(self): + pass diff --git a/meta/lib/oeqa/core/tests/cases/depends.py b/meta/lib/oeqa/core/tests/cases/depends.py new file mode 100644 index 0000000000..17cdd90b15 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/depends.py @@ -0,0 +1,38 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.depends import OETestDepends + +class DependsTest(OETestCase): + + def testDependsFirst(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsFirst']) + def testDependsSecond(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsSecond']) + def testDependsThird(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsSecond']) + def testDependsFourth(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsThird', 'testDependsFourth']) + def testDependsFifth(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsCircular3']) + def testDependsCircular1(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsCircular1']) + def testDependsCircular2(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestDepends(['testDependsCircular2']) + def testDependsCircular3(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py new file mode 100644 index 0000000000..038d445931 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py @@ -0,0 +1,15 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class AnotherIDTest(OETestCase): + + def testAnotherIdGood(self): + self.assertTrue(True, msg='How is this possible?') + + def testAnotherIdOther(self): + self.assertTrue(True, msg='How is this possible?') + + def testAnotherIdNone(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/valid/another.py b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py new file mode 100644 index 0000000000..c9ffd17773 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py @@ -0,0 +1,9 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase + +class AnotherTest(OETestCase): + + def testAnother(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/oeid.py b/meta/lib/oeqa/core/tests/cases/oeid.py new file mode 100644 index 0000000000..c2d3d32f2d --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/oeid.py @@ -0,0 +1,18 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oeid import OETestID + +class IDTest(OETestCase): + + @OETestID(101) + def testIdGood(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestID(102) + def testIdOther(self): + self.assertTrue(True, msg='How is this possible?') + + def testIdNone(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/oetag.py b/meta/lib/oeqa/core/tests/cases/oetag.py new file mode 100644 index 0000000000..0cae02e75c --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/oetag.py @@ -0,0 +1,18 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oetag import OETestTag + +class TagTest(OETestCase): + + @OETestTag('goodTag') + def testTagGood(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestTag('otherTag') + def testTagOther(self): + self.assertTrue(True, msg='How is this possible?') + + def testTagNone(self): + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/timeout.py b/meta/lib/oeqa/core/tests/cases/timeout.py new file mode 100644 index 0000000000..870c3157f7 --- /dev/null +++ b/meta/lib/oeqa/core/tests/cases/timeout.py @@ -0,0 +1,18 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from time import sleep + +from oeqa.core.case import OETestCase +from oeqa.core.decorator.oetimeout import OETimeout + +class TimeoutTest(OETestCase): + + @OETimeout(1) + def testTimeoutPass(self): + self.assertTrue(True, msg='How is this possible?') + + @OETimeout(1) + def testTimeoutFail(self): + sleep(2) + self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/common.py b/meta/lib/oeqa/core/tests/common.py new file mode 100644 index 0000000000..52b18a1c3e --- /dev/null +++ b/meta/lib/oeqa/core/tests/common.py @@ -0,0 +1,35 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import sys +import os + +import unittest +import logging +import os + +logger = logging.getLogger("oeqa") +logger.setLevel(logging.INFO) +consoleHandler = logging.StreamHandler() +formatter = logging.Formatter('OEQATest: %(message)s') +consoleHandler.setFormatter(formatter) +logger.addHandler(consoleHandler) + +def setup_sys_path(): + directory = os.path.dirname(os.path.abspath(__file__)) + oeqa_lib = os.path.realpath(os.path.join(directory, '../../../')) + if not oeqa_lib in sys.path: + sys.path.insert(0, oeqa_lib) + +class TestBase(unittest.TestCase): + def setUp(self): + self.logger = logger + directory = os.path.dirname(os.path.abspath(__file__)) + self.cases_path = os.path.join(directory, 'cases') + + def _testLoader(self, d={}, modules=[], tests=[], filters={}): + from oeqa.core.context import OETestContext + tc = OETestContext(d, self.logger) + tc.loadTests(self.cases_path, modules=modules, tests=tests, + filters=filters) + return tc diff --git a/meta/lib/oeqa/core/tests/test_data.py b/meta/lib/oeqa/core/tests/test_data.py new file mode 100755 index 0000000000..320468cbe4 --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_data.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import unittest +import logging +import os + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.exception import OEQAMissingVariable +from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames + +class TestData(TestBase): + modules = ['data'] + + def test_data_fail_missing_variable(self): + expectedException = "oeqa.core.exception.OEQAMissingVariable" + + tc = self._testLoader(modules=self.modules) + self.assertEqual(False, tc.runTests().wasSuccessful()) + for test, data in tc._results['errors']: + expect = False + if expectedException in data: + expect = True + + self.assertTrue(expect) + + def test_data_fail_wrong_variable(self): + expectedError = 'AssertionError' + d = {'IMAGE' : 'core-image-sato', 'ARCH' : 'arm'} + + tc = self._testLoader(d=d, modules=self.modules) + self.assertEqual(False, tc.runTests().wasSuccessful()) + for test, data in tc._results['failures']: + expect = False + if expectedError in data: + expect = True + + self.assertTrue(expect) + + def test_data_ok(self): + d = {'IMAGE' : 'core-image-minimal', 'ARCH' : 'x86', 'MACHINE' : 'qemuarm'} + + tc = self._testLoader(d=d, modules=self.modules) + self.assertEqual(True, tc.runTests().wasSuccessful()) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py new file mode 100755 index 0000000000..f7d11e885a --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_decorators.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import signal +import unittest + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.exception import OEQADependency +from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs + +class TestFilterDecorator(TestBase): + + def _runFilterTest(self, modules, filters, expect, msg): + tc = self._testLoader(modules=modules, filters=filters) + test_loaded = set(getSuiteCasesNames(tc.suites)) + self.assertEqual(expect, test_loaded, msg=msg) + + def test_oetag(self): + # Get all cases without filtering. + filter_all = {} + test_all = {'testTagGood', 'testTagOther', 'testTagNone'} + msg_all = 'Failed to get all oetag cases without filtering.' + + # Get cases with 'goodTag'. + filter_good = {'oetag':'goodTag'} + test_good = {'testTagGood'} + msg_good = 'Failed to get just one test filtering with "goodTag" oetag.' + + # Get cases with an invalid tag. + filter_invalid = {'oetag':'invalidTag'} + test_invalid = set() + msg_invalid = 'Failed to filter all test using an invalid oetag.' + + tests = ((filter_all, test_all, msg_all), + (filter_good, test_good, msg_good), + (filter_invalid, test_invalid, msg_invalid)) + + for test in tests: + self._runFilterTest(['oetag'], test[0], test[1], test[2]) + + def test_oeid(self): + # Get all cases without filtering. + filter_all = {} + test_all = {'testIdGood', 'testIdOther', 'testIdNone'} + msg_all = 'Failed to get all oeid cases without filtering.' + + # Get cases with '101' oeid. + filter_good = {'oeid': 101} + test_good = {'testIdGood'} + msg_good = 'Failed to get just one tes filtering with "101" oeid.' + + # Get cases with an invalid id. + filter_invalid = {'oeid':999} + test_invalid = set() + msg_invalid = 'Failed to filter all test using an invalid oeid.' + + tests = ((filter_all, test_all, msg_all), + (filter_good, test_good, msg_good), + (filter_invalid, test_invalid, msg_invalid)) + + for test in tests: + self._runFilterTest(['oeid'], test[0], test[1], test[2]) + +class TestDependsDecorator(TestBase): + modules = ['depends'] + + def test_depends_order(self): + tests = ['depends.DependsTest.testDependsFirst', + 'depends.DependsTest.testDependsSecond', + 'depends.DependsTest.testDependsThird', + 'depends.DependsTest.testDependsFourth', + 'depends.DependsTest.testDependsFifth'] + tests2 = list(tests) + tests2[2], tests2[3] = tests[3], tests[2] + tc = self._testLoader(modules=self.modules, tests=tests) + test_loaded = getSuiteCasesIDs(tc.suites) + result = True if test_loaded == tests or test_loaded == tests2 else False + msg = 'Failed to order tests using OETestDepends decorator.\nTest order:'\ + ' %s.\nExpected: %s\nOr: %s' % (test_loaded, tests, tests2) + self.assertTrue(result, msg=msg) + + def test_depends_fail_missing_dependency(self): + expect = "TestCase depends.DependsTest.testDependsSecond depends on "\ + "depends.DependsTest.testDependsFirst and isn't available" + tests = ['depends.DependsTest.testDependsSecond'] + try: + # Must throw OEQADependency because missing 'testDependsFirst' + tc = self._testLoader(modules=self.modules, tests=tests) + self.fail('Expected OEQADependency exception') + except OEQADependency as e: + result = True if expect in str(e) else False + msg = 'Expected OEQADependency exception missing testDependsFirst test' + self.assertTrue(result, msg=msg) + + def test_depends_fail_circular_dependency(self): + expect = 'have a circular dependency' + tests = ['depends.DependsTest.testDependsCircular1', + 'depends.DependsTest.testDependsCircular2', + 'depends.DependsTest.testDependsCircular3'] + try: + # Must throw OEQADependency because circular dependency + tc = self._testLoader(modules=self.modules, tests=tests) + self.fail('Expected OEQADependency exception') + except OEQADependency as e: + result = True if expect in str(e) else False + msg = 'Expected OEQADependency exception having a circular dependency' + self.assertTrue(result, msg=msg) + +class TestTimeoutDecorator(TestBase): + modules = ['timeout'] + + def test_timeout(self): + tests = ['timeout.TimeoutTest.testTimeoutPass'] + msg = 'Failed to run test using OETestTimeout' + alarm_signal = signal.getsignal(signal.SIGALRM) + tc = self._testLoader(modules=self.modules, tests=tests) + self.assertTrue(tc.runTests().wasSuccessful(), msg=msg) + msg = "OETestTimeout didn't restore SIGALRM" + self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) + + def test_timeout_fail(self): + tests = ['timeout.TimeoutTest.testTimeoutFail'] + msg = "OETestTimeout test didn't timeout as expected" + alarm_signal = signal.getsignal(signal.SIGALRM) + tc = self._testLoader(modules=self.modules, tests=tests) + self.assertFalse(tc.runTests().wasSuccessful(), msg=msg) + msg = "OETestTimeout didn't restore SIGALRM" + self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_loader.py b/meta/lib/oeqa/core/tests/test_loader.py new file mode 100755 index 0000000000..b79b8bad4d --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_loader.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import unittest + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.exception import OEQADependency +from oeqa.core.utils.test import getSuiteModules, getSuiteCasesIDs + +class TestLoader(TestBase): + + def test_fail_empty_filter(self): + filters = {'oetag' : ''} + expect = 'Filter oetag specified is empty' + msg = 'Expected TypeError exception for having invalid filter' + try: + # Must throw TypeError because empty filter + tc = self._testLoader(filters=filters) + self.fail(msg) + except TypeError as e: + result = True if expect in str(e) else False + self.assertTrue(result, msg=msg) + + def test_fail_invalid_filter(self): + filters = {'invalid' : 'good'} + expect = 'filter but not declared in any of' + msg = 'Expected TypeError exception for having invalid filter' + try: + # Must throw TypeError because invalid filter + tc = self._testLoader(filters=filters) + self.fail(msg) + except TypeError as e: + result = True if expect in str(e) else False + self.assertTrue(result, msg=msg) + + def test_fail_duplicated_module(self): + cases_path = self.cases_path + invalid_path = os.path.join(cases_path, 'loader', 'invalid') + self.cases_path = [self.cases_path, invalid_path] + expect = 'Duplicated oeid module found in' + msg = 'Expected ImportError exception for having duplicated module' + try: + # Must throw ImportEror because duplicated module + tc = self._testLoader() + self.fail(msg) + except ImportError as e: + result = True if expect in str(e) else False + self.assertTrue(result, msg=msg) + finally: + self.cases_path = cases_path + + def test_filter_modules(self): + expected_modules = {'oeid', 'oetag'} + tc = self._testLoader(modules=expected_modules) + modules = getSuiteModules(tc.suites) + msg = 'Expected just %s modules' % ', '.join(expected_modules) + self.assertEqual(modules, expected_modules, msg=msg) + + def test_filter_cases(self): + modules = ['oeid', 'oetag', 'data'] + expected_cases = {'data.DataTest.testDataOk', + 'oetag.TagTest.testTagGood', + 'oeid.IDTest.testIdGood'} + tc = self._testLoader(modules=modules, tests=expected_cases) + cases = set(getSuiteCasesIDs(tc.suites)) + msg = 'Expected just %s cases' % ', '.join(expected_cases) + self.assertEqual(cases, expected_cases, msg=msg) + + def test_import_from_paths(self): + cases_path = self.cases_path + cases2_path = os.path.join(cases_path, 'loader', 'valid') + expected_modules = {'oeid', 'another'} + self.cases_path = [self.cases_path, cases2_path] + tc = self._testLoader(modules=expected_modules) + modules = getSuiteModules(tc.suites) + self.cases_path = cases_path + msg = 'Expected modules from two different paths' + self.assertEqual(modules, expected_modules, msg=msg) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_runner.py b/meta/lib/oeqa/core/tests/test_runner.py new file mode 100755 index 0000000000..a3f3861fed --- /dev/null +++ b/meta/lib/oeqa/core/tests/test_runner.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import unittest +import logging +import tempfile + +from common import setup_sys_path, TestBase +setup_sys_path() + +from oeqa.core.runner import OEStreamLogger + +class TestRunner(TestBase): + def test_stream_logger(self): + fp = tempfile.TemporaryFile(mode='w+') + + logging.basicConfig(format='%(message)s', stream=fp) + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + oeSL = OEStreamLogger(logger) + + lines = ['init', 'bigline_' * 65535, 'morebigline_' * 65535 * 4, 'end'] + for line in lines: + oeSL.write(line) + + fp.seek(0) + fp_lines = fp.readlines() + for i, fp_line in enumerate(fp_lines): + fp_line = fp_line.strip() + self.assertEqual(lines[i], fp_line) + + fp.close() + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/core/utils/__init__.py b/meta/lib/oeqa/core/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/core/utils/__init__.py diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py new file mode 100644 index 0000000000..0b223b5d08 --- /dev/null +++ b/meta/lib/oeqa/core/utils/misc.py @@ -0,0 +1,44 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +def toList(obj, obj_type, obj_name="Object"): + if isinstance(obj, obj_type): + return [obj] + elif isinstance(obj, list): + return obj + else: + raise TypeError("%s must be %s or list" % (obj_name, obj_type)) + +def toSet(obj, obj_type, obj_name="Object"): + if isinstance(obj, obj_type): + return {obj} + elif isinstance(obj, list): + return set(obj) + elif isinstance(obj, set): + return obj + else: + raise TypeError("%s must be %s or set" % (obj_name, obj_type)) + +def strToList(obj, obj_name="Object"): + return toList(obj, str, obj_name) + +def strToSet(obj, obj_name="Object"): + return toSet(obj, str, obj_name) + +def intToList(obj, obj_name="Object"): + return toList(obj, int, obj_name) + +def dataStoteToDict(d, variables): + data = {} + + for v in variables: + data[v] = d.getVar(v) + + return data + +def updateTestData(d, td, variables): + """ + Updates variables with values of data store to test data. + """ + for var in variables: + td[var] = d.getVar(var) diff --git a/meta/lib/oeqa/core/utils/path.py b/meta/lib/oeqa/core/utils/path.py new file mode 100644 index 0000000000..a21caad5cb --- /dev/null +++ b/meta/lib/oeqa/core/utils/path.py @@ -0,0 +1,19 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys + +def findFile(file_name, directory): + """ + Search for a file in directory and returns its complete path. + """ + for r, d, f in os.walk(directory): + if file_name in f: + return os.path.join(r, file_name) + return None + +def remove_safe(path): + if os.path.exists(path): + os.remove(path) + diff --git a/meta/lib/oeqa/core/utils/test.py b/meta/lib/oeqa/core/utils/test.py new file mode 100644 index 0000000000..88d5d13981 --- /dev/null +++ b/meta/lib/oeqa/core/utils/test.py @@ -0,0 +1,86 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import inspect +import unittest + +def getSuiteCases(suite): + """ + Returns individual test from a test suite. + """ + tests = [] + + if isinstance(suite, unittest.TestCase): + tests.append(suite) + elif isinstance(suite, unittest.suite.TestSuite): + for item in suite: + tests.extend(getSuiteCases(item)) + + return tests + +def getSuiteModules(suite): + """ + Returns modules in a test suite. + """ + modules = set() + for test in getSuiteCases(suite): + modules.add(getCaseModule(test)) + return modules + +def getSuiteCasesInfo(suite, func): + """ + Returns test case info from suite. Info is fetched from func. + """ + tests = [] + for test in getSuiteCases(suite): + tests.append(func(test)) + return tests + +def getSuiteCasesNames(suite): + """ + Returns test case names from suite. + """ + return getSuiteCasesInfo(suite, getCaseMethod) + +def getSuiteCasesIDs(suite): + """ + Returns test case ids from suite. + """ + return getSuiteCasesInfo(suite, getCaseID) + +def getSuiteCasesFiles(suite): + """ + Returns test case files paths from suite. + """ + return getSuiteCasesInfo(suite, getCaseFile) + +def getCaseModule(test_case): + """ + Returns test case module name. + """ + return test_case.__module__ + +def getCaseClass(test_case): + """ + Returns test case class name. + """ + return test_case.__class__.__name__ + +def getCaseID(test_case): + """ + Returns test case complete id. + """ + return test_case.id() + +def getCaseFile(test_case): + """ + Returns test case file path. + """ + return inspect.getsourcefile(test_case.__class__) + +def getCaseMethod(test_case): + """ + Returns test case method name. + """ + return getCaseID(test_case).split('.')[-1] diff --git a/meta/lib/oeqa/runtime/files/test.c b/meta/lib/oeqa/files/test.c index 2d8389c92e..2d8389c92e 100644 --- a/meta/lib/oeqa/runtime/files/test.c +++ b/meta/lib/oeqa/files/test.c diff --git a/meta/lib/oeqa/runtime/files/test.cpp b/meta/lib/oeqa/files/test.cpp index 9e1a76473d..9e1a76473d 100644 --- a/meta/lib/oeqa/runtime/files/test.cpp +++ b/meta/lib/oeqa/files/test.cpp diff --git a/meta/lib/oeqa/runtime/files/test.pl b/meta/lib/oeqa/files/test.pl index 689c8f1635..689c8f1635 100644 --- a/meta/lib/oeqa/runtime/files/test.pl +++ b/meta/lib/oeqa/files/test.pl diff --git a/meta/lib/oeqa/runtime/files/test.py b/meta/lib/oeqa/files/test.py index f389225d72..f389225d72 100644 --- a/meta/lib/oeqa/runtime/files/test.py +++ b/meta/lib/oeqa/files/test.py diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index b4cf34b720..f7171260e7 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -27,7 +27,6 @@ try: except ImportError: pass from oeqa.utils.decorators import LogResults, gettag, getResults -from oeqa.utils import avoid_paths_in_environ logger = logging.getLogger("BitBake") @@ -61,14 +60,24 @@ class oeTest(unittest.TestCase): @classmethod def hasPackage(self, pkg): - for item in oeTest.tc.pkgmanifest.split('\n'): - if re.match(pkg, item): + """ + True if the full package name exists in the manifest, False otherwise. + """ + return pkg in oeTest.tc.pkgmanifest + + @classmethod + def hasPackageMatch(self, match): + """ + True if match exists in the manifest as a regular expression substring, + False otherwise. + """ + for s in oeTest.tc.pkgmanifest: + if re.match(match, s): return True return False @classmethod def hasFeature(self,feature): - if feature in oeTest.tc.imagefeatures or \ feature in oeTest.tc.distrofeatures: return True @@ -81,6 +90,9 @@ class oeRuntimeTest(oeTest): super(oeRuntimeTest, self).__init__(methodName) def setUp(self): + # Install packages in the DUT + self.tc.install_uninstall_packages(self.id()) + # Check if test needs to run if self.tc.sigterm: self.fail("Got SIGTERM") @@ -94,6 +106,9 @@ class oeRuntimeTest(oeTest): pass def tearDown(self): + # Uninstall packages in the DUT + self.tc.install_uninstall_packages(self.id(), False) + res = getResults() # If a test fails or there is an exception dump # for QemuTarget only @@ -113,40 +128,6 @@ class oeRuntimeTest(oeTest): def tearDownLocal(self): pass - #TODO: use package_manager.py to install packages on any type of image - def install_packages(self, packagelist): - for package in packagelist: - (status, result) = self.target.run("smart install -y "+package) - if status != 0: - return status - -class oeSDKTest(oeTest): - def __init__(self, methodName='runTest'): - self.sdktestdir = oeSDKTest.tc.sdktestdir - super(oeSDKTest, self).__init__(methodName) - - @classmethod - def hasHostPackage(self, pkg): - - if re.search(pkg, oeTest.tc.hostpkgmanifest): - return True - return False - - def _run(self, cmd): - return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True).decode("utf-8") - -class oeSDKExtTest(oeSDKTest): - def _run(self, cmd): - # extensible sdk shows a warning if found bitbake in the path - # because can cause contamination, i.e. use devtool from - # poky/scripts instead of eSDK one. - env = os.environ.copy() - paths_to_avoid = ['bitbake/bin', 'poky/scripts'] - env['PATH'] = avoid_paths_in_environ(paths_to_avoid) - - return subprocess.check_output(". %s > /dev/null;"\ - " %s;" % (self.tc.sdkenv, cmd), shell=True, env=env).decode("utf-8") - def getmodule(pos=2): # stack returns a list of tuples containg frame information # First element of the list the is current frame, caller is 1 @@ -197,16 +178,16 @@ class TestContext(object): path = [os.path.dirname(os.path.abspath(__file__))] extrapath = "" else: - path = d.getVar("BBPATH", True).split(':') + path = d.getVar("BBPATH").split(':') extrapath = "lib/oeqa" self.testslist = self._get_tests_list(path, extrapath) self.testsrequired = self._get_test_suites_required() - self.filesdir = os.path.join(os.path.dirname(os.path.abspath( - oeqa.runtime.__file__)), "files") - self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() - self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() + self.filesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime/files") + self.corefilesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + self.imagefeatures = d.getVar("IMAGE_FEATURES").split() + self.distrofeatures = d.getVar("DISTRO_FEATURES").split() # get testcase list from specified file # if path is a relative path, then relative to build/conf/ @@ -250,8 +231,6 @@ class TestContext(object): if "auto" in self.testsuites: def add_auto_list(path): - if not os.path.exists(os.path.join(path, '__init__.py')): - bb.fatal('Tests directory %s exists but is missing __init__.py' % path) files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) for f in files: module = 'oeqa.' + type + '.' + f[:-3] @@ -282,6 +261,19 @@ class TestContext(object): return modules + def getModulefromID(self, test_id): + """ + Returns the test module based on a test id. + """ + + module_name = ".".join(test_id.split(".")[:3]) + modules = self.getTestModules() + for module in modules: + if module.name == module_name: + return module + + return None + def getTests(self, test): '''Return all individual tests executed when running the suite.''' # Unfortunately unittest does not have an API for this, so we have @@ -369,21 +361,20 @@ class RuntimeTestContext(TestContext): def __init__(self, d, target, exported=False): super(RuntimeTestContext, self).__init__(d, exported) - self.tagexp = d.getVar("TEST_SUITES_TAGS", True) - self.target = target - manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), - d.getVar("IMAGE_LINK_NAME", True) + ".manifest") - nomanifest = d.getVar("IMAGE_NO_MANIFEST", True) + self.pkgmanifest = {} + manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), + d.getVar("IMAGE_LINK_NAME") + ".manifest") + nomanifest = d.getVar("IMAGE_NO_MANIFEST") if nomanifest is None or nomanifest != "1": try: with open(manifest) as f: - self.pkgmanifest = f.read() + for line in f: + (pkg, arch, version) = line.strip().split() + self.pkgmanifest[pkg] = (version, arch) except IOError as e: bb.fatal("No package manifest file found. Did you build the image?\n%s" % e) - else: - self.pkgmanifest = "" def _get_test_namespace(self): return "runtime" @@ -391,19 +382,19 @@ class RuntimeTestContext(TestContext): def _get_test_suites(self): testsuites = [] - manifests = (self.d.getVar("TEST_SUITES_MANIFEST", True) or '').split() + manifests = (self.d.getVar("TEST_SUITES_MANIFEST") or '').split() if manifests: for manifest in manifests: testsuites.extend(self._read_testlist(manifest, - self.d.getVar("TOPDIR", True)).split()) + self.d.getVar("TOPDIR")).split()) else: - testsuites = self.d.getVar("TEST_SUITES", True).split() + testsuites = self.d.getVar("TEST_SUITES").split() return testsuites def _get_test_suites_required(self): - return [t for t in self.d.getVar("TEST_SUITES", True).split() if t != "auto"] + return [t for t in self.d.getVar("TEST_SUITES").split() if t != "auto"] def loadTests(self): super(RuntimeTestContext, self).loadTests() @@ -412,19 +403,29 @@ class RuntimeTestContext(TestContext): def extract_packages(self): """ - Find and extract packages that will be needed during runtime. + Find packages that will be needed during runtime. """ - needed_packages = {} - extracted_path = self.d.getVar("TEST_EXTRACTED_DIR", True) - packaged_path = self.d.getVar("TEST_PACKAGED_DIR", True) modules = self.getTestModules() - bbpaths = self.d.getVar("BBPATH", True).split(":") + bbpaths = self.d.getVar("BBPATH").split(":") + shutil.rmtree(self.d.getVar("TEST_EXTRACTED_DIR")) + shutil.rmtree(self.d.getVar("TEST_PACKAGED_DIR")) for module in modules: json_file = self._getJsonFile(module) if json_file: needed_packages = self._getNeededPackages(json_file) + self._perform_package_extraction(needed_packages) + + def _perform_package_extraction(self, needed_packages): + """ + Extract packages that will be needed during runtime. + """ + + import oe.path + + extracted_path = self.d.getVar("TEST_EXTRACTED_DIR") + packaged_path = self.d.getVar("TEST_PACKAGED_DIR") for key,value in needed_packages.items(): packages = () @@ -446,13 +447,18 @@ class RuntimeTestContext(TestContext): dst_dir = os.path.join(packaged_path) # Extract package and copy it to TEST_EXTRACTED_DIR - if extract and not os.path.exists(dst_dir): - pkg_dir = self._extract_in_tmpdir(pkg) - shutil.copytree(pkg_dir, dst_dir) + pkg_dir = self._extract_in_tmpdir(pkg) + if extract: + + # Same package used for more than one test, + # don't need to extract again. + if os.path.exists(dst_dir): + continue + oe.path.copytree(pkg_dir, dst_dir) shutil.rmtree(pkg_dir) # Copy package to TEST_PACKAGED_DIR - elif not extract: + else: self._copy_package(pkg) def _getJsonFile(self, module): @@ -460,7 +466,7 @@ class RuntimeTestContext(TestContext): Returns the path of the JSON file for a module, empty if doesn't exitst. """ - module_file = module.filename + module_file = module.path json_file = "%s.json" % module_file.rsplit(".", 1)[0] if os.path.isfile(module_file) and os.path.isfile(json_file): return json_file @@ -500,7 +506,7 @@ class RuntimeTestContext(TestContext): from oeqa.utils.package_manager import get_package_manager - pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR", True), pkg) + pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR"), pkg) pm = get_package_manager(self.d, pkg_path) extract_dir = pm.extract(pkg) shutil.rmtree(pkg_path) @@ -514,18 +520,57 @@ class RuntimeTestContext(TestContext): from oeqa.utils.package_manager import get_package_manager - pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR", True), pkg) - dst_dir = self.d.getVar("TEST_PACKAGED_DIR", True) + pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR"), pkg) + dst_dir = self.d.getVar("TEST_PACKAGED_DIR") pm = get_package_manager(self.d, pkg_path) pkg_info = pm.package_info(pkg) file_path = pkg_info[pkg]["filepath"] shutil.copy2(file_path, dst_dir) shutil.rmtree(pkg_path) + def install_uninstall_packages(self, test_id, pkg_dir, install): + """ + Check if the test requires a package and Install/Uninstall it in the DUT + """ + + test = test_id.split(".")[4] + module = self.getModulefromID(test_id) + json = self._getJsonFile(module) + if json: + needed_packages = self._getNeededPackages(json, test) + if needed_packages: + self._install_uninstall_packages(needed_packages, pkg_dir, install) + + def _install_uninstall_packages(self, needed_packages, pkg_dir, install=True): + """ + Install/Uninstall packages in the DUT without using a package manager + """ + + if isinstance(needed_packages, dict): + packages = [needed_packages] + elif isinstance(needed_packages, list): + packages = needed_packages + + for package in packages: + pkg = package["pkg"] + rm = package.get("rm", False) + extract = package.get("extract", True) + src_dir = os.path.join(pkg_dir, pkg) + + # Install package + if install and extract: + self.target.connection.copy_dir_to(src_dir, "/") + + # Uninstall package + elif not install and rm: + self.target.connection.delete_dir_structure(src_dir, "/") + class ImageTestContext(RuntimeTestContext): def __init__(self, d, target, host_dumper): super(ImageTestContext, self).__init__(d, target) + self.tagexp = d.getVar("TEST_SUITES_TAGS") + self.host_dumper = host_dumper self.sigterm = False @@ -537,68 +582,35 @@ class ImageTestContext(RuntimeTestContext): self.sigterm = True self.target.stop() -class ExportTestContext(RuntimeTestContext): - def __init__(self, d, target, exported=False): - super(ExportTestContext, self).__init__(d, target, exported) - self.sigterm = None - -class SDKTestContext(TestContext): - def __init__(self, d, sdktestdir, sdkenv, tcname, *args): - super(SDKTestContext, self).__init__(d) - - self.sdktestdir = sdktestdir - self.sdkenv = sdkenv - self.tcname = tcname - - if not hasattr(self, 'target_manifest'): - self.target_manifest = d.getVar("SDK_TARGET_MANIFEST", True) - try: - with open(self.target_manifest) as f: - self.pkgmanifest = f.read() - except IOError as e: - bb.fatal("No package manifest file found. Did you build the sdk image?\n%s" % e) - - if not hasattr(self, 'host_manifest'): - self.host_manifest = d.getVar("SDK_HOST_MANIFEST", True) - try: - with open(self.host_manifest) as f: - self.hostpkgmanifest = f.read() - except IOError as e: - bb.fatal("No host package manifest file found. Did you build the sdk image?\n%s" % e) - - def _get_test_namespace(self): - return "sdk" + def install_uninstall_packages(self, test_id, install=True): + """ + Check if the test requires a package and Install/Uninstall it in the DUT + """ - def _get_test_suites(self): - return (self.d.getVar("TEST_SUITES_SDK", True) or "auto").split() + pkg_dir = self.d.getVar("TEST_EXTRACTED_DIR") + super(ImageTestContext, self).install_uninstall_packages(test_id, pkg_dir, install) - def _get_test_suites_required(self): - return [t for t in (self.d.getVar("TEST_SUITES_SDK", True) or \ - "auto").split() if t != "auto"] - -class SDKExtTestContext(SDKTestContext): - def __init__(self, d, sdktestdir, sdkenv, tcname, *args): - self.target_manifest = d.getVar("SDK_EXT_TARGET_MANIFEST", True) - self.host_manifest = d.getVar("SDK_EXT_HOST_MANIFEST", True) - if args: - self.cm = args[0] # Compatibility mode for run SDK tests - else: - self.cm = False +class ExportTestContext(RuntimeTestContext): + def __init__(self, d, target, exported=False, parsedArgs={}): + """ + This class is used when exporting tests and when are executed outside OE environment. - super(SDKExtTestContext, self).__init__(d, sdktestdir, sdkenv, tcname) + parsedArgs can contain the following: + - tag: Filter test by tag. + """ + super(ExportTestContext, self).__init__(d, target, exported) - self.sdkextfilesdir = os.path.join(os.path.dirname(os.path.abspath( - oeqa.sdkext.__file__)), "files") + tag = parsedArgs.get("tag", None) + self.tagexp = tag if tag != None else d.getVar("TEST_SUITES_TAGS") - def _get_test_namespace(self): - if self.cm: - return "sdk" - else: - return "sdkext" + self.sigterm = None - def _get_test_suites(self): - return (self.d.getVar("TEST_SUITES_SDK_EXT", True) or "auto").split() + def install_uninstall_packages(self, test_id, install=True): + """ + Check if the test requires a package and Install/Uninstall it in the DUT + """ - def _get_test_suites_required(self): - return [t for t in (self.d.getVar("TEST_SUITES_SDK_EXT", True) or \ - "auto").split() if t != "auto"] + export_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + extracted_dir = self.d.getVar("TEST_EXPORT_EXTRACTED_DIR") + pkg_dir = os.path.join(export_dir, extracted_dir) + super(ExportTestContext, self).install_uninstall_packages(test_id, pkg_dir, install) diff --git a/meta/lib/oeqa/runexported.py b/meta/lib/oeqa/runexported.py index 52c1171a3a..9cfea0f7ab 100755 --- a/meta/lib/oeqa/runexported.py +++ b/meta/lib/oeqa/runexported.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) 2013 Intel Corporation @@ -31,8 +31,8 @@ except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa"))) from oeqa.oetest import ExportTestContext +from oeqa.utils.commands import runCmd, updateEnv from oeqa.utils.sshcontrol import SSHControl -from oeqa.utils.dump import get_host_dumper # this isn't pretty but we need a fake target object # for running the tests externally as we don't care @@ -43,8 +43,8 @@ class FakeTarget(object): self.ip = None self.server_ip = None self.datetime = time.strftime('%Y%m%d%H%M%S',time.gmtime()) - self.testdir = d.getVar("TEST_LOG_DIR", True) - self.pn = d.getVar("PN", True) + self.testdir = d.getVar("TEST_LOG_DIR") + self.pn = d.getVar("PN") def exportStart(self): self.sshlog = os.path.join(self.testdir, "ssh_target_log.%s" % self.datetime) @@ -81,6 +81,7 @@ def main(): specified in the json if that directory actually exists or it will error out.") parser.add_argument("-l", "--log-dir", dest="log_dir", help="This sets the path for TEST_LOG_DIR. If not specified \ the current dir is used. This is used for usually creating a ssh log file and a scp test file.") + parser.add_argument("-a", "--tag", dest="tag", help="Only run test with specified tag.") parser.add_argument("json", help="The json file exported by the build system", default="testdata.json", nargs='?') args = parser.parse_args() @@ -107,17 +108,41 @@ def main(): if not os.path.isdir(d["DEPLOY_DIR"]): print("WARNING: The path to DEPLOY_DIR does not exist: %s" % d["DEPLOY_DIR"]) + parsedArgs = {} + parsedArgs["tag"] = args.tag + + extract_sdk(d) + target = FakeTarget(d) for key in loaded["target"].keys(): setattr(target, key, loaded["target"][key]) target.exportStart() - tc = ExportTestContext(d, target, True) + tc = ExportTestContext(d, target, True, parsedArgs) tc.loadTests() tc.runTests() return 0 +def extract_sdk(d): + """ + Extract SDK if needed + """ + + export_dir = os.path.dirname(os.path.realpath(__file__)) + tools_dir = d.getVar("TEST_EXPORT_SDK_DIR") + tarball_name = "%s.sh" % d.getVar("TEST_EXPORT_SDK_NAME") + tarball_path = os.path.join(export_dir, tools_dir, tarball_name) + extract_path = os.path.join(export_dir, "sysroot") + if os.path.isfile(tarball_path): + print ("Found SDK tarball %s. Extracting..." % tarball_path) + result = runCmd("%s -y -d %s" % (tarball_path, extract_path)) + for f in os.listdir(extract_path): + if f.startswith("environment-setup"): + print("Setting up SDK environment...") + env_file = os.path.join(extract_path, f) + updateEnv(env_file) + if __name__ == "__main__": try: ret = main() diff --git a/meta/lib/oeqa/runtime/__init__.py b/meta/lib/oeqa/runtime/__init__.py deleted file mode 100644 index 4cf3fa76b6..0000000000 --- a/meta/lib/oeqa/runtime/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Enable other layers to have tests in the same named directory -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/runtime/_ptest.py b/meta/lib/oeqa/runtime/_ptest.py deleted file mode 100644 index 0621028b86..0000000000 --- a/meta/lib/oeqa/runtime/_ptest.py +++ /dev/null @@ -1,125 +0,0 @@ -import unittest, os, shutil -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.logparser import * -from oeqa.utils.httpserver import HTTPService -import bb -import glob -from oe.package_manager import RpmPkgsList -import subprocess - -def setUpModule(): - if not oeRuntimeTest.hasFeature("package-management"): - skipModule("Image doesn't have package management feature") - if not oeRuntimeTest.hasPackage("smart"): - skipModule("Image doesn't have smart installed") - if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: - skipModule("Rpm is not the primary package manager") - -class PtestRunnerTest(oeRuntimeTest): - - # a ptest log parser - def parse_ptest(self, logfile): - parser = Lparser(test_0_pass_regex="^PASS:(.+)", test_0_fail_regex="^FAIL:(.+)", section_0_begin_regex="^BEGIN: .*/(.+)/ptest", section_0_end_regex="^END: .*/(.+)/ptest") - parser.init() - result = Result() - - with open(logfile) as f: - for line in f: - result_tuple = parser.parse_line(line) - if not result_tuple: - continue - result_tuple = line_type, category, status, name = parser.parse_line(line) - - if line_type == 'section' and status == 'begin': - current_section = name - continue - - if line_type == 'section' and status == 'end': - current_section = None - continue - - if line_type == 'test' and status == 'pass': - result.store(current_section, name, status) - continue - - if line_type == 'test' and status == 'fail': - result.store(current_section, name, status) - continue - - result.sort_tests() - return result - - @classmethod - def setUpClass(self): - #note the existing channels that are on the board before creating new ones -# self.existingchannels = set() -# (status, result) = oeRuntimeTest.tc.target.run('smart channel --show | grep "\["', 0) -# for x in result.split("\n"): -# self.existingchannels.add(x) - self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip) - self.repo_server.start() - - @classmethod - def tearDownClass(self): - self.repo_server.stop() - #remove created channels to be able to repeat the tests on same image -# (status, result) = oeRuntimeTest.tc.target.run('smart channel --show | grep "\["', 0) -# for x in result.split("\n"): -# if x not in self.existingchannels: -# oeRuntimeTest.tc.target.run('smart channel --remove '+x[1:-1]+' -y', 0) - - def add_smart_channel(self): - image_pkgtype = self.tc.d.getVar('IMAGE_PKGTYPE', True) - deploy_url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, image_pkgtype) - pkgarchs = self.tc.d.getVar('PACKAGE_ARCHS', True).replace("-","_").split() - for arch in os.listdir('%s/%s' % (self.repo_server.root_dir, image_pkgtype)): - if arch in pkgarchs: - self.target.run('smart channel -y --add {a} type=rpm-md baseurl={u}/{a}'.format(a=arch, u=deploy_url), 0) - self.target.run('smart update', 0) - - def install_complementary(self, globs=None): - installed_pkgs_file = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), - "installed_pkgs.txt") - self.pkgs_list = RpmPkgsList(oeRuntimeTest.tc.d, oeRuntimeTest.tc.d.getVar('IMAGE_ROOTFS', True), oeRuntimeTest.tc.d.getVar('arch_var', True), oeRuntimeTest.tc.d.getVar('os_var', True)) - with open(installed_pkgs_file, "w+") as installed_pkgs: - installed_pkgs.write(self.pkgs_list.list("arch")) - - cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), - "-p", oeRuntimeTest.tc.d.getVar('PKGDATA_DIR', True), "glob", installed_pkgs_file, - globs] - try: - bb.note("Installing complementary packages ...") - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - bb.fatal("Could not compute complementary packages list. Command " - "'%s' returned %d:\n%s" % - (' '.join(cmd), e.returncode, e.output)) - - return complementary_pkgs.split() - - def setUpLocal(self): - self.ptest_log = os.path.join(oeRuntimeTest.tc.d.getVar("TEST_LOG_DIR",True), "ptest-%s.log" % oeRuntimeTest.tc.d.getVar('DATETIME', True)) - - @skipUnlessPassed('test_ssh') - def test_ptestrunner(self): - self.add_smart_channel() - (runnerstatus, result) = self.target.run('which ptest-runner', 0) - cond = oeRuntimeTest.hasPackage("ptest-runner") and oeRuntimeTest.hasFeature("ptest") and oeRuntimeTest.hasPackage("-ptest") and (runnerstatus != 0) - if cond: - self.install_packages(self.install_complementary("*-ptest")) - self.install_packages(['ptest-runner']) - - (runnerstatus, result) = self.target.run('/usr/bin/ptest-runner > /tmp/ptest.log 2>&1', 0) - #exit code is !=0 even if ptest-runner executes because some ptest tests fail. - self.assertTrue(runnerstatus != 127, msg="Cannot execute ptest-runner!") - self.target.copy_from('/tmp/ptest.log', self.ptest_log) - shutil.copyfile(self.ptest_log, "ptest.log") - - result = self.parse_ptest("ptest.log") - log_results_to_location = "./results" - if os.path.exists(log_results_to_location): - shutil.rmtree(log_results_to_location) - os.makedirs(log_results_to_location) - - result.log_as_files(log_results_to_location, test_status = ['pass','fail']) diff --git a/meta/lib/oeqa/runtime/_qemutiny.py b/meta/lib/oeqa/runtime/_qemutiny.py deleted file mode 100644 index a3c29f3572..0000000000 --- a/meta/lib/oeqa/runtime/_qemutiny.py +++ /dev/null @@ -1,9 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.qemutinyrunner import * - -class QemuTinyTest(oeRuntimeTest): - - def test_boot_tiny(self): - (status, output) = self.target.run_serial('uname -a') - self.assertTrue("yocto-tiny" in output, msg="Cannot detect poky tiny boot!")
\ No newline at end of file diff --git a/meta/lib/oeqa/runtime/buildcvs.py b/meta/lib/oeqa/runtime/buildcvs.py deleted file mode 100644 index fe6cbfbcd5..0000000000 --- a/meta/lib/oeqa/runtime/buildcvs.py +++ /dev/null @@ -1,31 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class BuildCvsTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2") - self.project.download_archive() - - @testcase(205) - @skipUnlessPassed("test_ssh") - def test_cvs(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - self.assertEqual(self.project.run_install(), 0, - msg="Running make install failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/runtime/buildiptables.py b/meta/lib/oeqa/runtime/buildiptables.py deleted file mode 100644 index 09e252df8c..0000000000 --- a/meta/lib/oeqa/runtime/buildiptables.py +++ /dev/null @@ -1,31 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class BuildIptablesTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") - self.project.download_archive() - - @testcase(206) - @skipUnlessPassed("test_ssh") - def test_iptables(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - self.assertEqual(self.project.run_install(), 0, - msg="Running make install failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/runtime/buildsudoku.py b/meta/lib/oeqa/runtime/buildsudoku.py deleted file mode 100644 index 802b060010..0000000000 --- a/meta/lib/oeqa/runtime/buildsudoku.py +++ /dev/null @@ -1,28 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class SudokuTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") - self.project.download_archive() - - @testcase(207) - @skipUnlessPassed("test_ssh") - def test_sudoku(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/runtime/case.py b/meta/lib/oeqa/runtime/case.py new file mode 100644 index 0000000000..c1485c9860 --- /dev/null +++ b/meta/lib/oeqa/runtime/case.py @@ -0,0 +1,17 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.case import OETestCase +from oeqa.utils.package_manager import install_package, uninstall_package + +class OERuntimeTestCase(OETestCase): + # target instance set by OERuntimeTestLoader. + target = None + + def _oeSetUp(self): + super(OERuntimeTestCase, self)._oeSetUp() + install_package(self) + + def _oeTearDown(self): + super(OERuntimeTestCase, self)._oeTearDown() + uninstall_package(self) diff --git a/meta/lib/oeqa/runtime/cases/_ptest.py b/meta/lib/oeqa/runtime/cases/_ptest.py new file mode 100644 index 0000000000..aaed9a5352 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/_ptest.py @@ -0,0 +1,103 @@ +import os +import shutil +import subprocess + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature +from oeqa.runtime.decorator.package import OEHasPackage + +from oeqa.runtime.cases.dnf import DnfTest +from oeqa.utils.logparser import * +from oeqa.utils.httpserver import HTTPService + +class PtestRunnerTest(DnfTest): + + @classmethod + def setUpClass(cls): + rpm_deploy = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm') + cls.repo_server = HTTPService(rpm_deploy, cls.tc.target.server_ip) + cls.repo_server.start() + + @classmethod + def tearDownClass(cls): + cls.repo_server.stop() + + # a ptest log parser + def parse_ptest(self, logfile): + parser = Lparser(test_0_pass_regex="^PASS:(.+)", + test_0_fail_regex="^FAIL:(.+)", + section_0_begin_regex="^BEGIN: .*/(.+)/ptest", + section_0_end_regex="^END: .*/(.+)/ptest") + parser.init() + result = Result() + + with open(logfile, errors='replace') as f: + for line in f: + result_tuple = parser.parse_line(line) + if not result_tuple: + continue + result_tuple = line_type, category, status, name = parser.parse_line(line) + + if line_type == 'section' and status == 'begin': + current_section = name + continue + + if line_type == 'section' and status == 'end': + current_section = None + continue + + if line_type == 'test' and status == 'pass': + result.store(current_section, name, status) + continue + + if line_type == 'test' and status == 'fail': + result.store(current_section, name, status) + continue + + result.sort_tests() + return result + + def _install_ptest_packages(self): + # Get ptest packages that can be installed in the image. + packages_dir = os.path.join(self.tc.td['DEPLOY_DIR'], 'rpm') + ptest_pkgs = [pkg[:pkg.find('-ptest')+6] + for _, _, filenames in os.walk(packages_dir) + for pkg in filenames + if 'ptest' in pkg + and pkg[:pkg.find('-ptest')] in self.tc.image_packages] + + repo_url = 'http://%s:%s' % (self.target.server_ip, + self.repo_server.port) + dnf_options = ('--repofrompath=oe-ptest-repo,%s ' + '--nogpgcheck ' + 'install -y' % repo_url) + self.dnf('%s %s ptest-runner' % (dnf_options, ' '.join(ptest_pkgs))) + + @skipIfNotFeature('package-management', + 'Test requires package-management to be in DISTRO_FEATURES') + @skipIfNotFeature('ptest', + 'Test requires package-management to be in DISTRO_FEATURES') + @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm', + 'RPM is not the primary package manager') + @OEHasPackage(['dnf']) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_ptestrunner(self): + self.ptest_log = os.path.join(self.tc.td['TEST_LOG_DIR'], + 'ptest-%s.log' % self.tc.td['DATETIME']) + self._install_ptest_packages() + + (runnerstatus, result) = self.target.run('/usr/bin/ptest-runner > /tmp/ptest.log 2>&1', 0) + #exit code is !=0 even if ptest-runner executes because some ptest tests fail. + self.assertTrue(runnerstatus != 127, msg="Cannot execute ptest-runner!") + self.target.copyFrom('/tmp/ptest.log', self.ptest_log) + shutil.copyfile(self.ptest_log, "ptest.log") + + result = self.parse_ptest("ptest.log") + log_results_to_location = "./results" + if os.path.exists(log_results_to_location): + shutil.rmtree(log_results_to_location) + os.makedirs(log_results_to_location) + + result.log_as_files(log_results_to_location, test_status = ['pass','fail']) diff --git a/meta/lib/oeqa/runtime/cases/_qemutiny.py b/meta/lib/oeqa/runtime/cases/_qemutiny.py new file mode 100644 index 0000000000..7b5b48141f --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/_qemutiny.py @@ -0,0 +1,8 @@ +from oeqa.runtime.case import OERuntimeTestCase + +class QemuTinyTest(OERuntimeTestCase): + + def test_boot_tiny(self): + status, output = self.target.run_serial('uname -a') + msg = "Cannot detect poky tiny boot!" + self.assertTrue("yocto-tiny" in output, msg) diff --git a/meta/lib/oeqa/runtime/cases/buildcpio.py b/meta/lib/oeqa/runtime/cases/buildcpio.py new file mode 100644 index 0000000000..59edc9c2c1 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/buildcpio.py @@ -0,0 +1,30 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +from oeqa.runtime.utils.targetbuildproject import TargetBuildProject + +class BuildCpioTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + uri = 'https://ftp.gnu.org/gnu/cpio' + uri = '%s/cpio-2.12.tar.bz2' % uri + cls.project = TargetBuildProject(cls.tc.target, + uri, + dl_dir = cls.tc.td['DL_DIR']) + cls.project.download_archive() + + @classmethod + def tearDownClass(cls): + cls.project.clean() + + @OETestID(205) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_cpio(self): + self.project.run_configure() + self.project.run_make() + self.project.run_install() diff --git a/meta/lib/oeqa/runtime/cases/buildgalculator.py b/meta/lib/oeqa/runtime/cases/buildgalculator.py new file mode 100644 index 0000000000..7c9d4a392b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/buildgalculator.py @@ -0,0 +1,28 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +from oeqa.runtime.utils.targetbuildproject import TargetBuildProject + +class GalculatorTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + uri = 'http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2' + cls.project = TargetBuildProject(cls.tc.target, + uri, + dl_dir = cls.tc.td['DL_DIR']) + cls.project.download_archive() + + @classmethod + def tearDownClass(cls): + cls.project.clean() + + @OETestID(1526) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_galculator(self): + self.project.run_configure() + self.project.run_make() diff --git a/meta/lib/oeqa/runtime/cases/buildiptables.py b/meta/lib/oeqa/runtime/cases/buildiptables.py new file mode 100644 index 0000000000..002b16c483 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/buildiptables.py @@ -0,0 +1,34 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +from oeqa.runtime.utils.targetbuildproject import TargetBuildProject + +class BuildIptablesTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + uri = 'http://downloads.yoctoproject.org/mirror/sources' + uri = '%s/iptables-1.4.13.tar.bz2' % uri + cls.project = TargetBuildProject(cls.tc.target, + uri, + dl_dir = cls.tc.td['DL_DIR']) + cls.project.download_archive() + + @classmethod + def tearDownClass(cls): + cls.project.clean() + + @OETestID(206) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_iptables(self): + self.project.run_configure() + self.project.run_make() + self.project.run_install() + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/runtime/cases/connman.py b/meta/lib/oeqa/runtime/cases/connman.py new file mode 100644 index 0000000000..12456b4172 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/connman.py @@ -0,0 +1,30 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class ConnmanTest(OERuntimeTestCase): + + def service_status(self, service): + if 'systemd' in self.tc.td['DISTRO_FEATURES']: + (_, output) = self.target.run('systemctl status -l %s' % service) + return output + else: + return "Unable to get status or logs for %s" % service + + @OETestID(961) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(["connman"]) + def test_connmand_help(self): + (status, output) = self.target.run('/usr/sbin/connmand --help') + msg = 'Failed to get connman help. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(221) + @OETestDepends(['connman.ConnmanTest.test_connmand_help']) + def test_connmand_running(self): + cmd = '%s | grep [c]onnmand' % self.tc.target_cmds['ps'] + (status, output) = self.target.run(cmd) + if status != 0: + self.logger.info(self.service_status("connman")) + self.fail("No connmand process running") diff --git a/meta/lib/oeqa/runtime/cases/date.py b/meta/lib/oeqa/runtime/cases/date.py new file mode 100644 index 0000000000..ece7338de7 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/date.py @@ -0,0 +1,38 @@ +import re + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class DateTest(OERuntimeTestCase): + + def setUp(self): + if self.tc.td.get('VIRTUAL-RUNTIME_init_manager') == 'systemd': + self.logger.debug('Stopping systemd-timesyncd daemon') + self.target.run('systemctl stop systemd-timesyncd') + + def tearDown(self): + if self.tc.td.get('VIRTUAL-RUNTIME_init_manager') == 'systemd': + self.logger.debug('Starting systemd-timesyncd daemon') + self.target.run('systemctl start systemd-timesyncd') + + @OETestID(211) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_date(self): + (status, output) = self.target.run('date +"%Y-%m-%d %T"') + msg = 'Failed to get initial date, output: %s' % output + self.assertEqual(status, 0, msg=msg) + oldDate = output + + sampleDate = '"2016-08-09 10:00:00"' + (status, output) = self.target.run("date -s %s" % sampleDate) + self.assertEqual(status, 0, msg='Date set failed, output: %s' % output) + + (status, output) = self.target.run("date -R") + p = re.match('Tue, 09 Aug 2016 10:00:.. \+0000', output) + msg = 'The date was not set correctly, output: %s' % output + self.assertTrue(p, msg=msg) + + (status, output) = self.target.run('date -s "%s"' % oldDate) + msg = 'Failed to reset date, output: %s' % output + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/df.py b/meta/lib/oeqa/runtime/cases/df.py new file mode 100644 index 0000000000..aecc32d7ce --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/df.py @@ -0,0 +1,13 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class DfTest(OERuntimeTestCase): + + @OETestID(234) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_df(self): + cmd = "df / | sed -n '2p' | awk '{print $4}'" + (status,output) = self.target.run(cmd) + msg = 'Not enough space on image. Current size is %s' % output + self.assertTrue(int(output)>5120, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/dnf.py b/meta/lib/oeqa/runtime/cases/dnf.py new file mode 100644 index 0000000000..2f87296b4e --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/dnf.py @@ -0,0 +1,123 @@ +import os +import re +import subprocess +from oeqa.utils.httpserver import HTTPService + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature +from oeqa.runtime.decorator.package import OEHasPackage + +class DnfTest(OERuntimeTestCase): + + def dnf(self, command, expected = 0): + command = 'dnf %s' % command + status, output = self.target.run(command, 1500) + message = os.linesep.join([command, output]) + self.assertEqual(status, expected, message) + return output + +class DnfBasicTest(DnfTest): + + @skipIfNotFeature('package-management', + 'Test requires package-management to be in IMAGE_FEATURES') + @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm', + 'RPM is not the primary package manager') + @OEHasPackage(['dnf']) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OETestID(1735) + def test_dnf_help(self): + self.dnf('--help') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1739) + def test_dnf_version(self): + self.dnf('--version') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1737) + def test_dnf_info(self): + self.dnf('info dnf') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1738) + def test_dnf_search(self): + self.dnf('search dnf') + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1736) + def test_dnf_history(self): + self.dnf('history') + +class DnfRepoTest(DnfTest): + + @classmethod + def setUpClass(cls): + cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-testimage-repo'), + cls.tc.target.server_ip) + cls.repo_server.start() + + @classmethod + def tearDownClass(cls): + cls.repo_server.stop() + + def dnf_with_repo(self, command): + pkgarchs = os.listdir(os.path.join(self.tc.td['WORKDIR'], 'oe-testimage-repo')) + deploy_url = 'http://%s:%s/' %(self.target.server_ip, self.repo_server.port) + cmdlinerepoopts = ["--repofrompath=oe-testimage-repo-%s,%s%s" %(arch, deploy_url, arch) for arch in pkgarchs] + + self.dnf(" ".join(cmdlinerepoopts) + " --nogpgcheck " + command) + + @OETestDepends(['dnf.DnfBasicTest.test_dnf_help']) + @OETestID(1744) + def test_dnf_makecache(self): + self.dnf_with_repo('makecache') + + +# Does not work when repo is specified on the command line +# @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) +# def test_dnf_repolist(self): +# self.dnf_with_repo('repolist') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) + @OETestID(1746) + def test_dnf_repoinfo(self): + self.dnf_with_repo('repoinfo') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache']) + @OETestID(1740) + def test_dnf_install(self): + self.dnf_with_repo('install -y run-postinsts-dev') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install']) + @OETestID(1741) + def test_dnf_install_dependency(self): + self.dnf_with_repo('remove -y run-postinsts') + self.dnf_with_repo('install -y run-postinsts-dev') + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install_dependency']) + @OETestID(1742) + def test_dnf_install_from_disk(self): + self.dnf_with_repo('remove -y run-postinsts-dev') + self.dnf_with_repo('install -y --downloadonly run-postinsts-dev') + status, output = self.target.run('find /var/cache/dnf -name run-postinsts-dev*rpm', 1500) + self.assertEqual(status, 0, output) + self.dnf_with_repo('install -y %s' % output) + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install_from_disk']) + @OETestID(1743) + def test_dnf_install_from_http(self): + output = subprocess.check_output('%s %s -name run-postinsts-dev*' % (bb.utils.which(os.getenv('PATH'), "find"), + os.path.join(self.tc.td['WORKDIR'], 'oe-testimage-repo')), shell=True).decode("utf-8") + rpm_path = output.split("/")[-2] + "/" + output.split("/")[-1] + url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, rpm_path) + self.dnf_with_repo('remove -y run-postinsts-dev') + self.dnf_with_repo('install -y %s' % url) + + @OETestDepends(['dnf.DnfRepoTest.test_dnf_install']) + @OETestID(1745) + def test_dnf_reinstall(self): + self.dnf_with_repo('reinstall -y run-postinsts-dev') + + diff --git a/meta/lib/oeqa/runtime/cases/gcc.py b/meta/lib/oeqa/runtime/cases/gcc.py new file mode 100644 index 0000000000..911083156f --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/gcc.py @@ -0,0 +1,73 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class GccCompileTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + dst = '/tmp/' + src = os.path.join(cls.tc.files_dir, 'test.c') + cls.tc.target.copyTo(src, dst) + + src = os.path.join(cls.tc.runtime_files_dir, 'testmakefile') + cls.tc.target.copyTo(src, dst) + + src = os.path.join(cls.tc.files_dir, 'test.cpp') + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + files = '/tmp/test.c /tmp/test.o /tmp/test /tmp/testmakefile' + cls.tc.target.run('rm %s' % files) + + @OETestID(203) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_gcc_compile(self): + status, output = self.target.run('gcc /tmp/test.c -o /tmp/test -lm') + msg = 'gcc compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(200) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_gpp_compile(self): + status, output = self.target.run('g++ /tmp/test.c -o /tmp/test -lm') + msg = 'g++ compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(1142) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_gpp2_compile(self): + status, output = self.target.run('g++ /tmp/test.cpp -o /tmp/test -lm') + msg = 'g++ compile failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('/tmp/test') + msg = 'running compiled file failed, output: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(204) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_make(self): + status, output = self.target.run('cd /tmp; make -f testmakefile') + msg = 'running make failed, output %s' % output + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/kernelmodule.py b/meta/lib/oeqa/runtime/cases/kernelmodule.py new file mode 100644 index 0000000000..11ad7b7f01 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/kernelmodule.py @@ -0,0 +1,40 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class KernelModuleTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + src = os.path.join(cls.tc.runtime_files_dir, 'hellomod.c') + dst = '/tmp/hellomod.c' + cls.tc.target.copyTo(src, dst) + + src = os.path.join(cls.tc.runtime_files_dir, 'hellomod_makefile') + dst = '/tmp/Makefile' + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + files = '/tmp/Makefile /tmp/hellomod.c' + cls.tc.target.run('rm %s' % files) + + @OETestID(1541) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['gcc.GccCompileTest.test_gcc_compile']) + def test_kernel_module(self): + cmds = [ + 'cd /usr/src/kernel && make scripts', + 'cd /tmp && make', + 'cd /tmp && insmod hellomod.ko', + 'lsmod | grep hellomod', + 'dmesg | grep Hello', + 'rmmod hellomod', 'dmesg | grep "Cleaning up hellomod"' + ] + for cmd in cmds: + status, output = self.target.run(cmd, 900) + self.assertEqual(status, 0, msg='\n'.join([cmd, output])) diff --git a/meta/lib/oeqa/runtime/cases/ldd.py b/meta/lib/oeqa/runtime/cases/ldd.py new file mode 100644 index 0000000000..c6d92fd5af --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/ldd.py @@ -0,0 +1,25 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class LddTest(OERuntimeTestCase): + + @OETestID(962) + @skipIfNotFeature('tools-sdk', + 'Test requires tools-sdk to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_ldd_exists(self): + status, output = self.target.run('which ldd') + msg = 'ldd does not exist in PATH: which ldd: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(239) + @OETestDepends(['ldd.LddTest.test_ldd_exists']) + def test_ldd_rtldlist_check(self): + cmd = ('for i in $(which ldd | xargs cat | grep "^RTLDLIST"| ' + 'cut -d\'=\' -f2|tr -d \'"\'); ' + 'do test -f $i && echo $i && break; done') + status, output = self.target.run(cmd) + msg = "ldd path not correct or RTLDLIST files don't exist." + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/logrotate.py b/meta/lib/oeqa/runtime/cases/logrotate.py new file mode 100644 index 0000000000..992fef2989 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/logrotate.py @@ -0,0 +1,42 @@ +# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=289 testcase +# Note that the image under test must have logrotate installed + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class LogrotateTest(OERuntimeTestCase): + + @classmethod + def tearDownClass(cls): + cls.tc.target.run('rm -rf $HOME/logrotate_dir') + + @OETestID(1544) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['logrotate']) + def test_1_logrotate_setup(self): + status, output = self.target.run('mkdir $HOME/logrotate_dir') + msg = 'Could not create logrotate_dir. Output: %s' % output + self.assertEqual(status, 0, msg = msg) + + cmd = ('sed -i "s#wtmp {#wtmp {\\n olddir $HOME/logrotate_dir#"' + ' /etc/logrotate.conf') + status, output = self.target.run(cmd) + msg = ('Could not write to logrotate.conf file. Status and output: ' + ' %s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + @OETestID(1542) + @OETestDepends(['logrotate.LogrotateTest.test_1_logrotate_setup']) + def test_2_logrotate(self): + status, output = self.target.run('logrotate -f /etc/logrotate.conf') + msg = ('logrotate service could not be reloaded. Status and output: ' + '%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + _, output = self.target.run('ls -la $HOME/logrotate_dir/ | wc -l') + msg = ('new logfile could not be created. List of files within log ' + 'directory: %s' % ( + self.target.run('ls -la $HOME/logrotate_dir')[1])) + self.assertTrue(int(output)>=3, msg = msg) diff --git a/meta/lib/oeqa/runtime/cases/multilib.py b/meta/lib/oeqa/runtime/cases/multilib.py new file mode 100644 index 0000000000..8c167f1008 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/multilib.py @@ -0,0 +1,41 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotInDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +class MultilibTest(OERuntimeTestCase): + + def archtest(self, binary, arch): + """ + Check that ``binary`` has the ELF class ``arch`` (e.g. ELF32/ELF64). + """ + + status, output = self.target.run('readelf -h %s' % binary) + self.assertEqual(status, 0, 'Failed to readelf %s' % binary) + + l = [l.split()[1] for l in output.split('\n') if "Class:" in l] + if l: + theclass = l[0] + else: + self.fail('Cannot parse readelf. Output:\n%s' % output) + + msg = "%s isn't %s (is %s)" % (binary, arch, theclass) + self.assertEqual(theclass, arch, msg=msg) + + @OETestID(1593) + @skipIfNotInDataVar('MULTILIBS', 'multilib:lib32', + "This isn't a multilib:lib32 image") + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_check_multilib_libc(self): + """ + Check that a multilib image has both 32-bit and 64-bit libc in. + """ + self.archtest("/lib/libc.so.6", "ELF32") + self.archtest("/lib64/libc.so.6", "ELF64") + + @OETestID(279) + @OETestDepends(['multilib.MultilibTest.test_check_multilib_libc']) + @OEHasPackage(['lib32-connman']) + def test_file_connman(self): + self.archtest("/usr/sbin/connmand", "ELF32") diff --git a/meta/lib/oeqa/runtime/cases/oe_syslog.py b/meta/lib/oeqa/runtime/cases/oe_syslog.py new file mode 100644 index 0000000000..005b6978d9 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/oe_syslog.py @@ -0,0 +1,66 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +class SyslogTest(OERuntimeTestCase): + + @OETestID(201) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(["busybox-syslog", "sysklogd"]) + def test_syslog_running(self): + cmd = '%s | grep -i [s]yslogd' % self.tc.target_cmds['ps'] + status, output = self.target.run(cmd) + msg = "No syslogd process; ps output: %s" % output + self.assertEqual(status, 0, msg=msg) + +class SyslogTestConfig(OERuntimeTestCase): + + @OETestID(1149) + @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running']) + def test_syslog_logger(self): + status, output = self.target.run('logger foobar') + msg = "Can't log into syslog. Output: %s " % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('grep foobar /var/log/messages') + if status != 0: + if self.tc.td.get("VIRTUAL-RUNTIME_init_manager") == "systemd": + status, output = self.target.run('journalctl -o cat | grep foobar') + else: + status, output = self.target.run('logread | grep foobar') + msg = ('Test log string not found in /var/log/messages or logread.' + ' Output: %s ' % output) + self.assertEqual(status, 0, msg=msg) + + @OETestID(1150) + @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running']) + def test_syslog_restart(self): + if "systemd" != self.tc.td.get("VIRTUAL-RUNTIME_init_manager", ""): + (_, _) = self.target.run('/etc/init.d/syslog restart') + else: + (_, _) = self.target.run('systemctl restart syslog.service') + + + @OETestID(202) + @OETestDepends(['oe_syslog.SyslogTestConfig.test_syslog_logger']) + @OEHasPackage(["!sysklogd", "busybox"]) + @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', + 'Not appropiate for systemd image') + def test_syslog_startup_config(self): + cmd = 'echo "LOGFILE=/var/log/test" >> /etc/syslog-startup.conf' + self.target.run(cmd) + status, output = self.target.run('/etc/init.d/syslog restart') + msg = ('Could not restart syslog service. Status and output:' + ' %s and %s' % (status,output)) + self.assertEqual(status, 0, msg) + + cmd = 'logger foobar && grep foobar /var/log/test' + status,output = self.target.run(cmd) + msg = 'Test log string not found. Output: %s ' % output + self.assertEqual(status, 0, msg=msg) + + cmd = "sed -i 's#LOGFILE=/var/log/test##' /etc/syslog-startup.conf" + self.target.run(cmd) + self.target.run('/etc/init.d/syslog restart') diff --git a/meta/lib/oeqa/runtime/cases/pam.py b/meta/lib/oeqa/runtime/cases/pam.py new file mode 100644 index 0000000000..3654cdc946 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/pam.py @@ -0,0 +1,33 @@ +# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=287 testcase +# Note that the image under test must have "pam" in DISTRO_FEATURES + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class PamBasicTest(OERuntimeTestCase): + + @OETestID(1543) + @skipIfNotFeature('pam', 'Test requires pam to be in DISTRO_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_pam(self): + status, output = self.target.run('login --help') + msg = ('login command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 1, msg = msg) + + status, output = self.target.run('passwd --help') + msg = ('passwd command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + status, output = self.target.run('su --help') + msg = ('su command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) + + status, output = self.target.run('useradd --help') + msg = ('useradd command does not work as expected. ' + 'Status and output:%s and %s' % (status, output)) + self.assertEqual(status, 0, msg = msg) diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py new file mode 100644 index 0000000000..6e929469c4 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/parselogs.py @@ -0,0 +1,359 @@ +import os + +from subprocess import check_output +from shutil import rmtree +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +#in the future these lists could be moved outside of module +errors = ["error", "cannot", "can\'t", "failed"] + +common_errors = [ + "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", + "dma timeout", + "can\'t add hid device:", + "usbhid: probe of ", + "_OSC failed (AE_ERROR)", + "_OSC failed (AE_SUPPORT)", + "AE_ALREADY_EXISTS", + "ACPI _OSC request failed (AE_SUPPORT)", + "can\'t disable ASPM", + "Failed to load module \"vesa\"", + "Failed to load module vesa", + "Failed to load module \"modesetting\"", + "Failed to load module modesetting", + "Failed to load module \"glx\"", + "Failed to load module \"fbdev\"", + "Failed to load module fbdev", + "Failed to load module glx", + "[drm] Cannot find any crtc or sizes - going 1024x768", + "_OSC failed (AE_NOT_FOUND); disabling ASPM", + "Open ACPI failed (/var/run/acpid.socket) (No such file or directory)", + "NX (Execute Disable) protection cannot be enabled: non-PAE kernel!", + "hd.: possibly failed opcode", + 'NETLINK INITIALIZATION FAILED', + 'kernel: Cannot find map file', + 'omap_hwmod: debugss: _wait_target_disable failed', + 'VGA arbiter: cannot open kernel arbiter, no multi-card support', + 'Failed to find URL:http://ipv4.connman.net/online/status.html', + 'Online check failed for', + 'netlink init failed', + 'Fast TSC calibration', + "BAR 0-9", + "Failed to load module \"ati\"", + "controller can't do DEVSLP, turning off", + "stmmac_dvr_probe: warning: cannot get CSR clock", + "error: couldn\'t mount because of unsupported optional features", + "GPT: Use GNU Parted to correct GPT errors", + "Cannot set xattr user.Librepo.DownloadInProgress", + ] + +video_related = [ + "uvesafb", +] + +x86_common = [ + '[drm:psb_do_init] *ERROR* Debug is', + 'wrong ELF class', + 'Could not enable PowerButton event', + 'probe of LNXPWRBN:00 failed with error -22', + 'pmd_set_huge: Cannot satisfy', + 'failed to setup card detect gpio', + 'amd_nb: Cannot enumerate AMD northbridges', + 'failed to retrieve link info, disabling eDP', + 'Direct firmware load for iwlwifi', +] + common_errors + +qemux86_common = [ + 'wrong ELF class', + "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", + "can't claim BAR ", + 'amd_nb: Cannot enumerate AMD northbridges', + 'uvesafb: 5000 ms task timeout, infinitely waiting', + 'tsc: HPET/PMTIMER calibration failed', +] + common_errors + +ignore_errors = { + 'default' : common_errors, + 'qemux86' : [ + 'Failed to access perfctr msr (MSR', + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', + ] + qemux86_common, + 'qemux86-64' : qemux86_common, + 'qemumips' : [ + 'Failed to load module "glx"', + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', + ] + common_errors, + 'qemumips64' : [ + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', + ] + common_errors, + 'qemuppc' : [ + 'PCI 0000:00 Cannot reserve Legacy IO [io 0x0000-0x0fff]', + 'host side 80-wire cable detection failed, limiting max speed', + 'mode "640x480" test failed', + 'Failed to load module "glx"', + 'can\'t handle BAR above 4GB', + 'Cannot reserve Legacy IO', + ] + common_errors, + 'qemuarm' : [ + 'mmci-pl18x: probe of fpga:05 failed with error -22', + 'mmci-pl18x: probe of fpga:0b failed with error -22', + 'Failed to load module "glx"', + 'OF: amba_device_add() failed (-19) for /amba/smc@10100000', + 'OF: amba_device_add() failed (-19) for /amba/mpmc@10110000', + 'OF: amba_device_add() failed (-19) for /amba/sctl@101e0000', + 'OF: amba_device_add() failed (-19) for /amba/watchdog@101e1000', + 'OF: amba_device_add() failed (-19) for /amba/sci@101f0000', + 'OF: amba_device_add() failed (-19) for /amba/ssp@101f4000', + 'OF: amba_device_add() failed (-19) for /amba/fpga/sci@a000', + 'Failed to initialize \'/amba/timer@101e3000\': -22', + 'jitterentropy: Initialization failed with host not compliant with requirements: 2', + ] + common_errors, + 'qemuarm64' : [ + 'Fatal server error:', + '(EE) Server terminated with error (1). Closing log file.', + 'dmi: Firmware registration failed.', + 'irq: type mismatch, failed to map hwirq-27 for /intc', + ] + common_errors, + 'emenlow' : [ + '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', + '(EE) Failed to load module "psb"', + '(EE) Failed to load module psb', + '(EE) Failed to load module "psbdrv"', + '(EE) Failed to load module psbdrv', + '(EE) open /dev/fb0: No such file or directory', + '(EE) AIGLX: reverting to software rendering', + ] + x86_common, + 'intel-core2-32' : [ + 'ACPI: No _BQC method, cannot determine initial brightness', + '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', + '(EE) Failed to load module "psb"', + '(EE) Failed to load module psb', + '(EE) Failed to load module "psbdrv"', + '(EE) Failed to load module psbdrv', + '(EE) open /dev/fb0: No such file or directory', + '(EE) AIGLX: reverting to software rendering', + 'dmi: Firmware registration failed.', + 'ioremap error for 0x78', + ] + x86_common, + 'intel-corei7-64' : [ + 'can\'t set Max Payload Size to 256', + 'intel_punit_ipc: can\'t request region for resource', + '[drm] parse error at position 4 in video mode \'efifb\'', + 'ACPI Error: Could not enable RealTimeClock event', + 'ACPI Warning: Could not enable fixed event - RealTimeClock', + 'hci_intel INT33E1:00: Unable to retrieve gpio', + 'hci_intel: probe of INT33E1:00 failed', + 'can\'t derive routing for PCI INT A', + 'failed to read out thermal zone', + 'Bluetooth: hci0: Setting Intel event mask failed', + 'ttyS2 - failed to request DMA', + ] + x86_common, + 'crownbay' : x86_common, + 'genericx86' : x86_common, + 'genericx86-64' : [ + 'Direct firmware load for i915', + 'Failed to load firmware i915', + 'Failed to fetch GuC', + 'Failed to initialize GuC', + 'Failed to load DMC firmware', + 'The driver is built-in, so to load the firmware you need to', + ] + x86_common, + 'edgerouter' : [ + 'Fatal server error:', + ] + common_errors, + 'jasperforest' : [ + 'Activated service \'org.bluez\' failed:', + 'Unable to find NFC netlink family', + ] + common_errors, +} + +log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] + +class ParseLogsTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + cls.errors = errors + + # When systemd is enabled we need to notice errors on + # circular dependencies in units. + if 'systemd' in cls.td.get('DISTRO_FEATURES', ''): + cls.errors.extend([ + 'Found ordering cycle on', + 'Breaking ordering cycle by deleting job', + 'deleted to break ordering cycle', + 'Ordering cycle found, skipping', + ]) + + cls.ignore_errors = ignore_errors + cls.log_locations = log_locations + cls.msg = '' + is_lsb, _ = cls.tc.target.run("which LSB_Test.sh") + if is_lsb == 0: + for machine in cls.ignore_errors: + cls.ignore_errors[machine] = cls.ignore_errors[machine] \ + + video_related + + def getMachine(self): + return self.td.get('MACHINE', '') + + def getWorkdir(self): + return self.td.get('WORKDIR', '') + + # Get some information on the CPU of the machine to display at the + # beginning of the output. This info might be useful in some cases. + def getHardwareInfo(self): + hwi = "" + cmd = ('cat /proc/cpuinfo | grep "model name" | head -n1 | ' + " awk 'BEGIN{FS=\":\"}{print $2}'") + _, cpu_name = self.target.run(cmd) + + cmd = ('cat /proc/cpuinfo | grep "cpu cores" | head -n1 | ' + "awk {'print $4'}") + _, cpu_physical_cores = self.target.run(cmd) + + cmd = 'cat /proc/cpuinfo | grep "processor" | wc -l' + _, cpu_logical_cores = self.target.run(cmd) + + _, cpu_arch = self.target.run('uname -m') + + hwi += 'Machine information: \n' + hwi += '*******************************\n' + hwi += 'Machine name: ' + self.getMachine() + '\n' + hwi += 'CPU: ' + str(cpu_name) + '\n' + hwi += 'Arch: ' + str(cpu_arch)+ '\n' + hwi += 'Physical cores: ' + str(cpu_physical_cores) + '\n' + hwi += 'Logical cores: ' + str(cpu_logical_cores) + '\n' + hwi += '*******************************\n' + + return hwi + + # Go through the log locations provided and if it's a folder + # create a list with all the .log files in it, if it's a file + # just add it to that list. + def getLogList(self, log_locations): + logs = [] + for location in log_locations: + status, _ = self.target.run('test -f ' + str(location)) + if status == 0: + logs.append(str(location)) + else: + status, _ = self.target.run('test -d ' + str(location)) + if status == 0: + cmd = 'find ' + str(location) + '/*.log -maxdepth 1 -type f' + status, output = self.target.run(cmd) + if status == 0: + output = output.splitlines() + for logfile in output: + logs.append(os.path.join(location, str(logfile))) + return logs + + # Copy the log files to be parsed locally + def transfer_logs(self, log_list): + workdir = self.getWorkdir() + self.target_logs = workdir + '/' + 'target_logs' + target_logs = self.target_logs + if os.path.exists(target_logs): + rmtree(self.target_logs) + os.makedirs(target_logs) + for f in log_list: + self.target.copyFrom(str(f), target_logs) + + # Get the local list of logs + def get_local_log_list(self, log_locations): + self.transfer_logs(self.getLogList(log_locations)) + list_dir = os.listdir(self.target_logs) + dir_files = [os.path.join(self.target_logs, f) for f in list_dir] + logs = [f for f in dir_files if os.path.isfile(f)] + return logs + + # Build the grep command to be used with filters and exclusions + def build_grepcmd(self, errors, ignore_errors, log): + grepcmd = 'grep ' + grepcmd += '-Ei "' + for error in errors: + grepcmd += error + '|' + grepcmd = grepcmd[:-1] + grepcmd += '" ' + str(log) + " | grep -Eiv \'" + + try: + errorlist = ignore_errors[self.getMachine()] + except KeyError: + self.msg += 'No ignore list found for this machine, using default\n' + errorlist = ignore_errors['default'] + + for ignore_error in errorlist: + ignore_error = ignore_error.replace('(', '\(') + ignore_error = ignore_error.replace(')', '\)') + ignore_error = ignore_error.replace("'", '.') + ignore_error = ignore_error.replace('?', '\?') + ignore_error = ignore_error.replace('[', '\[') + ignore_error = ignore_error.replace(']', '\]') + ignore_error = ignore_error.replace('*', '\*') + ignore_error = ignore_error.replace('0-9', '[0-9]') + grepcmd += ignore_error + '|' + grepcmd = grepcmd[:-1] + grepcmd += "\'" + + return grepcmd + + # Grep only the errors so that their context could be collected. + # Default context is 10 lines before and after the error itself + def parse_logs(self, errors, ignore_errors, logs, + lines_before = 10, lines_after = 10): + results = {} + rez = [] + grep_output = '' + + for log in logs: + result = None + thegrep = self.build_grepcmd(errors, ignore_errors, log) + + try: + result = check_output(thegrep, shell=True).decode('utf-8') + except: + pass + + if result is not None: + results[log.replace('target_logs/','')] = {} + rez = result.splitlines() + + for xrez in rez: + try: + cmd = ['grep', '-F', xrez, '-B', str(lines_before)] + cmd += ['-A', str(lines_after), log] + grep_output = check_output(cmd).decode('utf-8') + except: + pass + results[log.replace('target_logs/','')][xrez]=grep_output + + return results + + # Get the output of dmesg and write it in a file. + # This file is added to log_locations. + def write_dmesg(self): + (status, dmesg) = self.target.run('dmesg > /tmp/dmesg_output.log') + + @OETestID(1059) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_parselogs(self): + self.write_dmesg() + log_list = self.get_local_log_list(self.log_locations) + result = self.parse_logs(self.errors, self.ignore_errors, log_list) + print(self.getHardwareInfo()) + errcount = 0 + for log in result: + self.msg += 'Log: ' + log + '\n' + self.msg += '-----------------------\n' + for error in result[log]: + errcount += 1 + self.msg += 'Central error: ' + str(error) + '\n' + self.msg += '***********************\n' + self.msg += result[str(log)][str(error)] + '\n' + self.msg += '***********************\n' + self.msg += '%s errors found in logs.' % errcount + self.assertEqual(errcount, 0, msg=self.msg) diff --git a/meta/lib/oeqa/runtime/cases/perl.py b/meta/lib/oeqa/runtime/cases/perl.py new file mode 100644 index 0000000000..d0b7e8ed92 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/perl.py @@ -0,0 +1,37 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class PerlTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + src = os.path.join(cls.tc.files_dir, 'test.pl') + dst = '/tmp/test.pl' + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + dst = '/tmp/test.pl' + cls.tc.target.run('rm %s' % dst) + + @OETestID(1141) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['perl']) + def test_perl_exists(self): + status, output = self.target.run('which perl') + msg = 'Perl binary not in PATH or not on target.' + self.assertEqual(status, 0, msg=msg) + + @OETestID(208) + @OETestDepends(['perl.PerlTest.test_perl_exists']) + def test_perl_works(self): + status, output = self.target.run('perl /tmp/test.pl') + msg = 'Exit status was not 0. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + msg = 'Incorrect output: %s' % output + self.assertEqual(output, "the value of a is 0.01", msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ping.py b/meta/lib/oeqa/runtime/cases/ping.py new file mode 100644 index 0000000000..02f580abee --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/ping.py @@ -0,0 +1,24 @@ +from subprocess import Popen, PIPE + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.oetimeout import OETimeout + +class PingTest(OERuntimeTestCase): + + @OETimeout(30) + @OETestID(964) + def test_ping(self): + output = '' + count = 0 + while count < 5: + cmd = 'ping -c 1 %s' % self.target.ip + proc = Popen(cmd, shell=True, stdout=PIPE) + output += proc.communicate()[0].decode('utf-8') + if proc.poll() == 0: + count += 1 + else: + count = 0 + msg = ('Expected 5 consecutive, got %d.\n' + 'ping output is:\n%s' % (count,output)) + self.assertEqual(count, 5, msg = msg) diff --git a/meta/lib/oeqa/runtime/cases/python.py b/meta/lib/oeqa/runtime/cases/python.py new file mode 100644 index 0000000000..bf3e179163 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/python.py @@ -0,0 +1,43 @@ +import os + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class PythonTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + src = os.path.join(cls.tc.files_dir, 'test.py') + dst = '/tmp/test.py' + cls.tc.target.copyTo(src, dst) + + @classmethod + def tearDownClass(cls): + dst = '/tmp/test.py' + cls.tc.target.run('rm %s' % dst) + + @OETestID(1145) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['python-core']) + def test_python_exists(self): + status, output = self.target.run('which python') + msg = 'Python binary not in PATH or not on target.' + self.assertEqual(status, 0, msg=msg) + + @OETestID(965) + @OETestDepends(['python.PythonTest.test_python_exists']) + def test_python_stdout(self): + status, output = self.target.run('python /tmp/test.py') + msg = 'Exit status was not 0. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + msg = 'Incorrect output: %s' % output + self.assertEqual(output, "the value of a is 0.01", msg=msg) + + @OETestID(1146) + @OETestDepends(['python.PythonTest.test_python_stdout']) + def test_python_testfile(self): + status, output = self.target.run('ls /tmp/testfile.python') + self.assertEqual(status, 0, msg='Python test file generate failed.') diff --git a/meta/lib/oeqa/runtime/cases/rpm.py b/meta/lib/oeqa/runtime/cases/rpm.py new file mode 100644 index 0000000000..05b94c7b40 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/rpm.py @@ -0,0 +1,142 @@ +import os +import fnmatch + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage +from oeqa.core.utils.path import findFile + +class RpmBasicTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + if cls.tc.td['PACKAGE_CLASSES'].split()[0] != 'package_rpm': + cls.skipTest('Tests require image to be build from rpm') + + @OETestID(960) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_rpm_help(self): + status, output = self.target.run('rpm --help') + msg = 'status and output: %s and %s' % (status, output) + self.assertEqual(status, 0, msg=msg) + + @OETestID(191) + @OETestDepends(['rpm.RpmBasicTest.test_rpm_help']) + def test_rpm_query(self): + status, output = self.target.run('rpm -q rpm') + msg = 'status and output: %s and %s' % (status, output) + self.assertEqual(status, 0, msg=msg) + +class RpmInstallRemoveTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + if cls.tc.td['PACKAGE_CLASSES'].split()[0] != 'package_rpm': + cls.skipTest('Tests require image to be build from rpm') + + pkgarch = cls.td['TUNE_PKGARCH'].replace('-', '_') + rpmdir = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm', pkgarch) + # Pick rpm-doc as a test file to get installed, because it's small + # and it will always be built for standard targets + rpm_doc = 'rpm-doc-*.%s.rpm' % pkgarch + for f in fnmatch.filter(os.listdir(rpmdir), rpm_doc): + test_file = os.path.join(rpmdir, f) + dst = '/tmp/rpm-doc.rpm' + cls.tc.target.copyTo(test_file, dst) + + @classmethod + def tearDownClass(cls): + dst = '/tmp/rpm-doc.rpm' + cls.tc.target.run('rm -f %s' % dst) + + @OETestID(192) + @OETestDepends(['rpm.RpmBasicTest.test_rpm_help']) + def test_rpm_install(self): + status, output = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') + msg = 'Failed to install rpm-doc package: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(194) + @OETestDepends(['rpm.RpmInstallRemoveTest.test_rpm_install']) + def test_rpm_remove(self): + status,output = self.target.run('rpm -e rpm-doc') + msg = 'Failed to remove rpm-doc package: %s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(1096) + @OETestDepends(['rpm.RpmBasicTest.test_rpm_query']) + def test_rpm_query_nonroot(self): + + def set_up_test_user(u): + status, output = self.target.run('id -u %s' % u) + if status: + status, output = self.target.run('useradd %s' % u) + msg = 'Failed to create new user: %s' % output + self.assertTrue(status == 0, msg=msg) + + def exec_as_test_user(u): + status, output = self.target.run('su -c id %s' % u) + msg = 'Failed to execute as new user' + self.assertTrue("({0})".format(u) in output, msg=msg) + + status, output = self.target.run('su -c "rpm -qa" %s ' % u) + msg = 'status: %s. Cannot run rpm -qa: %s' % (status, output) + self.assertEqual(status, 0, msg=msg) + + def unset_up_test_user(u): + status, output = self.target.run('userdel -r %s' % u) + msg = 'Failed to erase user: %s' % output + self.assertTrue(status == 0, msg=msg) + + tuser = 'test1' + + try: + set_up_test_user(tuser) + exec_as_test_user(tuser) + finally: + unset_up_test_user(tuser) + + @OETestID(195) + @OETestDepends(['rpm.RpmInstallRemoveTest.test_rpm_remove']) + def test_check_rpm_install_removal_log_file_size(self): + """ + Summary: Check that rpm writes into /var/log/messages + Expected: There should be some RPM prefixed entries in the above file. + Product: BSPs + Author: Alexandru Georgescu <alexandru.c.georgescu@intel.com> + Author: Alexander Kanavin <alexander.kanavin@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + db_files_cmd = 'ls /var/lib/rpm/__db.*' + check_log_cmd = "grep RPM /var/log/messages | wc -l" + + # Make sure that some database files are under /var/lib/rpm as '__db.xxx' + status, output = self.target.run(db_files_cmd) + msg = 'Failed to find database files under /var/lib/rpm/ as __db.xxx' + self.assertEqual(0, status, msg=msg) + + # Remove the package just in case + self.target.run('rpm -e rpm-doc') + + # Install/Remove a package 10 times + for i in range(10): + status, output = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') + msg = 'Failed to install rpm-doc package. Reason: {}'.format(output) + self.assertEqual(0, status, msg=msg) + + status, output = self.target.run('rpm -e rpm-doc') + msg = 'Failed to remove rpm-doc package. Reason: {}'.format(output) + self.assertEqual(0, status, msg=msg) + + # if using systemd this should ensure all entries are flushed to /var + status, output = self.target.run("journalctl --sync") + # Get the amount of entries in the log file + status, output = self.target.run(check_log_cmd) + msg = 'Failed to get the final size of the log file.' + self.assertEqual(0, status, msg=msg) + + # Check that there's enough of them + self.assertGreaterEqual(int(output), 80, + 'Cound not find sufficient amount of rpm entries in /var/log/messages, found {} entries'.format(output)) diff --git a/meta/lib/oeqa/runtime/cases/scanelf.py b/meta/lib/oeqa/runtime/cases/scanelf.py new file mode 100644 index 0000000000..3ba1f78af9 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/scanelf.py @@ -0,0 +1,26 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.runtime.decorator.package import OEHasPackage + +class ScanelfTest(OERuntimeTestCase): + scancmd = 'scanelf --quiet --recursive --mount --ldpath --path' + + @OETestID(966) + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['pax-utils']) + def test_scanelf_textrel(self): + # print TEXTREL information + cmd = '%s --textrel' % self.scancmd + status, output = self.target.run(cmd) + msg = '\n'.join([cmd, output]) + self.assertEqual(output.strip(), '', msg=msg) + + @OETestID(967) + @OETestDepends(['scanelf.ScanelfTest.test_scanelf_textrel']) + def test_scanelf_rpath(self): + # print RPATH information + cmd = '%s --textrel --rpath' % self.scancmd + status, output = self.target.run(cmd) + msg = '\n'.join([cmd, output]) + self.assertEqual(output.strip(), '', msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/scp.py b/meta/lib/oeqa/runtime/cases/scp.py new file mode 100644 index 0000000000..f488a6175b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/scp.py @@ -0,0 +1,33 @@ +import os +from tempfile import mkstemp + +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class ScpTest(OERuntimeTestCase): + + @classmethod + def setUpClass(cls): + cls.tmp_fd, cls.tmp_path = mkstemp() + with os.fdopen(cls.tmp_fd, 'w') as f: + f.seek(2 ** 22 -1) + f.write(os.linesep) + + @classmethod + def tearDownClass(cls): + os.remove(cls.tmp_path) + + @OETestID(220) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_scp_file(self): + dst = '/tmp/test_scp_file' + + (status, output) = self.target.copyTo(self.tmp_path, dst) + msg = 'File could not be copied. Output: %s' % output + self.assertEqual(status, 0, msg=msg) + + (status, output) = self.target.run('ls -la %s' % dst) + self.assertEqual(status, 0, msg = 'SCP test failed') + + self.target.run('rm %s' % dst) diff --git a/meta/lib/oeqa/runtime/cases/skeletoninit.py b/meta/lib/oeqa/runtime/cases/skeletoninit.py new file mode 100644 index 0000000000..4fdcf033a3 --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/skeletoninit.py @@ -0,0 +1,33 @@ +# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=284 +# testcase. Image under test must have meta-skeleton layer in bblayers and +# IMAGE_INSTALL_append = " service" in local.conf +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar +from oeqa.runtime.decorator.package import OEHasPackage + +class SkeletonBasicTest(OERuntimeTestCase): + + @OETestDepends(['ssh.SSHTest.test_ssh']) + @OEHasPackage(['service']) + @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', + 'Not appropiate for systemd image') + def test_skeleton_availability(self): + status, output = self.target.run('ls /etc/init.d/skeleton') + msg = 'skeleton init script not found. Output:\n%s' % output + self.assertEqual(status, 0, msg=msg) + + status, output = self.target.run('ls /usr/sbin/skeleton-test') + msg = 'skeleton-test not found. Output:\n%s' % output + self.assertEqual(status, 0, msg=msg) + + @OETestID(284) + @OETestDepends(['skeletoninit.SkeletonBasicTest.test_skeleton_availability']) + def test_skeleton_script(self): + output1 = self.target.run("/etc/init.d/skeleton start")[1] + cmd = '%s | grep [s]keleton-test' % self.tc.target_cmds['ps'] + status, output2 = self.target.run(cmd) + msg = ('Skeleton script could not be started:' + '\n%s\n%s' % (output1, output2)) + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ssh.py b/meta/lib/oeqa/runtime/cases/ssh.py new file mode 100644 index 0000000000..eca167969a --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/ssh.py @@ -0,0 +1,15 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class SSHTest(OERuntimeTestCase): + + @OETestID(224) + @OETestDepends(['ping.PingTest.test_ping']) + def test_ssh(self): + (status, output) = self.target.run('uname -a') + self.assertEqual(status, 0, msg='SSH Test failed: %s' % output) + (status, output) = self.target.run('cat /etc/masterimage') + msg = "This isn't the right image - /etc/masterimage " \ + "shouldn't be here %s" % output + self.assertEqual(status, 1, msg=msg) diff --git a/meta/lib/oeqa/runtime/systemd.py b/meta/lib/oeqa/runtime/cases/systemd.py index a96efa28dc..db69384c8a 100644 --- a/meta/lib/oeqa/runtime/systemd.py +++ b/meta/lib/oeqa/runtime/cases/systemd.py @@ -1,28 +1,27 @@ -import unittest import re -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * +import time -def setUpModule(): - if not oeRuntimeTest.hasFeature("systemd"): - skipModule("target doesn't have systemd in DISTRO_FEATURES") - if "systemd" != oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", True): - skipModule("systemd is not the init manager for this image") +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfDataVar, skipIfNotDataVar +from oeqa.runtime.decorator.package import OEHasPackage +from oeqa.core.decorator.data import skipIfNotFeature +class SystemdTest(OERuntimeTestCase): -class SystemdTest(oeRuntimeTest): - - def systemctl(self, action = '', target = '', expected = 0, verbose = False): + def systemctl(self, action='', target='', expected=0, verbose=False): command = 'systemctl %s %s' % (action, target) status, output = self.target.run(command) message = '\n'.join([command, output]) if status != expected and verbose: - message += self.target.run('systemctl status --full %s' % target)[1] + cmd = 'systemctl status --full %s' % target + message += self.target.run(cmd)[1] self.assertEqual(status, expected, message) return output #TODO: use pyjournalctl instead - def journalctl(self, args='',l_match_units=[]): + def journalctl(self, args='',l_match_units=None): """ Request for the journalctl output to the current target system @@ -36,31 +35,23 @@ class SystemdTest(oeRuntimeTest): -ValueError, on a journalctl call with filtering by l_match_units that returned no entries """ - query_units="" - if len(l_match_units): + + query_units='' + if l_match_units: query_units = ['_SYSTEMD_UNIT='+unit for unit in l_match_units] - query_units = " ".join(query_units) + query_units = ' '.join(query_units) command = 'journalctl %s %s' %(args, query_units) status, output = self.target.run(command) if status: - raise AssertionError("Command '%s' returned non-zero exit \ - code %d:\n%s" % (command, status, output)) + raise AssertionError("Command '%s' returned non-zero exit " + 'code %d:\n%s' % (command, status, output)) if len(output) == 1 and "-- No entries --" in output: - raise ValueError("List of units to match: %s, returned no entries" + raise ValueError('List of units to match: %s, returned no entries' % l_match_units) return output class SystemdBasicTests(SystemdTest): - @skipUnlessPassed('test_ssh') - def test_systemd_basic(self): - self.systemctl('--version') - - @testcase(551) - @skipUnlessPassed('test_system_basic') - def test_systemd_list(self): - self.systemctl('list-unit-files') - def settle(self): """ Block until systemd has finished activating any units being activated, @@ -70,7 +61,6 @@ class SystemdBasicTests(SystemdTest): activating, or (False, message string) if there are still units activating (generally, failing units that restart). """ - import time endtime = time.time() + (60 * 2) while True: status, output = self.target.run('systemctl --state=activating') @@ -80,55 +70,65 @@ class SystemdBasicTests(SystemdTest): return (False, output) time.sleep(10) - @testcase(550) - @skipUnlessPassed('test_systemd_basic') + @skipIfNotFeature('systemd', + 'Test requires systemd to be in DISTRO_FEATURES') + @skipIfNotDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd', + 'systemd is not the init manager for this image') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_systemd_basic(self): + self.systemctl('--version') + + @OETestID(551) + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) + def test_systemd_list(self): + self.systemctl('list-unit-files') + + @OETestID(550) + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) def test_systemd_failed(self): settled, output = self.settle() - self.assertTrue(settled, msg="Timed out waiting for systemd to settle:\n" + output) + msg = "Timed out waiting for systemd to settle:\n%s" % output + self.assertTrue(settled, msg=msg) output = self.systemctl('list-units', '--failed') - match = re.search("0 loaded units listed", output) + match = re.search('0 loaded units listed', output) if not match: output += self.systemctl('status --full --failed') - self.assertTrue(match, msg="Some systemd units failed:\n%s" % output) + self.assertTrue(match, msg='Some systemd units failed:\n%s' % output) class SystemdServiceTests(SystemdTest): - def check_for_avahi(self): - if not self.hasPackage('avahi-daemon'): - raise unittest.SkipTest("Testcase dependency not met: need avahi-daemon installed on target") - - @skipUnlessPassed('test_systemd_basic') + @OEHasPackage(['avahi-daemon']) + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) def test_systemd_status(self): - self.check_for_avahi() self.systemctl('status --full', 'avahi-daemon.service') - @testcase(695) - @skipUnlessPassed('test_systemd_status') + @OETestID(695) + @OETestDepends(['systemd.SystemdServiceTests.test_systemd_status']) def test_systemd_stop_start(self): - self.check_for_avahi() self.systemctl('stop', 'avahi-daemon.service') - self.systemctl('is-active', 'avahi-daemon.service', expected=3, verbose=True) + self.systemctl('is-active', 'avahi-daemon.service', + expected=3, verbose=True) self.systemctl('start','avahi-daemon.service') self.systemctl('is-active', 'avahi-daemon.service', verbose=True) - @testcase(696) - @skipUnlessPassed('test_systemd_basic') + @OETestID(696) + @OETestDepends(['systemd.SystemdServiceTests.test_systemd_status']) def test_systemd_disable_enable(self): - self.check_for_avahi() self.systemctl('disable', 'avahi-daemon.service') self.systemctl('is-enabled', 'avahi-daemon.service', expected=1) self.systemctl('enable', 'avahi-daemon.service') self.systemctl('is-enabled', 'avahi-daemon.service') class SystemdJournalTests(SystemdTest): - @skipUnlessPassed('test_ssh') + + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) def test_systemd_journal(self): - (status, output) = self.target.run('journalctl') + status, output = self.target.run('journalctl') self.assertEqual(status, 0, output) - @skipUnlessPassed('test_systemd_basic') + @OETestDepends(['systemd.SystemdBasicTests.test_systemd_basic']) def test_systemd_boot_time(self, systemd_TimeoutStartSec=90): """ Get the target boot time from journalctl and log it @@ -138,41 +138,44 @@ class SystemdJournalTests(SystemdTest): unit start timeout to compare against """ - # the expression chain that uniquely identifies the time boot message - expr_items=["Startup finished","kernel", "userspace","\.$"] + # The expression chain that uniquely identifies the time boot message. + expr_items=['Startup finished', 'kernel', 'userspace','\.$'] try: - output = self.journalctl(args="-o cat --reverse") + output = self.journalctl(args='-o cat --reverse') except AssertionError: - self.fail("Error occurred while calling journalctl") + self.fail('Error occurred while calling journalctl') if not len(output): - self.fail("Error, unable to get startup time from systemd journal") + self.fail('Error, unable to get startup time from systemd journal') - # check for the regular expression items that match the startup time + # Check for the regular expression items that match the startup time. for line in output.split('\n'): - check_match = "".join(re.findall(".*".join(expr_items), line)) - if check_match: break - # put the startup time in the test log + check_match = ''.join(re.findall('.*'.join(expr_items), line)) + if check_match: + break + # Put the startup time in the test log if check_match: - print("%s" % check_match) + self.tc.logger.info('%s' % check_match) else: - self.skipTest("Error at obtaining the boot time from journalctl") + self.skipTest('Error at obtaining the boot time from journalctl') boot_time_sec = 0 - # get the numeric values from the string and convert them to seconds - # same data will be placed in list and string for manipulation - l_boot_time = check_match.split(" ")[-2:] - s_boot_time = " ".join(l_boot_time) + # Get the numeric values from the string and convert them to seconds + # same data will be placed in list and string for manipulation. + l_boot_time = check_match.split(' ')[-2:] + s_boot_time = ' '.join(l_boot_time) try: - # Obtain the minutes it took to boot + # Obtain the minutes it took to boot. if l_boot_time[0].endswith('min') and l_boot_time[0][0].isdigit(): - boot_time_min = s_boot_time.split("min")[0] - # convert to seconds and accumulate it + boot_time_min = s_boot_time.split('min')[0] + # Convert to seconds and accumulate it. boot_time_sec += int(boot_time_min) * 60 - # Obtain the seconds it took to boot and accumulate - boot_time_sec += float(l_boot_time[1].split("s")[0]) + # Obtain the seconds it took to boot and accumulate. + boot_time_sec += float(l_boot_time[1].split('s')[0]) except ValueError: - self.skipTest("Error when parsing time from boot string") - #Assert the target boot time against systemd's unit start timeout + self.skipTest('Error when parsing time from boot string') + + # Assert the target boot time against systemd's unit start timeout. if boot_time_sec > systemd_TimeoutStartSec: - print("Target boot time %s exceeds systemd's TimeoutStartSec %s"\ - %(boot_time_sec, systemd_TimeoutStartSec)) + msg = ("Target boot time %s exceeds systemd's TimeoutStartSec %s" + % (boot_time_sec, systemd_TimeoutStartSec)) + self.tc.logger.info(msg) diff --git a/meta/lib/oeqa/runtime/cases/x32lib.py b/meta/lib/oeqa/runtime/cases/x32lib.py new file mode 100644 index 0000000000..8da0154e7b --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/x32lib.py @@ -0,0 +1,19 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotInDataVar + +class X32libTest(OERuntimeTestCase): + + @skipIfNotInDataVar('DEFAULTTUNE', 'x86-64-x32', + 'DEFAULTTUNE is not set to x86-64-x32') + @OETestID(281) + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_x32_file(self): + cmd = 'readelf -h /bin/ls | grep Class | grep ELF32' + status1 = self.target.run(cmd)[0] + cmd = 'readelf -h /bin/ls | grep Machine | grep X86-64' + status2 = self.target.run(cmd)[0] + msg = ("/bin/ls isn't an X86-64 ELF32 binary. readelf says: %s" % + self.target.run("readelf -h /bin/ls")[1]) + self.assertTrue(status1 == 0 and status2 == 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/xorg.py b/meta/lib/oeqa/runtime/cases/xorg.py new file mode 100644 index 0000000000..2124813e3c --- /dev/null +++ b/meta/lib/oeqa/runtime/cases/xorg.py @@ -0,0 +1,17 @@ +from oeqa.runtime.case import OERuntimeTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator.data import skipIfNotFeature + +class XorgTest(OERuntimeTestCase): + + @OETestID(1151) + @skipIfNotFeature('x11-base', + 'Test requires x11 to be in IMAGE_FEATURES') + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_xorg_running(self): + cmd ='%s | grep -v xinit | grep [X]org' % self.tc.target_cmds['ps'] + status, output = self.target.run(cmd) + msg = ('Xorg does not appear to be running %s' % + self.target.run(self.tc.target_cmds['ps'])[1]) + self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/connman.py b/meta/lib/oeqa/runtime/connman.py deleted file mode 100644 index 003fefe2ce..0000000000 --- a/meta/lib/oeqa/runtime/connman.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("connman"): - skipModule("No connman package in image") - - -class ConnmanTest(oeRuntimeTest): - - def service_status(self, service): - if oeRuntimeTest.hasFeature("systemd"): - (status, output) = self.target.run('systemctl status -l %s' % service) - return output - else: - return "Unable to get status or logs for %s" % service - - @testcase(961) - @skipUnlessPassed('test_ssh') - def test_connmand_help(self): - (status, output) = self.target.run('/usr/sbin/connmand --help') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - - @testcase(221) - @skipUnlessPassed('test_connmand_help') - def test_connmand_running(self): - (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep [c]onnmand') - if status != 0: - print(self.service_status("connman")) - self.fail("No connmand process running") diff --git a/meta/lib/oeqa/runtime/context.py b/meta/lib/oeqa/runtime/context.py new file mode 100644 index 0000000000..c4cd76cf44 --- /dev/null +++ b/meta/lib/oeqa/runtime/context.py @@ -0,0 +1,220 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os + +from oeqa.core.context import OETestContext, OETestContextExecutor +from oeqa.core.target.ssh import OESSHTarget +from oeqa.core.target.qemu import OEQemuTarget +from oeqa.utils.dump import HostDumper + +from oeqa.runtime.loader import OERuntimeTestLoader + +class OERuntimeTestContext(OETestContext): + loaderClass = OERuntimeTestLoader + runtime_files_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "files") + + def __init__(self, td, logger, target, + host_dumper, image_packages, extract_dir): + super(OERuntimeTestContext, self).__init__(td, logger) + + self.target = target + self.image_packages = image_packages + self.host_dumper = host_dumper + self.extract_dir = extract_dir + self._set_target_cmds() + + def _set_target_cmds(self): + self.target_cmds = {} + + self.target_cmds['ps'] = 'ps' + if 'procps' in self.image_packages: + self.target_cmds['ps'] = self.target_cmds['ps'] + ' -ef' + +class OERuntimeTestContextExecutor(OETestContextExecutor): + _context_class = OERuntimeTestContext + + name = 'runtime' + help = 'runtime test component' + description = 'executes runtime tests over targets' + + default_cases = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'cases') + default_data = None + default_test_data = 'data/testdata.json' + default_tests = '' + + default_target_type = 'simpleremote' + default_manifest = 'data/manifest' + default_server_ip = '192.168.7.1' + default_target_ip = '192.168.7.2' + default_host_dumper_dir = '/tmp/oe-saved-tests' + default_extract_dir = 'packages/extracted' + + def register_commands(self, logger, subparsers): + super(OERuntimeTestContextExecutor, self).register_commands(logger, subparsers) + + runtime_group = self.parser.add_argument_group('runtime options') + + runtime_group.add_argument('--target-type', action='store', + default=self.default_target_type, choices=['simpleremote', 'qemu'], + help="Target type of device under test, default: %s" \ + % self.default_target_type) + runtime_group.add_argument('--target-ip', action='store', + default=self.default_target_ip, + help="IP address of device under test, default: %s" \ + % self.default_target_ip) + runtime_group.add_argument('--server-ip', action='store', + default=self.default_target_ip, + help="IP address of device under test, default: %s" \ + % self.default_server_ip) + + runtime_group.add_argument('--host-dumper-dir', action='store', + default=self.default_host_dumper_dir, + help="Directory where host status is dumped, if tests fails, default: %s" \ + % self.default_host_dumper_dir) + + runtime_group.add_argument('--packages-manifest', action='store', + default=self.default_manifest, + help="Package manifest of the image under testi, default: %s" \ + % self.default_manifest) + + runtime_group.add_argument('--extract-dir', action='store', + default=self.default_extract_dir, + help='Directory where extracted packages reside, default: %s' \ + % self.default_extract_dir) + + runtime_group.add_argument('--qemu-boot', action='store', + help="Qemu boot configuration, only needed when target_type is QEMU.") + + @staticmethod + def getTarget(target_type, logger, target_ip, server_ip, **kwargs): + target = None + + if target_type == 'simpleremote': + target = OESSHTarget(logger, target_ip, server_ip, **kwargs) + elif target_type == 'qemu': + target = OEQemuTarget(logger, target_ip, server_ip, **kwargs) + else: + # XXX: This code uses the old naming convention for controllers and + # targets, the idea it is to leave just targets as the controller + # most of the time was just a wrapper. + # XXX: This code tries to import modules from lib/oeqa/controllers + # directory and treat them as controllers, it will less error prone + # to use introspection to load such modules. + # XXX: Don't base your targets on this code it will be refactored + # in the near future. + # Custom target module loading + try: + target_modules_path = kwargs.get('target_modules_path', '') + controller = OERuntimeTestContextExecutor.getControllerModule(target_type, target_modules_path) + target = controller(logger, target_ip, server_ip, **kwargs) + except ImportError as e: + raise TypeError("Failed to import %s from available controller modules" % target_type) + + return target + + # Search oeqa.controllers module directory for and return a controller + # corresponding to the given target name. + # AttributeError raised if not found. + # ImportError raised if a provided module can not be imported. + @staticmethod + def getControllerModule(target, target_modules_path): + controllerslist = OERuntimeTestContextExecutor._getControllerModulenames(target_modules_path) + controller = OERuntimeTestContextExecutor._loadControllerFromName(target, controllerslist) + return controller + + # Return a list of all python modules in lib/oeqa/controllers for each + # layer in bbpath + @staticmethod + def _getControllerModulenames(target_modules_path): + + controllerslist = [] + + def add_controller_list(path): + if not os.path.exists(os.path.join(path, '__init__.py')): + raise OSError('Controllers directory %s exists but is missing __init__.py' % path) + files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) + for f in files: + module = 'oeqa.controllers.' + f[:-3] + if module not in controllerslist: + controllerslist.append(module) + else: + raise RuntimeError("Duplicate controller module found for %s. Layers should create unique controller module names" % module) + + extpath = target_modules_path.split(':') + for p in extpath: + controllerpath = os.path.join(p, 'lib', 'oeqa', 'controllers') + if os.path.exists(controllerpath): + add_controller_list(controllerpath) + return controllerslist + + # Search for and return a controller from given target name and + # set of module names. + # Raise AttributeError if not found. + # Raise ImportError if a provided module can not be imported + @staticmethod + def _loadControllerFromName(target, modulenames): + for name in modulenames: + obj = OERuntimeTestContextExecutor._loadControllerFromModule(target, name) + if obj: + return obj + raise AttributeError("Unable to load {0} from available modules: {1}".format(target, str(modulenames))) + + # Search for and return a controller or None from given module name + @staticmethod + def _loadControllerFromModule(target, modulename): + obj = None + # import module, allowing it to raise import exception + try: + module = __import__(modulename, globals(), locals(), [target]) + except Exception as e: + return obj + # look for target class in the module, catching any exceptions as it + # is valid that a module may not have the target class. + try: + obj = getattr(module, target) + except: + obj = None + return obj + + @staticmethod + def readPackagesManifest(manifest): + if not manifest or not os.path.exists(manifest): + raise OSError("Manifest file not exists: %s" % manifest) + + image_packages = set() + with open(manifest, 'r') as f: + for line in f.readlines(): + line = line.strip() + if line and not line.startswith("#"): + image_packages.add(line.split()[0]) + + return image_packages + + @staticmethod + def getHostDumper(cmds, directory): + return HostDumper(cmds, directory) + + def _process_args(self, logger, args): + if not args.packages_manifest: + raise TypeError('Manifest file not provided') + + super(OERuntimeTestContextExecutor, self)._process_args(logger, args) + + target_kwargs = {} + target_kwargs['qemuboot'] = args.qemu_boot + + self.tc_kwargs['init']['target'] = \ + OERuntimeTestContextExecutor.getTarget(args.target_type, + None, args.target_ip, args.server_ip, **target_kwargs) + self.tc_kwargs['init']['host_dumper'] = \ + OERuntimeTestContextExecutor.getHostDumper(None, + args.host_dumper_dir) + self.tc_kwargs['init']['image_packages'] = \ + OERuntimeTestContextExecutor.readPackagesManifest( + args.packages_manifest) + self.tc_kwargs['init']['extract_dir'] = args.extract_dir + +_executor_class = OERuntimeTestContextExecutor diff --git a/meta/lib/oeqa/runtime/date.py b/meta/lib/oeqa/runtime/date.py deleted file mode 100644 index 447987e075..0000000000 --- a/meta/lib/oeqa/runtime/date.py +++ /dev/null @@ -1,31 +0,0 @@ -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * -import re - -class DateTest(oeRuntimeTest): - - def setUpLocal(self): - if oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", True) == "systemd": - self.target.run('systemctl stop systemd-timesyncd') - - def tearDownLocal(self): - if oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", True) == "systemd": - self.target.run('systemctl start systemd-timesyncd') - - @testcase(211) - @skipUnlessPassed("test_ssh") - def test_date(self): - (status, output) = self.target.run('date +"%Y-%m-%d %T"') - self.assertEqual(status, 0, msg="Failed to get initial date, output: %s" % output) - oldDate = output - - sampleDate = '"2016-08-09 10:00:00"' - (status, output) = self.target.run("date -s %s" % sampleDate) - self.assertEqual(status, 0, msg="Date set failed, output: %s" % output) - - (status, output) = self.target.run("date -R") - p = re.match('Tue, 09 Aug 2016 10:00:.. \+0000', output) - self.assertTrue(p, msg="The date was not set correctly, output: %s" % output) - - (status, output) = self.target.run('date -s "%s"' % oldDate) - self.assertEqual(status, 0, msg="Failed to reset date, output: %s" % output) diff --git a/meta/lib/oeqa/runtime/decorator/package.py b/meta/lib/oeqa/runtime/decorator/package.py new file mode 100644 index 0000000000..aa6ecb68fa --- /dev/null +++ b/meta/lib/oeqa/runtime/decorator/package.py @@ -0,0 +1,53 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.decorator import OETestDecorator, registerDecorator +from oeqa.core.utils.misc import strToSet + +@registerDecorator +class OEHasPackage(OETestDecorator): + """ + Checks if image has packages (un)installed. + + The argument must be a string, set, or list of packages that must be + installed or not present in the image. + + The way to tell a package must not be in an image is using an + exclamation point ('!') before the name of the package. + + If test depends on pkg1 or pkg2 you need to use: + @OEHasPackage({'pkg1', 'pkg2'}) + + If test depends on pkg1 and pkg2 you need to use: + @OEHasPackage('pkg1') + @OEHasPackage('pkg2') + + If test depends on pkg1 but pkg2 must not be present use: + @OEHasPackage({'pkg1', '!pkg2'}) + """ + + attrs = ('need_pkgs',) + + def setUpDecorator(self): + need_pkgs = set() + unneed_pkgs = set() + pkgs = strToSet(self.need_pkgs) + for pkg in pkgs: + if pkg.startswith('!'): + unneed_pkgs.add(pkg[1:]) + else: + need_pkgs.add(pkg) + + if unneed_pkgs: + msg = 'Checking if %s is not installed' % ', '.join(unneed_pkgs) + self.logger.debug(msg) + if not self.case.tc.image_packages.isdisjoint(unneed_pkgs): + msg = "Test can't run with %s installed" % ', or'.join(unneed_pkgs) + self.case.skipTest(msg) + + if need_pkgs: + msg = 'Checking if at least one of %s is installed' % ', '.join(need_pkgs) + self.logger.debug(msg) + if self.case.tc.image_packages.isdisjoint(need_pkgs): + msg = "Test requires %s to be installed" % ', or'.join(need_pkgs) + self.case.skipTest(msg) diff --git a/meta/lib/oeqa/runtime/df.py b/meta/lib/oeqa/runtime/df.py deleted file mode 100644 index 09569d5ff6..0000000000 --- a/meta/lib/oeqa/runtime/df.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * - - -class DfTest(oeRuntimeTest): - - @testcase(234) - @skipUnlessPassed("test_ssh") - def test_df(self): - (status,output) = self.target.run("df / | sed -n '2p' | awk '{print $4}'") - self.assertTrue(int(output)>5120, msg="Not enough space on image. Current size is %s" % output) diff --git a/meta/lib/oeqa/runtime/gcc.py b/meta/lib/oeqa/runtime/gcc.py deleted file mode 100644 index d90cd1799a..0000000000 --- a/meta/lib/oeqa/runtime/gcc.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - - -class GccCompileTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.c"), "/tmp/test.c") - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "testmakefile"), "/tmp/testmakefile") - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.cpp"), "/tmp/test.cpp") - - @testcase(203) - def test_gcc_compile(self): - (status, output) = self.target.run('gcc /tmp/test.c -o /tmp/test -lm') - self.assertEqual(status, 0, msg="gcc compile failed, output: %s" % output) - (status, output) = self.target.run('/tmp/test') - self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) - - @testcase(200) - def test_gpp_compile(self): - (status, output) = self.target.run('g++ /tmp/test.c -o /tmp/test -lm') - self.assertEqual(status, 0, msg="g++ compile failed, output: %s" % output) - (status, output) = self.target.run('/tmp/test') - self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) - - @testcase(1142) - def test_gpp2_compile(self): - (status, output) = self.target.run('g++ /tmp/test.cpp -o /tmp/test -lm') - self.assertEqual(status, 0, msg="g++ compile failed, output: %s" % output) - (status, output) = self.target.run('/tmp/test') - self.assertEqual(status, 0, msg="running compiled file failed, output %s" % output) - - @testcase(204) - def test_make(self): - (status, output) = self.target.run('cd /tmp; make -f testmakefile') - self.assertEqual(status, 0, msg="running make failed, output %s" % output) - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run("rm /tmp/test.c /tmp/test.o /tmp/test /tmp/testmakefile") diff --git a/meta/lib/oeqa/runtime/kernelmodule.py b/meta/lib/oeqa/runtime/kernelmodule.py deleted file mode 100644 index 38ca184540..0000000000 --- a/meta/lib/oeqa/runtime/kernelmodule.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - - -class KernelModuleTest(oeRuntimeTest): - - def setUpLocal(self): - self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod.c"), "/tmp/hellomod.c") - self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod_makefile"), "/tmp/Makefile") - - @testcase('316') - @skipUnlessPassed('test_ssh') - @skipUnlessPassed('test_gcc_compile') - def test_kernel_module(self): - cmds = [ - 'cd /usr/src/kernel && make scripts', - 'cd /tmp && make', - 'cd /tmp && insmod hellomod.ko', - 'lsmod | grep hellomod', - 'dmesg | grep Hello', - 'rmmod hellomod', 'dmesg | grep "Cleaning up hellomod"' - ] - for cmd in cmds: - (status, output) = self.target.run(cmd, 900) - self.assertEqual(status, 0, msg="\n".join([cmd, output])) - - def tearDownLocal(self): - self.target.run('rm -f /tmp/Makefile /tmp/hellomod.c') diff --git a/meta/lib/oeqa/runtime/ldd.py b/meta/lib/oeqa/runtime/ldd.py deleted file mode 100644 index 47b3885df2..0000000000 --- a/meta/lib/oeqa/runtime/ldd.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class LddTest(oeRuntimeTest): - - @testcase(962) - @skipUnlessPassed('test_ssh') - def test_ldd_exists(self): - (status, output) = self.target.run('which ldd') - self.assertEqual(status, 0, msg = "ldd does not exist in PATH: which ldd: %s" % output) - - @testcase(239) - @skipUnlessPassed('test_ldd_exists') - def test_ldd_rtldlist_check(self): - (status, output) = self.target.run('for i in $(which ldd | xargs cat | grep "^RTLDLIST"|cut -d\'=\' -f2|tr -d \'"\'); do test -f $i && echo $i && break; done') - self.assertEqual(status, 0, msg = "ldd path not correct or RTLDLIST files don't exist. ") diff --git a/meta/lib/oeqa/runtime/loader.py b/meta/lib/oeqa/runtime/loader.py new file mode 100644 index 0000000000..041ef976eb --- /dev/null +++ b/meta/lib/oeqa/runtime/loader.py @@ -0,0 +1,16 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.core.loader import OETestLoader +from oeqa.runtime.case import OERuntimeTestCase + +class OERuntimeTestLoader(OETestLoader): + caseClass = OERuntimeTestCase + + def _getTestCase(self, testCaseClass, tcName): + case = super(OERuntimeTestLoader, self)._getTestCase(testCaseClass, tcName) + + # Adds custom attributes to the OERuntimeTestCase + setattr(case, 'target', self.tc.target) + + return case diff --git a/meta/lib/oeqa/runtime/logrotate.py b/meta/lib/oeqa/runtime/logrotate.py deleted file mode 100644 index de300bf55f..0000000000 --- a/meta/lib/oeqa/runtime/logrotate.py +++ /dev/null @@ -1,28 +0,0 @@ -# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=289 testcase -# Note that the image under test must have logrotate installed - -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("logrotate"): - skipModule("No logrotate package in image") - - -class LogrotateTest(oeRuntimeTest): - - @skipUnlessPassed("test_ssh") - def test_1_logrotate_setup(self): - (status, output) = self.target.run('mkdir $HOME/logrotate_dir') - self.assertEqual(status, 0, msg = "Could not create logrotate_dir. Output: %s" % output) - (status, output) = self.target.run("sed -i \"s#wtmp {#wtmp {\\n olddir $HOME/logrotate_dir#\" /etc/logrotate.conf") - self.assertEqual(status, 0, msg = "Could not write to logrotate.conf file. Status and output: %s and %s)" % (status, output)) - - @testcase(289) - @skipUnlessPassed("test_1_logrotate_setup") - def test_2_logrotate(self): - (status, output) = self.target.run('logrotate -f /etc/logrotate.conf') - self.assertEqual(status, 0, msg = "logrotate service could not be reloaded. Status and output: %s and %s" % (status, output)) - output = self.target.run('ls -la $HOME/logrotate_dir/ | wc -l')[1] - self.assertTrue(int(output)>=3, msg = "new logfile could not be created. List of files within log directory: %s" %(self.target.run('ls -la $HOME/logrotate_dir')[1])) diff --git a/meta/lib/oeqa/runtime/multilib.py b/meta/lib/oeqa/runtime/multilib.py deleted file mode 100644 index 593d385021..0000000000 --- a/meta/lib/oeqa/runtime/multilib.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - multilibs = oeRuntimeTest.tc.d.getVar("MULTILIBS", True) or "" - if "multilib:lib32" not in multilibs: - skipModule("this isn't a multilib:lib32 image") - - -class MultilibTest(oeRuntimeTest): - - def archtest(self, binary, arch): - """ - Check that ``binary`` has the ELF class ``arch`` (e.g. ELF32/ELF64). - """ - - (status, output) = self.target.run("readelf -h %s" % binary) - self.assertEqual(status, 0, "Failed to readelf %s" % binary) - - l = [l.split()[1] for l in output.split('\n') if "Class:" in l] - if l: - theclass = l[0] - else: - self.fail("Cannot parse readelf output\n" + s) - - self.assertEqual(theclass, arch, msg="%s isn't %s (is %s)" % (binary, arch, theclass)) - - @skipUnlessPassed('test_ssh') - def test_check_multilib_libc(self): - """ - Check that a multilib image has both 32-bit and 64-bit libc in. - """ - self.archtest("/lib/libc.so.6", "ELF32") - self.archtest("/lib64/libc.so.6", "ELF64") - - @testcase('279') - @skipUnlessPassed('test_check_multilib_libc') - def test_file_connman(self): - self.assertTrue(oeRuntimeTest.hasPackage('lib32-connman'), msg="This test assumes lib32-connman is installed") - - self.archtest("/usr/sbin/connmand", "ELF32") diff --git a/meta/lib/oeqa/runtime/pam.py b/meta/lib/oeqa/runtime/pam.py deleted file mode 100644 index c8205c9abc..0000000000 --- a/meta/lib/oeqa/runtime/pam.py +++ /dev/null @@ -1,25 +0,0 @@ -# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=287 testcase -# Note that the image under test must have "pam" in DISTRO_FEATURES - -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("pam"): - skipModule("target doesn't have 'pam' in DISTRO_FEATURES") - - -class PamBasicTest(oeRuntimeTest): - - @testcase(287) - @skipUnlessPassed('test_ssh') - def test_pam(self): - (status, output) = self.target.run('login --help') - self.assertEqual(status, 1, msg = "login command does not work as expected. Status and output:%s and %s" %(status, output)) - (status, output) = self.target.run('passwd --help') - self.assertEqual(status, 0, msg = "passwd command does not work as expected. Status and output:%s and %s" %(status, output)) - (status, output) = self.target.run('su --help') - self.assertEqual(status, 0, msg = "su command does not work as expected. Status and output:%s and %s" %(status, output)) - (status, output) = self.target.run('useradd --help') - self.assertEqual(status, 0, msg = "useradd command does not work as expected. Status and output:%s and %s" %(status, output)) diff --git a/meta/lib/oeqa/runtime/parselogs.py b/meta/lib/oeqa/runtime/parselogs.py deleted file mode 100644 index 242cd8cdd5..0000000000 --- a/meta/lib/oeqa/runtime/parselogs.py +++ /dev/null @@ -1,277 +0,0 @@ -import os -import unittest -import subprocess -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * - -#in the future these lists could be moved outside of module -errors = ["error", "cannot", "can\'t", "failed"] - -common_errors = [ - "(WW) warning, (EE) error, (NI) not implemented, (??) unknown.", - "dma timeout", - "can\'t add hid device:", - "usbhid: probe of ", - "_OSC failed (AE_ERROR)", - "_OSC failed (AE_SUPPORT)", - "AE_ALREADY_EXISTS", - "ACPI _OSC request failed (AE_SUPPORT)", - "can\'t disable ASPM", - "Failed to load module \"vesa\"", - "Failed to load module vesa", - "Failed to load module \"modesetting\"", - "Failed to load module modesetting", - "Failed to load module \"glx\"", - "Failed to load module \"fbdev\"", - "Failed to load module fbdev", - "Failed to load module glx", - "[drm] Cannot find any crtc or sizes - going 1024x768", - "_OSC failed (AE_NOT_FOUND); disabling ASPM", - "Open ACPI failed (/var/run/acpid.socket) (No such file or directory)", - "NX (Execute Disable) protection cannot be enabled: non-PAE kernel!", - "hd.: possibly failed opcode", - 'NETLINK INITIALIZATION FAILED', - 'kernel: Cannot find map file', - 'omap_hwmod: debugss: _wait_target_disable failed', - 'VGA arbiter: cannot open kernel arbiter, no multi-card support', - 'Failed to find URL:http://ipv4.connman.net/online/status.html', - 'Online check failed for', - 'netlink init failed', - 'Fast TSC calibration', - "BAR 0-9", - "Failed to load module \"ati\"", - "controller can't do DEVSLP, turning off", - "stmmac_dvr_probe: warning: cannot get CSR clock", - "error: couldn\'t mount because of unsupported optional features", - ] - -video_related = [ - "uvesafb", -] - -x86_common = [ - '[drm:psb_do_init] *ERROR* Debug is', - 'wrong ELF class', - 'Could not enable PowerButton event', - 'probe of LNXPWRBN:00 failed with error -22', - 'pmd_set_huge: Cannot satisfy', -] + common_errors - -qemux86_common = [ - 'wrong ELF class', - "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", - "can't claim BAR ", -] + common_errors - -ignore_errors = { - 'default' : common_errors, - 'qemux86' : [ - 'Failed to access perfctr msr (MSR', - ] + qemux86_common, - 'qemux86-64' : qemux86_common, - 'qemumips' : [ - 'Failed to load module "glx"', - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', - ] + common_errors, - 'qemumips64' : [ - 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', - ] + common_errors, - 'qemuppc' : [ - 'PCI 0000:00 Cannot reserve Legacy IO [io 0x0000-0x0fff]', - 'host side 80-wire cable detection failed, limiting max speed', - 'mode "640x480" test failed', - 'Failed to load module "glx"', - ] + common_errors, - 'qemuarm' : [ - 'mmci-pl18x: probe of fpga:05 failed with error -22', - 'mmci-pl18x: probe of fpga:0b failed with error -22', - 'Failed to load module "glx"' - ] + common_errors, - 'qemuarm64' : [ - 'Fatal server error:', - '(EE) Server terminated with error (1). Closing log file.', - 'dmi: Firmware registration failed.', - ] + common_errors, - 'emenlow' : [ - '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', - '(EE) Failed to load module "psb"', - '(EE) Failed to load module psb', - '(EE) Failed to load module "psbdrv"', - '(EE) Failed to load module psbdrv', - '(EE) open /dev/fb0: No such file or directory', - '(EE) AIGLX: reverting to software rendering', - ] + x86_common, - 'intel-core2-32' : [ - 'ACPI: No _BQC method, cannot determine initial brightness', - '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', - '(EE) Failed to load module "psb"', - '(EE) Failed to load module psb', - '(EE) Failed to load module "psbdrv"', - '(EE) Failed to load module psbdrv', - '(EE) open /dev/fb0: No such file or directory', - '(EE) AIGLX: reverting to software rendering', - ] + x86_common, - 'intel-corei7-64' : x86_common, - 'crownbay' : x86_common, - 'genericx86' : x86_common, - 'genericx86-64' : x86_common, - 'edgerouter' : [ - 'Fatal server error:', - ] + common_errors, - 'jasperforest' : [ - 'Activated service \'org.bluez\' failed:', - 'Unable to find NFC netlink family', - ] + common_errors, -} - -log_locations = ["/var/log/","/var/log/dmesg", "/tmp/dmesg_output.log"] - -class ParseLogsTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.errors = errors - - # When systemd is enabled we need to notice errors on - # circular dependencies in units. - if self.hasFeature("systemd"): - self.errors.extend([ - 'Found ordering cycle on', - 'Breaking ordering cycle by deleting job', - 'deleted to break ordering cycle', - 'Ordering cycle found, skipping', - ]) - - self.ignore_errors = ignore_errors - self.log_locations = log_locations - self.msg = "" - (is_lsb, location) = oeRuntimeTest.tc.target.run("which LSB_Test.sh") - if is_lsb == 0: - for machine in self.ignore_errors: - self.ignore_errors[machine] = self.ignore_errors[machine] + video_related - - def getMachine(self): - return oeRuntimeTest.tc.d.getVar("MACHINE", True) - - #get some information on the CPU of the machine to display at the beginning of the output. This info might be useful in some cases. - def getHardwareInfo(self): - hwi = "" - (status, cpu_name) = self.target.run("cat /proc/cpuinfo | grep \"model name\" | head -n1 | awk 'BEGIN{FS=\":\"}{print $2}'") - (status, cpu_physical_cores) = self.target.run("cat /proc/cpuinfo | grep \"cpu cores\" | head -n1 | awk {'print $4'}") - (status, cpu_logical_cores) = self.target.run("cat /proc/cpuinfo | grep \"processor\" | wc -l") - (status, cpu_arch) = self.target.run("uname -m") - hwi += "Machine information: \n" - hwi += "*******************************\n" - hwi += "Machine name: "+self.getMachine()+"\n" - hwi += "CPU: "+str(cpu_name)+"\n" - hwi += "Arch: "+str(cpu_arch)+"\n" - hwi += "Physical cores: "+str(cpu_physical_cores)+"\n" - hwi += "Logical cores: "+str(cpu_logical_cores)+"\n" - hwi += "*******************************\n" - return hwi - - #go through the log locations provided and if it's a folder create a list with all the .log files in it, if it's a file just add - #it to that list - def getLogList(self, log_locations): - logs = [] - for location in log_locations: - (status, output) = self.target.run("test -f "+str(location)) - if (status == 0): - logs.append(str(location)) - else: - (status, output) = self.target.run("test -d "+str(location)) - if (status == 0): - (status, output) = self.target.run("find "+str(location)+"/*.log -maxdepth 1 -type f") - if (status == 0): - output = output.splitlines() - for logfile in output: - logs.append(os.path.join(location,str(logfile))) - return logs - - #copy the log files to be parsed locally - def transfer_logs(self, log_list): - target_logs = 'target_logs' - if not os.path.exists(target_logs): - os.makedirs(target_logs) - for f in log_list: - self.target.copy_from(f, target_logs) - - #get the local list of logs - def get_local_log_list(self, log_locations): - self.transfer_logs(self.getLogList(log_locations)) - logs = [ os.path.join('target_logs',f) for f in os.listdir('target_logs') if os.path.isfile(os.path.join('target_logs',f)) ] - return logs - - #build the grep command to be used with filters and exclusions - def build_grepcmd(self, errors, ignore_errors, log): - grepcmd = "grep " - grepcmd +="-Ei \"" - for error in errors: - grepcmd += error+"|" - grepcmd = grepcmd[:-1] - grepcmd += "\" "+str(log)+" | grep -Eiv \'" - try: - errorlist = ignore_errors[self.getMachine()] - except KeyError: - self.msg += "No ignore list found for this machine, using default\n" - errorlist = ignore_errors['default'] - for ignore_error in errorlist: - ignore_error = ignore_error.replace("(", "\(") - ignore_error = ignore_error.replace(")", "\)") - ignore_error = ignore_error.replace("'", ".") - ignore_error = ignore_error.replace("?", "\?") - ignore_error = ignore_error.replace("[", "\[") - ignore_error = ignore_error.replace("]", "\]") - ignore_error = ignore_error.replace("*", "\*") - ignore_error = ignore_error.replace("0-9", "[0-9]") - grepcmd += ignore_error+"|" - grepcmd = grepcmd[:-1] - grepcmd += "\'" - return grepcmd - - #grep only the errors so that their context could be collected. Default context is 10 lines before and after the error itself - def parse_logs(self, errors, ignore_errors, logs, lines_before = 10, lines_after = 10): - results = {} - rez = [] - grep_output = '' - for log in logs: - result = None - thegrep = self.build_grepcmd(errors, ignore_errors, log) - try: - result = subprocess.check_output(thegrep, shell=True).decode("utf-8") - except: - pass - if (result is not None): - results[log.replace('target_logs/','')] = {} - rez = result.splitlines() - for xrez in rez: - try: - grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]).decode("utf-8") - except: - pass - results[log.replace('target_logs/','')][xrez]=grep_output - return results - - #get the output of dmesg and write it in a file. This file is added to log_locations. - def write_dmesg(self): - (status, dmesg) = self.target.run("dmesg > /tmp/dmesg_output.log") - - @testcase(1059) - @skipUnlessPassed('test_ssh') - def test_parselogs(self): - self.write_dmesg() - log_list = self.get_local_log_list(self.log_locations) - result = self.parse_logs(self.errors, self.ignore_errors, log_list) - print(self.getHardwareInfo()) - errcount = 0 - for log in result: - self.msg += "Log: "+log+"\n" - self.msg += "-----------------------\n" - for error in result[log]: - errcount += 1 - self.msg += "Central error: "+str(error)+"\n" - self.msg += "***********************\n" - self.msg += result[str(log)][str(error)]+"\n" - self.msg += "***********************\n" - self.msg += "%s errors found in logs." % errcount - self.assertEqual(errcount, 0, msg=self.msg) diff --git a/meta/lib/oeqa/runtime/perl.py b/meta/lib/oeqa/runtime/perl.py deleted file mode 100644 index e044d0a5fe..0000000000 --- a/meta/lib/oeqa/runtime/perl.py +++ /dev/null @@ -1,30 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("perl"): - skipModule("No perl package in the image") - - -class PerlTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.pl"), "/tmp/test.pl") - - @testcase(1141) - def test_perl_exists(self): - (status, output) = self.target.run('which perl') - self.assertEqual(status, 0, msg="Perl binary not in PATH or not on target.") - - @testcase(208) - def test_perl_works(self): - (status, output) = self.target.run('perl /tmp/test.pl') - self.assertEqual(status, 0, msg="Exit status was not 0. Output: %s" % output) - self.assertEqual(output, "the value of a is 0.01", msg="Incorrect output: %s" % output) - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run("rm /tmp/test.pl") diff --git a/meta/lib/oeqa/runtime/ping.py b/meta/lib/oeqa/runtime/ping.py deleted file mode 100644 index 0f27447926..0000000000 --- a/meta/lib/oeqa/runtime/ping.py +++ /dev/null @@ -1,22 +0,0 @@ -import subprocess -import unittest -import sys -import time -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * - -class PingTest(oeRuntimeTest): - - @testcase(964) - def test_ping(self): - output = '' - count = 0 - endtime = time.time() + 60 - while count < 5 and time.time() < endtime: - proc = subprocess.Popen("ping -c 1 %s" % self.target.ip, shell=True, stdout=subprocess.PIPE) - output += proc.communicate()[0].decode("utf-8") - if proc.poll() == 0: - count += 1 - else: - count = 0 - self.assertEqual(count, 5, msg = "Expected 5 consecutive replies, got %d.\nping output is:\n%s" % (count,output)) diff --git a/meta/lib/oeqa/runtime/python.py b/meta/lib/oeqa/runtime/python.py deleted file mode 100644 index 26edb7a9b8..0000000000 --- a/meta/lib/oeqa/runtime/python.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("python"): - skipModule("No python package in the image") - - -class PythonTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - oeRuntimeTest.tc.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "test.py"), "/tmp/test.py") - - @testcase(1145) - def test_python_exists(self): - (status, output) = self.target.run('which python') - self.assertEqual(status, 0, msg="Python binary not in PATH or not on target.") - - @testcase(965) - def test_python_stdout(self): - (status, output) = self.target.run('python /tmp/test.py') - self.assertEqual(status, 0, msg="Exit status was not 0. Output: %s" % output) - self.assertEqual(output, "the value of a is 0.01", msg="Incorrect output: %s" % output) - - @testcase(1146) - def test_python_testfile(self): - (status, output) = self.target.run('ls /tmp/testfile.python') - self.assertEqual(status, 0, msg="Python test file generate failed.") - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run("rm /tmp/test.py /tmp/testfile.python") diff --git a/meta/lib/oeqa/runtime/rpm.py b/meta/lib/oeqa/runtime/rpm.py deleted file mode 100644 index 624c515aad..0000000000 --- a/meta/lib/oeqa/runtime/rpm.py +++ /dev/null @@ -1,101 +0,0 @@ -import unittest -import os -import fnmatch -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("package-management"): - skipModule("rpm module skipped: target doesn't have package-management in IMAGE_FEATURES") - if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: - skipModule("rpm module skipped: target doesn't have rpm as primary package manager") - - -class RpmBasicTest(oeRuntimeTest): - - @testcase(960) - @skipUnlessPassed('test_ssh') - def test_rpm_help(self): - (status, output) = self.target.run('rpm --help') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - - @testcase(191) - @skipUnlessPassed('test_rpm_help') - def test_rpm_query(self): - (status, output) = self.target.run('rpm -q rpm') - self.assertEqual(status, 0, msg="status and output: %s and %s" % (status,output)) - -class RpmInstallRemoveTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - pkgarch = oeRuntimeTest.tc.d.getVar('TUNE_PKGARCH', True).replace("-", "_") - rpmdir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), "rpm", pkgarch) - # pick rpm-doc as a test file to get installed, because it's small and it will always be built for standard targets - for f in fnmatch.filter(os.listdir(rpmdir), "rpm-doc-*.%s.rpm" % pkgarch): - testrpmfile = f - oeRuntimeTest.tc.target.copy_to(os.path.join(rpmdir,testrpmfile), "/tmp/rpm-doc.rpm") - - @testcase(192) - @skipUnlessPassed('test_rpm_help') - def test_rpm_install(self): - (status, output) = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') - self.assertEqual(status, 0, msg="Failed to install rpm-doc package: %s" % output) - - @testcase(194) - @skipUnlessPassed('test_rpm_install') - def test_rpm_remove(self): - (status,output) = self.target.run('rpm -e rpm-doc') - self.assertEqual(status, 0, msg="Failed to remove rpm-doc package: %s" % output) - - @testcase(1096) - @skipUnlessPassed('test_ssh') - def test_rpm_query_nonroot(self): - (status, output) = self.target.run('useradd test1') - self.assertTrue(status == 0, msg="Failed to create new user: " + output) - (status, output) = self.target.run('sudo -u test1 id') - self.assertTrue('(test1)' in output, msg="Failed to execute as new user") - (status, output) = self.target.run('sudo -u test1 rpm -qa') - self.assertEqual(status, 0, msg="status: %s. Cannot run rpm -qa: %s" % (status, output)) - - @testcase(195) - @skipUnlessPassed('test_rpm_install') - def test_check_rpm_install_removal_log_file_size(self): - """ - Summary: Check rpm install/removal log file size - Expected: There should be some method to keep rpm log in a small size . - Product: BSPs - Author: Alexandru Georgescu <alexandru.c.georgescu@intel.com> - AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> - """ - db_files_cmd = 'ls /var/lib/rpm/__db.*' - get_log_size_cmd = "du /var/lib/rpm/log/log.* | awk '{print $1}'" - - # Make sure that some database files are under /var/lib/rpm as '__db.xxx' - (status, output) = self.target.run(db_files_cmd) - self.assertEqual(0, status, 'Failed to find database files under /var/lib/rpm/ as __db.xxx') - - # Remove the package just in case - self.target.run('rpm -e rpm-doc') - - # Install/Remove a package 10 times - for i in range(10): - (status, output) = self.target.run('rpm -ivh /tmp/rpm-doc.rpm') - self.assertEqual(0, status, "Failed to install rpm-doc package. Reason: {}".format(output)) - - (status, output) = self.target.run('rpm -e rpm-doc') - self.assertEqual(0, status, "Failed to remove rpm-doc package. Reason: {}".format(output)) - - # Get the size of log file - (status, output) = self.target.run(get_log_size_cmd) - self.assertEqual(0, status, 'Failed to get the final size of the log file.') - - # Compare each log size - for log_file_size in output: - self.assertLessEqual(int(log_file_size), 11264, - 'Log file size is greater that expected (~10MB), found {} bytes'.format(log_file_size)) - - @classmethod - def tearDownClass(self): - oeRuntimeTest.tc.target.run('rm -f /tmp/rpm-doc.rpm') - diff --git a/meta/lib/oeqa/runtime/scanelf.py b/meta/lib/oeqa/runtime/scanelf.py deleted file mode 100644 index 67e02ff455..0000000000 --- a/meta/lib/oeqa/runtime/scanelf.py +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("pax-utils"): - skipModule("pax-utils package not installed") - -class ScanelfTest(oeRuntimeTest): - - def setUpLocal(self): - self.scancmd = 'scanelf --quiet --recursive --mount --ldpath --path' - - @testcase(966) - @skipUnlessPassed('test_ssh') - def test_scanelf_textrel(self): - # print TEXTREL information - self.scancmd += " --textrel" - (status, output) = self.target.run(self.scancmd) - self.assertEqual(output.strip(), "", "\n".join([self.scancmd, output])) - - @testcase(967) - @skipUnlessPassed('test_ssh') - def test_scanelf_rpath(self): - # print RPATH information - self.scancmd += " --rpath" - (status, output) = self.target.run(self.scancmd) - self.assertEqual(output.strip(), "", "\n".join([self.scancmd, output])) diff --git a/meta/lib/oeqa/runtime/scp.py b/meta/lib/oeqa/runtime/scp.py deleted file mode 100644 index 48e87d2d0b..0000000000 --- a/meta/lib/oeqa/runtime/scp.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import skipUnlessPassed, testcase - -def setUpModule(): - if not (oeRuntimeTest.hasPackage("dropbear") or oeRuntimeTest.hasPackage("openssh-sshd")): - skipModule("No ssh package in image") - -class ScpTest(oeRuntimeTest): - - @testcase(220) - @skipUnlessPassed('test_ssh') - def test_scp_file(self): - test_log_dir = oeRuntimeTest.tc.d.getVar("TEST_LOG_DIR", True) - test_file_path = os.path.join(test_log_dir, 'test_scp_file') - with open(test_file_path, 'w') as test_scp_file: - test_scp_file.seek(2 ** 22 - 1) - test_scp_file.write(os.linesep) - (status, output) = self.target.copy_to(test_file_path, '/tmp/test_scp_file') - self.assertEqual(status, 0, msg = "File could not be copied. Output: %s" % output) - (status, output) = self.target.run("ls -la /tmp/test_scp_file") - self.assertEqual(status, 0, msg = "SCP test failed") diff --git a/meta/lib/oeqa/runtime/skeletoninit.py b/meta/lib/oeqa/runtime/skeletoninit.py deleted file mode 100644 index cb0cb9b4cf..0000000000 --- a/meta/lib/oeqa/runtime/skeletoninit.py +++ /dev/null @@ -1,29 +0,0 @@ -# This test should cover https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=284 testcase -# Note that the image under test must have meta-skeleton layer in bblayers and IMAGE_INSTALL_append = " service" in local.conf - -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasPackage("service"): - skipModule("No service package in image") - - -class SkeletonBasicTest(oeRuntimeTest): - - @skipUnlessPassed('test_ssh') - @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", False), "Not appropiate for systemd image") - def test_skeleton_availability(self): - (status, output) = self.target.run('ls /etc/init.d/skeleton') - self.assertEqual(status, 0, msg = "skeleton init script not found. Output:\n%s " % output) - (status, output) = self.target.run('ls /usr/sbin/skeleton-test') - self.assertEqual(status, 0, msg = "skeleton-test not found. Output:\n%s" % output) - - @testcase(284) - @skipUnlessPassed('test_skeleton_availability') - @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", False), "Not appropiate for systemd image") - def test_skeleton_script(self): - output1 = self.target.run("/etc/init.d/skeleton start")[1] - (status, output2) = self.target.run(oeRuntimeTest.pscmd + ' | grep [s]keleton-test') - self.assertEqual(status, 0, msg = "Skeleton script could not be started:\n%s\n%s" % (output1, output2)) diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py deleted file mode 100644 index 126d614638..0000000000 --- a/meta/lib/oeqa/runtime/smart.py +++ /dev/null @@ -1,175 +0,0 @@ -import unittest -import re -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.httpserver import HTTPService - -def setUpModule(): - if not oeRuntimeTest.hasFeature("package-management"): - skipModule("Image doesn't have package management feature") - if not oeRuntimeTest.hasPackage("smart"): - skipModule("Image doesn't have smart installed") - if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: - skipModule("Rpm is not the primary package manager") - -class SmartTest(oeRuntimeTest): - - @skipUnlessPassed('test_smart_help') - def smart(self, command, expected = 0): - command = 'smart %s' % command - status, output = self.target.run(command, 1500) - message = os.linesep.join([command, output]) - self.assertEqual(status, expected, message) - self.assertFalse("Cannot allocate memory" in output, message) - return output - -class SmartBasicTest(SmartTest): - - @testcase(716) - @skipUnlessPassed('test_ssh') - def test_smart_help(self): - self.smart('--help') - - @testcase(968) - def test_smart_version(self): - self.smart('--version') - - @testcase(721) - def test_smart_info(self): - self.smart('info python-smartpm') - - @testcase(421) - def test_smart_query(self): - self.smart('query python-smartpm') - - @testcase(720) - def test_smart_search(self): - self.smart('search python-smartpm') - - @testcase(722) - def test_smart_stats(self): - self.smart('stats') - -class SmartRepoTest(SmartTest): - - @classmethod - def setUpClass(self): - self.repolist = [] - self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip) - self.repo_server.start() - - @classmethod - def tearDownClass(self): - self.repo_server.stop() - for i in self.repolist: - oeRuntimeTest.tc.target.run('smart channel -y --remove '+str(i)) - - @testcase(1143) - def test_smart_channel(self): - self.smart('channel', 1) - - @testcase(719) - def test_smart_channel_add(self): - image_pkgtype = self.tc.d.getVar('IMAGE_PKGTYPE', True) - deploy_url = 'http://%s:%s/%s' %(self.target.server_ip, self.repo_server.port, image_pkgtype) - pkgarchs = self.tc.d.getVar('PACKAGE_ARCHS', True).replace("-","_").split() - for arch in os.listdir('%s/%s' % (self.repo_server.root_dir, image_pkgtype)): - if arch in pkgarchs: - self.smart('channel -y --add {a} type=rpm-md baseurl={u}/{a}'.format(a=arch, u=deploy_url)) - self.repolist.append(arch) - self.smart('update') - - @testcase(969) - def test_smart_channel_help(self): - self.smart('channel --help') - - @testcase(970) - def test_smart_channel_list(self): - self.smart('channel --list') - - @testcase(971) - def test_smart_channel_show(self): - self.smart('channel --show') - - @testcase(717) - def test_smart_channel_rpmsys(self): - self.smart('channel --show rpmsys') - self.smart('channel --disable rpmsys') - self.smart('channel --enable rpmsys') - - @testcase(1144) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_install(self): - self.smart('remove -y psplash-default') - self.smart('install -y psplash-default') - - @testcase(728) - @skipUnlessPassed('test_smart_install') - def test_smart_install_dependency(self): - self.smart('remove -y psplash') - self.smart('install -y psplash-default') - - @testcase(723) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_install_from_disk(self): - self.smart('remove -y psplash-default') - self.smart('download psplash-default') - self.smart('install -y ./psplash-default*') - - @testcase(725) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_install_from_http(self): - output = self.smart('download --urls psplash-default') - url = re.search('(http://.*/psplash-default.*\.rpm)', output) - self.assertTrue(url, msg="Couln't find download url in %s" % output) - self.smart('remove -y psplash-default') - self.smart('install -y %s' % url.group(0)) - - @testcase(729) - @skipUnlessPassed('test_smart_install') - def test_smart_reinstall(self): - self.smart('reinstall -y psplash-default') - - @testcase(727) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_remote_repo(self): - self.smart('update') - self.smart('install -y psplash') - self.smart('remove -y psplash') - - @testcase(726) - def test_smart_local_dir(self): - self.target.run('mkdir /tmp/myrpmdir') - self.smart('channel --add myrpmdir type=rpm-dir path=/tmp/myrpmdir -y') - self.target.run('cd /tmp/myrpmdir') - self.smart('download psplash') - output = self.smart('channel --list') - for i in output.split("\n"): - if ("rpmsys" != str(i)) and ("myrpmdir" != str(i)): - self.smart('channel --disable '+str(i)) - self.target.run('cd $HOME') - self.smart('install psplash') - for i in output.split("\n"): - if ("rpmsys" != str(i)) and ("myrpmdir" != str(i)): - self.smart('channel --enable '+str(i)) - self.smart('channel --remove myrpmdir -y') - self.target.run("rm -rf /tmp/myrpmdir") - - @testcase(718) - def test_smart_add_rpmdir(self): - self.target.run('mkdir /tmp/myrpmdir') - self.smart('channel --add myrpmdir type=rpm-dir path=/tmp/myrpmdir -y') - self.smart('channel --disable myrpmdir -y') - output = self.smart('channel --show myrpmdir') - self.assertTrue("disabled = yes" in output, msg="Failed to disable rpm dir") - self.smart('channel --enable myrpmdir -y') - output = self.smart('channel --show myrpmdir') - self.assertFalse("disabled = yes" in output, msg="Failed to enable rpm dir") - self.smart('channel --remove myrpmdir -y') - self.target.run("rm -rf /tmp/myrpmdir") - - @testcase(731) - @skipUnlessPassed('test_smart_channel_add') - def test_smart_remove_package(self): - self.smart('install -y psplash') - self.smart('remove -y psplash') diff --git a/meta/lib/oeqa/runtime/ssh.py b/meta/lib/oeqa/runtime/ssh.py deleted file mode 100644 index 0e76d5d512..0000000000 --- a/meta/lib/oeqa/runtime/ssh.py +++ /dev/null @@ -1,19 +0,0 @@ -import subprocess -import unittest -import sys -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not (oeRuntimeTest.hasPackage("dropbear") or oeRuntimeTest.hasPackage("openssh")): - skipModule("No ssh package in image") - -class SshTest(oeRuntimeTest): - - @testcase(224) - @skipUnlessPassed('test_ping') - def test_ssh(self): - (status, output) = self.target.run('uname -a') - self.assertEqual(status, 0, msg="SSH Test failed: %s" % output) - (status, output) = self.target.run('cat /etc/masterimage') - self.assertEqual(status, 1, msg="This isn't the right image - /etc/masterimage shouldn't be here %s" % output) diff --git a/meta/lib/oeqa/runtime/syslog.py b/meta/lib/oeqa/runtime/syslog.py deleted file mode 100644 index 2601dd9ea8..0000000000 --- a/meta/lib/oeqa/runtime/syslog.py +++ /dev/null @@ -1,45 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not (oeRuntimeTest.hasPackage("busybox-syslog") or oeRuntimeTest.hasPackage("sysklogd")): - skipModule("No syslog package in image") - -class SyslogTest(oeRuntimeTest): - - @testcase(201) - @skipUnlessPassed("test_syslog_help") - def test_syslog_running(self): - (status,output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -i [s]yslogd') - self.assertEqual(status, 0, msg="no syslogd process, ps output: %s" % self.target.run(oeRuntimeTest.pscmd)[1]) - -class SyslogTestConfig(oeRuntimeTest): - - @testcase(1149) - @skipUnlessPassed("test_syslog_running") - def test_syslog_logger(self): - (status,output) = self.target.run('logger foobar && test -e /var/log/messages && grep foobar /var/log/messages || logread | grep foobar') - self.assertEqual(status, 0, msg="Test log string not found in /var/log/messages. Output: %s " % output) - - @testcase(1150) - @skipUnlessPassed("test_syslog_running") - def test_syslog_restart(self): - if "systemd" != oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", False): - (status,output) = self.target.run('/etc/init.d/syslog restart') - else: - (status,output) = self.target.run('systemctl restart syslog.service') - - @testcase(202) - @skipUnlessPassed("test_syslog_restart") - @skipUnlessPassed("test_syslog_logger") - @unittest.skipIf("systemd" == oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", False), "Not appropiate for systemd image") - @unittest.skipIf(oeRuntimeTest.hasPackage("sysklogd") or not oeRuntimeTest.hasPackage("busybox"), "Non-busybox syslog") - def test_syslog_startup_config(self): - self.target.run('echo "LOGFILE=/var/log/test" >> /etc/syslog-startup.conf') - (status,output) = self.target.run('/etc/init.d/syslog restart') - self.assertEqual(status, 0, msg="Could not restart syslog service. Status and output: %s and %s" % (status,output)) - (status,output) = self.target.run('logger foobar && grep foobar /var/log/test') - self.assertEqual(status, 0, msg="Test log string not found. Output: %s " % output) - self.target.run("sed -i 's#LOGFILE=/var/log/test##' /etc/syslog-startup.conf") - self.target.run('/etc/init.d/syslog restart') diff --git a/meta/lib/oeqa/runtime/utils/__init__.py b/meta/lib/oeqa/runtime/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/runtime/utils/__init__.py diff --git a/meta/lib/oeqa/runtime/utils/targetbuildproject.py b/meta/lib/oeqa/runtime/utils/targetbuildproject.py new file mode 100644 index 0000000000..5af55d736e --- /dev/null +++ b/meta/lib/oeqa/runtime/utils/targetbuildproject.py @@ -0,0 +1,39 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +from oeqa.utils.buildproject import BuildProject + +class TargetBuildProject(BuildProject): + + def __init__(self, target, uri, foldername=None, dl_dir=None): + self.target = target + self.targetdir = "~/" + BuildProject.__init__(self, uri, foldername, dl_dir=dl_dir) + + def download_archive(self): + self._download_archive() + + status, output = self.target.copyTo(self.localarchive, self.targetdir) + if status: + raise Exception('Failed to copy archive to target, ' + 'output: %s' % output) + + cmd = 'tar xf %s%s -C %s' % (self.targetdir, + self.archive, + self.targetdir) + status, output = self.target.run(cmd) + if status: + raise Exception('Failed to extract archive, ' + 'output: %s' % output) + + # Change targetdir to project folder + self.targetdir = self.targetdir + self.fname + + # The timeout parameter of target.run is set to 0 + # to make the ssh command run with no timeout. + def _run(self, cmd): + ret = self.target.run(cmd, 0) + msg = "Command %s failed with exit code %s: %s" % (cmd, ret[0], ret[1]) + if ret[0] != 0: + raise Exception(msg) + return ret[0] diff --git a/meta/lib/oeqa/runtime/x32lib.py b/meta/lib/oeqa/runtime/x32lib.py deleted file mode 100644 index ce5e214035..0000000000 --- a/meta/lib/oeqa/runtime/x32lib.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - #check if DEFAULTTUNE is set and it's value is: x86-64-x32 - defaulttune = oeRuntimeTest.tc.d.getVar("DEFAULTTUNE", True) - if "x86-64-x32" not in defaulttune: - skipModule("DEFAULTTUNE is not set to x86-64-x32") - -class X32libTest(oeRuntimeTest): - - @testcase(281) - @skipUnlessPassed("test_ssh") - def test_x32_file(self): - status1 = self.target.run("readelf -h /bin/ls | grep Class | grep ELF32")[0] - status2 = self.target.run("readelf -h /bin/ls | grep Machine | grep X86-64")[0] - self.assertTrue(status1 == 0 and status2 == 0, msg="/bin/ls isn't an X86-64 ELF32 binary. readelf says: %s" % self.target.run("readelf -h /bin/ls")[1]) diff --git a/meta/lib/oeqa/runtime/xorg.py b/meta/lib/oeqa/runtime/xorg.py deleted file mode 100644 index 12bcd371af..0000000000 --- a/meta/lib/oeqa/runtime/xorg.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeRuntimeTest.hasFeature("x11-base"): - skipModule("target doesn't have x11 in IMAGE_FEATURES") - - -class XorgTest(oeRuntimeTest): - - @testcase(1151) - @skipUnlessPassed('test_ssh') - def test_xorg_running(self): - (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -v xinit | grep [X]org') - self.assertEqual(status, 0, msg="Xorg does not appear to be running %s" % self.target.run(oeRuntimeTest.pscmd)[1]) diff --git a/meta/lib/oeqa/sdk/__init__.py b/meta/lib/oeqa/sdk/__init__.py index 4cf3fa76b6..e69de29bb2 100644 --- a/meta/lib/oeqa/sdk/__init__.py +++ b/meta/lib/oeqa/sdk/__init__.py @@ -1,3 +0,0 @@ -# Enable other layers to have tests in the same named directory -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/sdk/buildcvs.py b/meta/lib/oeqa/sdk/buildcvs.py deleted file mode 100644 index c7146fa4af..0000000000 --- a/meta/lib/oeqa/sdk/buildcvs.py +++ /dev/null @@ -1,25 +0,0 @@ -from oeqa.oetest import oeSDKTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import SDKBuildProject - -class BuildCvsTest(oeSDKTest): - - @classmethod - def setUpClass(self): - self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/cvs/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, - "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2") - self.project.download_archive() - - def test_cvs(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - self.assertEqual(self.project.run_install(), 0, - msg="Running make install failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/sdk/buildiptables.py b/meta/lib/oeqa/sdk/buildiptables.py deleted file mode 100644 index 062e5316e7..0000000000 --- a/meta/lib/oeqa/sdk/buildiptables.py +++ /dev/null @@ -1,26 +0,0 @@ -from oeqa.oetest import oeSDKTest -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import SDKBuildProject - - -class BuildIptablesTest(oeSDKTest): - - @classmethod - def setUpClass(self): - self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/iptables/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, - "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") - self.project.download_archive() - - def test_iptables(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - self.assertEqual(self.project.run_install(), 0, - msg="Running make install failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/sdk/buildsudoku.py b/meta/lib/oeqa/sdk/buildsudoku.py deleted file mode 100644 index dea77c6599..0000000000 --- a/meta/lib/oeqa/sdk/buildsudoku.py +++ /dev/null @@ -1,26 +0,0 @@ -from oeqa.oetest import oeSDKTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import SDKBuildProject - -def setUpModule(): - if not oeSDKTest.hasPackage("gtk\+"): - skipModule("Image doesn't have gtk+ in manifest") - -class SudokuTest(oeSDKTest): - - @classmethod - def setUpClass(self): - self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/sudoku/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, - "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") - self.project.download_archive() - - def test_sudoku(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/meta/lib/oeqa/sdk/case.py b/meta/lib/oeqa/sdk/case.py new file mode 100644 index 0000000000..963aa8d358 --- /dev/null +++ b/meta/lib/oeqa/sdk/case.py @@ -0,0 +1,12 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import subprocess + +from oeqa.core.case import OETestCase + +class OESDKTestCase(OETestCase): + def _run(self, cmd): + return subprocess.check_output(". %s > /dev/null; %s;" % \ + (self.tc.sdk_env, cmd), shell=True, + stderr=subprocess.STDOUT, universal_newlines=True) diff --git a/meta/lib/oeqa/sdk/cases/buildcpio.py b/meta/lib/oeqa/sdk/cases/buildcpio.py new file mode 100644 index 0000000000..333dc7c226 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/buildcpio.py @@ -0,0 +1,33 @@ +import unittest +from oeqa.sdk.case import OESDKTestCase +from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject + +class BuildCpioTest(OESDKTestCase): + td_vars = ['DATETIME'] + + @classmethod + def setUpClass(self): + dl_dir = self.td.get('DL_DIR', None) + + self.project = SDKBuildProject(self.tc.sdk_dir + "/cpio/", self.tc.sdk_env, + "https://ftp.gnu.org/gnu/cpio/cpio-2.12.tar.gz", + self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir) + self.project.download_archive() + + machine = self.td.get("MACHINE") + if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine): + raise unittest.SkipTest("SDK doesn't contain a cross-canadian toolchain") + + def test_cpio(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + self.assertEqual(self.project.run_install(), 0, + msg="Running make install failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/sdk/cases/buildgalculator.py b/meta/lib/oeqa/sdk/cases/buildgalculator.py new file mode 100644 index 0000000000..42e8ddb185 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/buildgalculator.py @@ -0,0 +1,35 @@ +import unittest + +from oeqa.sdk.case import OESDKTestCase +from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject + +class GalculatorTest(OESDKTestCase): + td_vars = ['DATETIME'] + + @classmethod + def setUpClass(self): + if not (self.tc.hasTargetPackage("gtk+3") or\ + self.tc.hasTargetPackage("libgtk-3.0")): + raise unittest.SkipTest("GalculatorTest class: SDK don't support gtk+3") + + def test_galculator(self): + dl_dir = self.td.get('DL_DIR', None) + project = None + try: + project = SDKBuildProject(self.tc.sdk_dir + "/galculator/", + self.tc.sdk_env, + "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2", + self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir) + + project.download_archive() + + # regenerate configure to get support for --with-libtool-sysroot + legacy_preconf=("autoreconf -i -f -I ${OECORE_TARGET_SYSROOT}/usr/share/aclocal -I m4;") + + self.assertEqual(project.run_configure(extra_cmds=legacy_preconf), + 0, msg="Running configure failed") + + self.assertEqual(project.run_make(), 0, + msg="Running make failed") + finally: + project.clean() diff --git a/meta/lib/oeqa/sdk/cases/buildiptables.py b/meta/lib/oeqa/sdk/cases/buildiptables.py new file mode 100644 index 0000000000..0bd00d125a --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/buildiptables.py @@ -0,0 +1,35 @@ +import unittest +from oeqa.sdk.case import OESDKTestCase +from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject + + +class BuildIptablesTest(OESDKTestCase): + td_vars = ['DATETIME'] + + @classmethod + def setUpClass(self): + dl_dir = self.td.get('DL_DIR', None) + + self.project = SDKBuildProject(self.tc.sdk_dir + "/iptables/", self.tc.sdk_env, + "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2", + self.tc.sdk_dir, self.td['DATETIME'], dl_dir=dl_dir) + self.project.download_archive() + + machine = self.td.get("MACHINE") + + if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine): + raise unittest.SkipTest("SDK doesn't contain a cross-canadian toolchain") + + def test_iptables(self): + self.assertEqual(self.project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(self.project.run_make(), 0, + msg="Running make failed") + + self.assertEqual(self.project.run_install(), 0, + msg="Running make install failed") + + @classmethod + def tearDownClass(self): + self.project.clean() diff --git a/meta/lib/oeqa/sdk/cases/gcc.py b/meta/lib/oeqa/sdk/cases/gcc.py new file mode 100644 index 0000000000..74ad2a2f2b --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/gcc.py @@ -0,0 +1,42 @@ +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase + +class GccCompileTest(OESDKTestCase): + td_vars = ['MACHINE'] + + @classmethod + def setUpClass(self): + files = {'test.c' : self.tc.files_dir, 'test.cpp' : self.tc.files_dir, + 'testsdkmakefile' : self.tc.sdk_files_dir} + for f in files: + shutil.copyfile(os.path.join(files[f], f), + os.path.join(self.tc.sdk_dir, f)) + + def setUp(self): + machine = self.td.get("MACHINE") + if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine): + raise unittest.SkipTest("GccCompileTest class: SDK doesn't contain a cross-canadian toolchain") + + def test_gcc_compile(self): + self._run('$CC %s/test.c -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir)) + + def test_gpp_compile(self): + self._run('$CXX %s/test.c -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir)) + + def test_gpp2_compile(self): + self._run('$CXX %s/test.cpp -o %s/test -lm' % (self.tc.sdk_dir, self.tc.sdk_dir)) + + def test_make(self): + self._run('cd %s; make -f testsdkmakefile' % self.tc.sdk_dir) + + @classmethod + def tearDownClass(self): + files = [os.path.join(self.tc.sdk_dir, f) \ + for f in ['test.c', 'test.cpp', 'test.o', 'test', + 'testsdkmakefile']] + for f in files: + remove_safe(f) diff --git a/meta/lib/oeqa/sdk/cases/perl.py b/meta/lib/oeqa/sdk/cases/perl.py new file mode 100644 index 0000000000..e1bded2ff2 --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/perl.py @@ -0,0 +1,27 @@ +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase + +class PerlTest(OESDKTestCase): + @classmethod + def setUpClass(self): + if not self.tc.hasHostPackage("nativesdk-perl"): + raise unittest.SkipTest("No perl package in the SDK") + + for f in ['test.pl']: + shutil.copyfile(os.path.join(self.tc.files_dir, f), + os.path.join(self.tc.sdk_dir, f)) + self.testfile = os.path.join(self.tc.sdk_dir, "test.pl") + + def test_perl_exists(self): + self._run('which perl') + + def test_perl_works(self): + self._run('perl %s' % self.testfile) + + @classmethod + def tearDownClass(self): + remove_safe(self.testfile) diff --git a/meta/lib/oeqa/sdk/cases/python.py b/meta/lib/oeqa/sdk/cases/python.py new file mode 100644 index 0000000000..94a296f0ec --- /dev/null +++ b/meta/lib/oeqa/sdk/cases/python.py @@ -0,0 +1,31 @@ +import os +import shutil +import unittest + +from oeqa.core.utils.path import remove_safe +from oeqa.sdk.case import OESDKTestCase + +class PythonTest(OESDKTestCase): + @classmethod + def setUpClass(self): + if not self.tc.hasHostPackage("nativesdk-python"): + raise unittest.SkipTest("No python package in the SDK") + + for f in ['test.py']: + shutil.copyfile(os.path.join(self.tc.files_dir, f), + os.path.join(self.tc.sdk_dir, f)) + + def test_python_exists(self): + self._run('which python') + + def test_python_stdout(self): + output = self._run('python %s/test.py' % self.tc.sdk_dir) + self.assertEqual(output.strip(), "the value of a is 0.01", msg="Incorrect output: %s" % output) + + def test_python_testfile(self): + self._run('ls /tmp/testfile.python') + + @classmethod + def tearDownClass(self): + remove_safe("%s/test.py" % self.tc.sdk_dir) + remove_safe("/tmp/testfile.python") diff --git a/meta/lib/oeqa/sdk/context.py b/meta/lib/oeqa/sdk/context.py new file mode 100644 index 0000000000..0189ed851e --- /dev/null +++ b/meta/lib/oeqa/sdk/context.py @@ -0,0 +1,133 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import sys +import glob +import re + +from oeqa.core.context import OETestContext, OETestContextExecutor + +class OESDKTestContext(OETestContext): + sdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + + def __init__(self, td=None, logger=None, sdk_dir=None, sdk_env=None, + target_pkg_manifest=None, host_pkg_manifest=None): + super(OESDKTestContext, self).__init__(td, logger) + + self.sdk_dir = sdk_dir + self.sdk_env = sdk_env + self.target_pkg_manifest = target_pkg_manifest + self.host_pkg_manifest = host_pkg_manifest + + def _hasPackage(self, manifest, pkg): + for host_pkg in manifest.keys(): + if re.search(pkg, host_pkg): + return True + return False + + def hasHostPackage(self, pkg): + return self._hasPackage(self.host_pkg_manifest, pkg) + + def hasTargetPackage(self, pkg): + return self._hasPackage(self.target_pkg_manifest, pkg) + +class OESDKTestContextExecutor(OETestContextExecutor): + _context_class = OESDKTestContext + + name = 'sdk' + help = 'sdk test component' + description = 'executes sdk tests' + + default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'cases')] + default_test_data = None + + def register_commands(self, logger, subparsers): + import argparse_oe + + super(OESDKTestContextExecutor, self).register_commands(logger, subparsers) + + sdk_group = self.parser.add_argument_group('sdk options') + sdk_group.add_argument('--sdk-env', action='store', + help='sdk environment') + sdk_group.add_argument('--target-manifest', action='store', + help='sdk target manifest') + sdk_group.add_argument('--host-manifest', action='store', + help='sdk host manifest') + + sdk_dgroup = self.parser.add_argument_group('sdk display options') + sdk_dgroup.add_argument('--list-sdk-env', action='store_true', + default=False, help='sdk list available environment') + + # XXX this option is required but argparse_oe has a bug handling + # required options, seems that don't keep track of already parsed + # options + sdk_rgroup = self.parser.add_argument_group('sdk required options') + sdk_rgroup.add_argument('--sdk-dir', required=False, action='store', + help='sdk installed directory') + + @staticmethod + def _load_manifest(manifest): + pkg_manifest = {} + if manifest: + with open(manifest) as f: + for line in f: + (pkg, arch, version) = line.strip().split() + pkg_manifest[pkg] = (version, arch) + + return pkg_manifest + + def _process_args(self, logger, args): + super(OESDKTestContextExecutor, self)._process_args(logger, args) + + self.tc_kwargs['init']['sdk_dir'] = args.sdk_dir + self.tc_kwargs['init']['sdk_env'] = self.sdk_env + self.tc_kwargs['init']['target_pkg_manifest'] = \ + OESDKTestContextExecutor._load_manifest(args.target_manifest) + self.tc_kwargs['init']['host_pkg_manifest'] = \ + OESDKTestContextExecutor._load_manifest(args.host_manifest) + + @staticmethod + def _get_sdk_environs(sdk_dir): + sdk_env = {} + + environ_pattern = sdk_dir + '/environment-setup-*' + full_sdk_env = glob.glob(sdk_dir + '/environment-setup-*') + for env in full_sdk_env: + m = re.search('environment-setup-(.*)', env) + if m: + sdk_env[m.group(1)] = env + + return sdk_env + + def _display_sdk_envs(self, log, args, sdk_envs): + log("Available SDK environments at directory %s:" \ + % args.sdk_dir) + log("") + for env in sdk_envs: + log(env) + + def run(self, logger, args): + if not args.sdk_dir: + raise argparse_oe.ArgumentUsageError("No SDK directory "\ + "specified please do, --sdk-dir SDK_DIR", self.name) + + sdk_envs = OESDKTestContextExecutor._get_sdk_environs(args.sdk_dir) + if not sdk_envs: + raise argparse_oe.ArgumentUsageError("No available SDK "\ + "enviroments found at %s" % args.sdk_dir, self.name) + + if args.list_sdk_env: + self._display_sdk_envs(logger.info, args, sdk_envs) + sys.exit(0) + + if not args.sdk_env in sdk_envs: + self._display_sdk_envs(logger.error, args, sdk_envs) + raise argparse_oe.ArgumentUsageError("No valid SDK "\ + "environment (%s) specified" % args.sdk_env, self.name) + + self.sdk_env = sdk_envs[args.sdk_env] + super(OESDKTestContextExecutor, self).run(logger, args) + +_executor_class = OESDKTestContextExecutor diff --git a/meta/lib/oeqa/runtime/files/testsdkmakefile b/meta/lib/oeqa/sdk/files/testsdkmakefile index fb05f822f3..fb05f822f3 100644 --- a/meta/lib/oeqa/runtime/files/testsdkmakefile +++ b/meta/lib/oeqa/sdk/files/testsdkmakefile diff --git a/meta/lib/oeqa/sdk/gcc.py b/meta/lib/oeqa/sdk/gcc.py deleted file mode 100644 index 8395b9b908..0000000000 --- a/meta/lib/oeqa/sdk/gcc.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest -import os -import shutil -from oeqa.oetest import oeSDKTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - machine = oeSDKTest.tc.d.getVar("MACHINE", True) - if not oeSDKTest.hasHostPackage("packagegroup-cross-canadian-" + machine): - skipModule("SDK doesn't contain a cross-canadian toolchain") - - -class GccCompileTest(oeSDKTest): - - @classmethod - def setUpClass(self): - for f in ['test.c', 'test.cpp', 'testsdkmakefile']: - shutil.copyfile(os.path.join(self.tc.filesdir, f), self.tc.sdktestdir + f) - - def test_gcc_compile(self): - self._run('$CC %s/test.c -o %s/test -lm' % (self.tc.sdktestdir, self.tc.sdktestdir)) - - def test_gpp_compile(self): - self._run('$CXX %s/test.c -o %s/test -lm' % (self.tc.sdktestdir, self.tc.sdktestdir)) - - def test_gpp2_compile(self): - self._run('$CXX %s/test.cpp -o %s/test -lm' % (self.tc.sdktestdir, self.tc.sdktestdir)) - - def test_make(self): - self._run('cd %s; make -f testsdkmakefile' % self.tc.sdktestdir) - - @classmethod - def tearDownClass(self): - files = [self.tc.sdktestdir + f for f in ['test.c', 'test.cpp', 'test.o', 'test', 'testsdkmakefile']] - for f in files: - bb.utils.remove(f) diff --git a/meta/lib/oeqa/sdk/perl.py b/meta/lib/oeqa/sdk/perl.py deleted file mode 100644 index 45f422ef0b..0000000000 --- a/meta/lib/oeqa/sdk/perl.py +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -import os -import shutil -from oeqa.oetest import oeSDKTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeSDKTest.hasHostPackage("nativesdk-perl"): - skipModule("No perl package in the SDK") - - -class PerlTest(oeSDKTest): - - @classmethod - def setUpClass(self): - for f in ['test.pl']: - shutil.copyfile(os.path.join(self.tc.filesdir, f), self.tc.sdktestdir + f) - self.testfile = self.tc.sdktestdir + "test.pl" - - def test_perl_exists(self): - self._run('which perl') - - def test_perl_works(self): - self._run('perl %s/test.pl' % self.tc.sdktestdir) - - @classmethod - def tearDownClass(self): - bb.utils.remove("%s/test.pl" % self.tc.sdktestdir) diff --git a/meta/lib/oeqa/sdk/python.py b/meta/lib/oeqa/sdk/python.py deleted file mode 100644 index 896fab4dfb..0000000000 --- a/meta/lib/oeqa/sdk/python.py +++ /dev/null @@ -1,32 +0,0 @@ -import unittest -import os -import shutil -from oeqa.oetest import oeSDKTest, skipModule -from oeqa.utils.decorators import * - -def setUpModule(): - if not oeSDKTest.hasHostPackage("nativesdk-python"): - skipModule("No python package in the SDK") - - -class PythonTest(oeSDKTest): - - @classmethod - def setUpClass(self): - for f in ['test.py']: - shutil.copyfile(os.path.join(self.tc.filesdir, f), self.tc.sdktestdir + f) - - def test_python_exists(self): - self._run('which python') - - def test_python_stdout(self): - output = self._run('python %s/test.py' % self.tc.sdktestdir) - self.assertEqual(output.strip(), "the value of a is 0.01", msg="Incorrect output: %s" % output) - - def test_python_testfile(self): - self._run('ls /tmp/testfile.python') - - @classmethod - def tearDownClass(self): - bb.utils.remove("%s/test.py" % self.tc.sdktestdir) - bb.utils.remove("/tmp/testfile.python") diff --git a/meta/lib/oeqa/sdk/utils/__init__.py b/meta/lib/oeqa/sdk/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/sdk/utils/__init__.py diff --git a/meta/lib/oeqa/sdk/utils/sdkbuildproject.py b/meta/lib/oeqa/sdk/utils/sdkbuildproject.py new file mode 100644 index 0000000000..4e251142d7 --- /dev/null +++ b/meta/lib/oeqa/sdk/utils/sdkbuildproject.py @@ -0,0 +1,45 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import subprocess + +from oeqa.utils.buildproject import BuildProject + +class SDKBuildProject(BuildProject): + def __init__(self, testpath, sdkenv, uri, testlogdir, builddatetime, + foldername=None, dl_dir=None): + self.sdkenv = sdkenv + self.testdir = testpath + self.targetdir = testpath + os.makedirs(testpath, exist_ok=True) + self.datetime = builddatetime + self.testlogdir = testlogdir + os.makedirs(self.testlogdir, exist_ok=True) + self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime) + BuildProject.__init__(self, uri, foldername, tmpdir=testpath, dl_dir=dl_dir) + + def download_archive(self): + + self._download_archive() + + cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir) + subprocess.check_output(cmd, shell=True) + + #Change targetdir to project folder + self.targetdir = os.path.join(self.targetdir, self.fname) + + def run_configure(self, configure_args='', extra_cmds=''): + return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=extra_cmds) + + def run_install(self, install_args=''): + return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir)) + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s\n" % msg) + + def _run(self, cmd): + self.log("Running . %s; " % self.sdkenv + cmd) + return subprocess.call(". %s; " % self.sdkenv + cmd, shell=True) diff --git a/meta/lib/oeqa/sdkext/__init__.py b/meta/lib/oeqa/sdkext/__init__.py index 4cf3fa76b6..e69de29bb2 100644 --- a/meta/lib/oeqa/sdkext/__init__.py +++ b/meta/lib/oeqa/sdkext/__init__.py @@ -1,3 +0,0 @@ -# Enable other layers to have tests in the same named directory -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/sdkext/case.py b/meta/lib/oeqa/sdkext/case.py new file mode 100644 index 0000000000..21b718831c --- /dev/null +++ b/meta/lib/oeqa/sdkext/case.py @@ -0,0 +1,21 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +import subprocess + +from oeqa.utils import avoid_paths_in_environ +from oeqa.sdk.case import OESDKTestCase + +class OESDKExtTestCase(OESDKTestCase): + def _run(self, cmd): + # extensible sdk shows a warning if found bitbake in the path + # because can cause contamination, i.e. use devtool from + # poky/scripts instead of eSDK one. + env = os.environ.copy() + paths_to_avoid = ['bitbake/bin', 'poky/scripts'] + env['PATH'] = avoid_paths_in_environ(paths_to_avoid) + + return subprocess.check_output(". %s > /dev/null;"\ + " %s;" % (self.tc.sdk_env, cmd), stderr=subprocess.STDOUT, + shell=True, env=env, universal_newlines=True) diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py new file mode 100644 index 0000000000..a01bc0bfe2 --- /dev/null +++ b/meta/lib/oeqa/sdkext/cases/devtool.py @@ -0,0 +1,97 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import shutil +import subprocess + +from oeqa.sdkext.case import OESDKExtTestCase +from oeqa.core.decorator.depends import OETestDepends +from oeqa.core.decorator.oeid import OETestID + +class DevtoolTest(OESDKExtTestCase): + @classmethod + def setUpClass(cls): + myapp_src = os.path.join(cls.tc.esdk_files_dir, "myapp") + cls.myapp_dst = os.path.join(cls.tc.sdk_dir, "myapp") + shutil.copytree(myapp_src, cls.myapp_dst) + + myapp_cmake_src = os.path.join(cls.tc.esdk_files_dir, "myapp_cmake") + cls.myapp_cmake_dst = os.path.join(cls.tc.sdk_dir, "myapp_cmake") + shutil.copytree(myapp_cmake_src, cls.myapp_cmake_dst) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.myapp_dst) + shutil.rmtree(cls.myapp_cmake_dst) + + def _test_devtool_build(self, directory): + self._run('devtool add myapp %s' % directory) + try: + self._run('devtool build myapp') + finally: + self._run('devtool reset myapp') + + def _test_devtool_build_package(self, directory): + self._run('devtool add myapp %s' % directory) + try: + self._run('devtool package myapp') + finally: + self._run('devtool reset myapp') + + def test_devtool_location(self): + output = self._run('which devtool') + self.assertEqual(output.startswith(self.tc.sdk_dir), True, \ + msg="Seems that devtool isn't the eSDK one: %s" % output) + + @OETestDepends(['test_devtool_location']) + def test_devtool_add_reset(self): + self._run('devtool add myapp %s' % self.myapp_dst) + self._run('devtool reset myapp') + + @OETestID(1605) + @OETestDepends(['test_devtool_location']) + def test_devtool_build_make(self): + self._test_devtool_build(self.myapp_dst) + + @OETestID(1606) + @OETestDepends(['test_devtool_location']) + def test_devtool_build_esdk_package(self): + self._test_devtool_build_package(self.myapp_dst) + + @OETestID(1607) + @OETestDepends(['test_devtool_location']) + def test_devtool_build_cmake(self): + self._test_devtool_build(self.myapp_cmake_dst) + + @OETestID(1608) + @OETestDepends(['test_devtool_location']) + def test_extend_autotools_recipe_creation(self): + req = 'https://github.com/rdfa/librdfa' + recipe = "librdfa" + self._run('devtool sdk-install libxml2') + self._run('devtool add %s %s' % (recipe, req) ) + try: + self._run('devtool build %s' % recipe) + finally: + self._run('devtool reset %s' % recipe) + + @OETestID(1609) + @OETestDepends(['test_devtool_location']) + def test_devtool_kernelmodule(self): + docfile = 'https://github.com/umlaeute/v4l2loopback.git' + recipe = 'v4l2loopback-driver' + self._run('devtool add %s %s' % (recipe, docfile) ) + try: + self._run('devtool build %s' % recipe) + finally: + self._run('devtool reset %s' % recipe) + + @OETestID(1610) + @OETestDepends(['test_devtool_location']) + def test_recipes_for_nodejs(self): + package_nodejs = "npm://registry.npmjs.org;name=winston;version=2.2.0" + self._run('devtool add %s ' % package_nodejs) + try: + self._run('devtool build %s ' % package_nodejs) + finally: + self._run('devtool reset %s '% package_nodejs) diff --git a/meta/lib/oeqa/sdkext/sdk_update.py b/meta/lib/oeqa/sdkext/cases/sdk_update.py index 2ade839c05..2f8598bbe5 100644 --- a/meta/lib/oeqa/sdkext/sdk_update.py +++ b/meta/lib/oeqa/sdkext/cases/sdk_update.py @@ -1,23 +1,26 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + import os import shutil import subprocess -from oeqa.oetest import oeSDKExtTest +from oeqa.sdkext.case import OESDKExtTestCase from oeqa.utils.httpserver import HTTPService -class SdkUpdateTest(oeSDKExtTest): - +class SdkUpdateTest(OESDKExtTestCase): @classmethod def setUpClass(self): - self.publish_dir = os.path.join(self.tc.sdktestdir, 'esdk_publish') + self.publish_dir = os.path.join(self.tc.sdk_dir, 'esdk_publish') if os.path.exists(self.publish_dir): shutil.rmtree(self.publish_dir) os.mkdir(self.publish_dir) - tcname_new = self.tc.d.expand( - "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}-new.sh") + base_tcname = "%s/%s" % (self.td.get("SDK_DEPLOY", ''), + self.td.get("TOOLCHAINEXT_OUTPUTNAME", '')) + tcname_new = "%s-new.sh" % base_tcname if not os.path.exists(tcname_new): - tcname_new = self.tc.tcname + tcname_new = "%s.sh" % base_tcname cmd = 'oe-publish-sdk %s %s' % (tcname_new, self.publish_dir) subprocess.check_output(cmd, shell=True) diff --git a/meta/lib/oeqa/sdkext/context.py b/meta/lib/oeqa/sdkext/context.py new file mode 100644 index 0000000000..65da4c6e1b --- /dev/null +++ b/meta/lib/oeqa/sdkext/context.py @@ -0,0 +1,29 @@ +# Copyright (C) 2016 Intel Corporation +# Released under the MIT license (see COPYING.MIT) + +import os +from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor + +class OESDKExtTestContext(OESDKTestContext): + esdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") + + # FIXME - We really need to do better mapping of names here, this at + # least allows some tests to run + def hasHostPackage(self, pkg): + # We force a toolchain to be installed into the eSDK even if its minimal + if pkg.startswith("packagegroup-cross-canadian-"): + return True + return self._hasPackage(self.host_pkg_manifest, pkg) + +class OESDKExtTestContextExecutor(OESDKTestContextExecutor): + _context_class = OESDKExtTestContext + + name = 'esdk' + help = 'esdk test component' + description = 'executes esdk tests' + + default_cases = OESDKTestContextExecutor.default_cases + \ + [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'cases')] + default_test_data = None + +_executor_class = OESDKExtTestContextExecutor diff --git a/meta/lib/oeqa/sdkext/devtool.py b/meta/lib/oeqa/sdkext/devtool.py deleted file mode 100644 index c5bb3102a6..0000000000 --- a/meta/lib/oeqa/sdkext/devtool.py +++ /dev/null @@ -1,32 +0,0 @@ -import shutil - -from oeqa.oetest import oeSDKExtTest -from oeqa.utils.decorators import * - -class DevtoolTest(oeSDKExtTest): - - @classmethod - def setUpClass(self): - self.myapp_src = os.path.join(self.tc.sdkextfilesdir, "myapp") - self.myapp_dst = os.path.join(self.tc.sdktestdir, "myapp") - shutil.copytree(self.myapp_src, self.myapp_dst) - - def test_devtool_location(self): - output = self._run('which devtool') - self.assertEqual(output.startswith(self.tc.sdktestdir), True, \ - msg="Seems that devtool isn't the eSDK one: %s" % output) - - @skipUnlessPassed('test_devtool_location') - def test_devtool_add_reset(self): - self._run('devtool add myapp %s' % self.myapp_dst) - self._run('devtool reset myapp') - - @skipUnlessPassed('test_devtool_location') - def test_devtool_build(self): - self._run('devtool add myapp %s' % self.myapp_dst) - self._run('devtool build myapp') - self._run('devtool reset myapp') - - @classmethod - def tearDownClass(self): - shutil.rmtree(self.myapp_dst) diff --git a/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt b/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt new file mode 100644 index 0000000000..19d773dd63 --- /dev/null +++ b/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required (VERSION 2.6) +project (myapp) +# The version number. +set (myapp_VERSION_MAJOR 1) +set (myapp_VERSION_MINOR 0) + +# add the executable +add_executable (myapp myapp.c) + +install(TARGETS myapp + RUNTIME DESTINATION bin) diff --git a/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c b/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c new file mode 100644 index 0000000000..f0b63f03f3 --- /dev/null +++ b/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c @@ -0,0 +1,9 @@ +#include <stdio.h> + +int +main(int argc, char *argv[]) +{ + printf("Hello world\n"); + + return 0; +} diff --git a/meta/lib/oeqa/selftest/_toaster.py b/meta/lib/oeqa/selftest/_toaster.py deleted file mode 100644 index 15ea9df9ef..0000000000 --- a/meta/lib/oeqa/selftest/_toaster.py +++ /dev/null @@ -1,320 +0,0 @@ -import unittest -import os -import sys -import shlex, subprocess -import urllib.request, urllib.parse, urllib.error, subprocess, time, getpass, re, json, shlex - -import oeqa.utils.ftools as ftools -from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../', 'bitbake/lib/toaster'))) -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings") - -import toastermain.settings -from django.db.models import Q -from orm.models import * -from oeqa.utils.decorators import testcase - -class ToasterSetup(oeSelfTest): - - def recipe_parse(self, file_path, var): - for line in open(file_path,'r'): - if line.find(var) > -1: - val = line.split(" = ")[1].replace("\"", "").strip() - return val - - def fix_file_path(self, file_path): - if ":" in file_path: - file_path=file_path.split(":")[2] - return file_path - -class Toaster_DB_Tests(ToasterSetup): - - # Check if build name is unique - tc_id=795 - @testcase(795) - def test_Build_Unique_Name(self): - all_builds = Build.objects.all().count() - distinct_builds = Build.objects.values('id').distinct().count() - self.assertEqual(distinct_builds, all_builds, msg = 'Build name is not unique') - - # Check if build coocker log path is unique - tc_id=819 - @testcase(819) - def test_Build_Unique_Cooker_Log_Path(self): - distinct_path = Build.objects.values('cooker_log_path').distinct().count() - total_builds = Build.objects.values('id').count() - self.assertEqual(distinct_path, total_builds, msg = 'Build coocker log path is not unique') - - # Check if task order is unique for one build - tc=824 - @testcase(824) - def test_Task_Unique_Order(self): - builds = Build.objects.values('id') - cnt_err = [] - for build in builds: - total_task_order = Task.objects.filter(build = build['id']).values('order').count() - distinct_task_order = Task.objects.filter(build = build['id']).values('order').distinct().count() - if (total_task_order != distinct_task_order): - cnt_err.append(build['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for build id: %s' % cnt_err) - - # Check task order sequence for one build - tc=825 - @testcase(825) - def test_Task_Order_Sequence(self): - builds = builds = Build.objects.values('id') - cnt_err = [] - for build in builds: - tasks = Task.objects.filter(Q(build = build['id']), ~Q(order = None), ~Q(task_name__contains = '_setscene')).values('id', 'order').order_by("order") - cnt_tasks = 0 - for task in tasks: - cnt_tasks += 1 - if (task['order'] != cnt_tasks): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if disk_io matches the difference between EndTimeIO and StartTimeIO in build stats - tc=828 - ### this needs to be updated ### - #def test_Task_Disk_IO_TC828(self): - - # Check if outcome = 2 (SSTATE) then sstate_result must be 3 (RESTORED) - tc=832 - @testcase(832) - def test_Task_If_Outcome_2_Sstate_Result_Must_Be_3(self): - tasks = Task.objects.filter(outcome = 2).values('id', 'sstate_result') - cnt_err = [] - for task in tasks: - if (row['sstate_result'] != 3): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if outcome = 1 (COVERED) or 3 (EXISTING) then sstate_result must be 0 (SSTATE_NA) - tc=833 - @testcase(833) - def test_Task_If_Outcome_1_3_Sstate_Result_Must_Be_0(self): - tasks = Task.objects.filter(outcome__in = (1, 3)).values('id', 'sstate_result') - cnt_err = [] - for task in tasks: - if (task['sstate_result'] != 0): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if outcome is 0 (SUCCESS) or 4 (FAILED) then sstate_result must be 0 (NA), 1 (MISS) or 2 (FAILED) - tc=834 - @testcase(834) - def test_Task_If_Outcome_0_4_Sstate_Result_Must_Be_0_1_2(self): - tasks = Task.objects.filter(outcome__in = (0, 4)).values('id', 'sstate_result') - cnt_err = [] - for task in tasks: - if (task['sstate_result'] not in [0, 1, 2]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = TRUE (1), script_type must be 0 (CODING_NA), 2 (CODING_PYTHON), 3 (CODING_SHELL) - tc=891 - @testcase(891) - def test_Task_If_Task_Executed_True_Script_Type_0_2_3(self): - tasks = Task.objects.filter(task_executed = 1).values('id', 'script_type') - cnt_err = [] - for task in tasks: - if (task['script_type'] not in [0, 2, 3]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = TRUE (1), outcome must be 0 (SUCCESS) or 4 (FAILED) - tc=836 - @testcase(836) - def test_Task_If_Task_Executed_True_Outcome_0_4(self): - tasks = Task.objects.filter(task_executed = 1).values('id', 'outcome') - cnt_err = [] - for task in tasks: - if (task['outcome'] not in [0, 4]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = FALSE (0), script_type must be 0 - tc=890 - @testcase(890) - def test_Task_If_Task_Executed_False_Script_Type_0(self): - tasks = Task.objects.filter(task_executed = 0).values('id', 'script_type') - cnt_err = [] - for task in tasks: - if (task['script_type'] != 0): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Check if task_executed = FALSE (0) and build outcome = SUCCEEDED (0), task outcome must be 1 (COVERED), 2 (CACHED), 3 (PREBUILT), 5 (EMPTY) - tc=837 - @testcase(837) - def test_Task_If_Task_Executed_False_Outcome_1_2_3_5(self): - builds = Build.objects.filter(outcome = 0).values('id') - cnt_err = [] - for build in builds: - tasks = Task.objects.filter(build = build['id'], task_executed = 0).values('id', 'outcome') - for task in tasks: - if (task['outcome'] not in [1, 2, 3, 5]): - cnt_err.append(task['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task id: %s' % cnt_err) - - # Key verification - tc=888 - @testcase(888) - def test_Target_Installed_Package(self): - rows = Target_Installed_Package.objects.values('id', 'target_id', 'package_id') - cnt_err = [] - for row in rows: - target = Target.objects.filter(id = row['target_id']).values('id') - package = Package.objects.filter(id = row['package_id']).values('id') - if (not target or not package): - cnt_err.append(row['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for target installed package id: %s' % cnt_err) - - # Key verification - tc=889 - @testcase(889) - def test_Task_Dependency(self): - rows = Task_Dependency.objects.values('id', 'task_id', 'depends_on_id') - cnt_err = [] - for row in rows: - task_id = Task.objects.filter(id = row['task_id']).values('id') - depends_on_id = Task.objects.filter(id = row['depends_on_id']).values('id') - if (not task_id or not depends_on_id): - cnt_err.append(row['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for task dependency id: %s' % cnt_err) - - # Check if build target file_name is populated only if is_image=true AND orm_build.outcome=0 then if the file exists and its size matches the file_size value - ### Need to add the tc in the test run - @testcase(1037) - def test_Target_File_Name_Populated(self): - builds = Build.objects.filter(outcome = 0).values('id') - for build in builds: - targets = Target.objects.filter(build_id = build['id'], is_image = 1).values('id') - for target in targets: - target_files = Target_Image_File.objects.filter(target_id = target['id']).values('id', 'file_name', 'file_size') - cnt_err = [] - for file_info in target_files: - target_id = file_info['id'] - target_file_name = file_info['file_name'] - target_file_size = file_info['file_size'] - if (not target_file_name or not target_file_size): - cnt_err.append(target_id) - else: - if (not os.path.exists(target_file_name)): - cnt_err.append(target_id) - else: - if (os.path.getsize(target_file_name) != target_file_size): - cnt_err.append(target_id) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for target image file id: %s' % cnt_err) - - # Key verification - tc=884 - @testcase(884) - def test_Package_Dependency(self): - cnt_err = [] - deps = Package_Dependency.objects.values('id', 'package_id', 'depends_on_id') - for dep in deps: - if (dep['package_id'] == dep['depends_on_id']): - cnt_err.append(dep['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package dependency id: %s' % cnt_err) - - # Recipe key verification, recipe name does not depends on a recipe having the same name - tc=883 - @testcase(883) - def test_Recipe_Dependency(self): - deps = Recipe_Dependency.objects.values('id', 'recipe_id', 'depends_on_id') - cnt_err = [] - for dep in deps: - if (not dep['recipe_id'] or not dep['depends_on_id']): - cnt_err.append(dep['id']) - else: - name = Recipe.objects.filter(id = dep['recipe_id']).values('name') - dep_name = Recipe.objects.filter(id = dep['depends_on_id']).values('name') - if (name == dep_name): - cnt_err.append(dep['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for recipe dependency id: %s' % cnt_err) - - # Check if package name does not start with a number (0-9) - tc=846 - @testcase(846) - def test_Package_Name_For_Number(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'name') - cnt_err = [] - for package in packages: - if (package['name'][0].isdigit() is True): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if package version starts with a number (0-9) - tc=847 - @testcase(847) - def test_Package_Version_Starts_With_Number(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'version') - cnt_err = [] - for package in packages: - if (package['version'][0].isdigit() is False): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if package revision starts with 'r' - tc=848 - @testcase(848) - def test_Package_Revision_Starts_With_r(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'revision') - cnt_err = [] - for package in packages: - if (package['revision'][0].startswith("r") is False): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check the validity of the package build_id - ### TC must be added in test run - @testcase(1038) - def test_Package_Build_Id(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'build_id') - cnt_err = [] - for package in packages: - build_id = Build.objects.filter(id = package['build_id']).values('id') - if (not build_id): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check the validity of package recipe_id - ### TC must be added in test run - @testcase(1039) - def test_Package_Recipe_Id(self): - packages = Package.objects.filter(~Q(size = -1)).values('id', 'recipe_id') - cnt_err = [] - for package in packages: - recipe_id = Recipe.objects.filter(id = package['recipe_id']).values('id') - if (not recipe_id): - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if package installed_size field is not null - ### TC must be aded in test run - @testcase(1040) - def test_Package_Installed_Size_Not_NULL(self): - packages = Package.objects.filter(installed_size__isnull = True).values('id') - cnt_err = [] - for package in packages: - cnt_err.append(package['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for package id: %s' % cnt_err) - - # Check if all layers requests return exit code is 200 - tc=843 - @testcase(843) - def test_Layers_Requests_Exit_Code(self): - layers = Layer.objects.values('id', 'layer_index_url') - cnt_err = [] - for layer in layers: - resp = urllib.request.urlopen(layer['layer_index_url']) - if (resp.getcode() != 200): - cnt_err.append(layer['id']) - self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) - - # Check if django server starts regardless of the timezone set on the machine - tc=905 - @testcase(905) - def test_Start_Django_Timezone(self): - current_path = os.getcwd() - zonefilelist = [] - ZONEINFOPATH = '/usr/share/zoneinfo/' - os.chdir("../bitbake/lib/toaster/") - cnt_err = 0 - for filename in os.listdir(ZONEINFOPATH): - if os.path.isfile(os.path.join(ZONEINFOPATH, filename)): - zonefilelist.append(filename) - for k in range(len(zonefilelist)): - if k <= 5: - files = zonefilelist[k] - os.system("export TZ="+str(files)+"; python manage.py runserver > /dev/null 2>&1 &") - time.sleep(3) - pid = subprocess.check_output("ps aux | grep '[/u]sr/bin/python manage.py runserver' | awk '{print $2}'", shell = True) - if pid: - os.system("kill -9 "+str(pid)) - else: - cnt_err.append(zonefilelist[k]) - self.assertEqual(cnt_err, 0, msg = 'Errors django server does not start with timezone: %s' % cnt_err) - os.chdir(current_path) diff --git a/meta/lib/oeqa/selftest/archiver.py b/meta/lib/oeqa/selftest/archiver.py index f2030c446d..d7f215cbf6 100644 --- a/meta/lib/oeqa/selftest/archiver.py +++ b/meta/lib/oeqa/selftest/archiver.py @@ -1,5 +1,5 @@ from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import bitbake, get_bb_var +from oeqa.utils.commands import bitbake, get_bb_vars from oeqa.utils.decorators import testcase import glob import os @@ -26,25 +26,18 @@ class Archiver(oeSelfTest): features += 'ARCHIVER_MODE[src] = "original"\n' features += 'COPYLEFT_PN_INCLUDE = "%s"\n' % include_recipe features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % exclude_recipe - - # Update local.conf self.write_config(features) - tmp_dir = get_bb_var('TMPDIR') - deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC') - target_sys = get_bb_var('TARGET_SYS') - src_path = os.path.join(deploy_dir_src, target_sys) - - # Delete tmp directory - shutil.rmtree(tmp_dir) + bitbake('-c clean %s %s' % (include_recipe, exclude_recipe)) + bitbake("%s %s" % (include_recipe, exclude_recipe)) - # Build core-image-minimal - bitbake('core-image-minimal') + bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS']) + src_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS']) # Check that include_recipe was included - is_included = len(glob.glob(src_path + '/%s*' % include_recipe)) - self.assertEqual(1, is_included, 'Recipe %s was not included.' % include_recipe) + included_present = len(glob.glob(src_path + '/%s-*' % include_recipe)) + self.assertTrue(included_present, 'Recipe %s was not included.' % include_recipe) # Check that exclude_recipe was excluded - is_excluded = len(glob.glob(src_path + '/%s*' % exclude_recipe)) - self.assertEqual(0, is_excluded, 'Recipe %s was not excluded.' % exclude_recipe) + excluded_present = len(glob.glob(src_path + '/%s-*' % exclude_recipe)) + self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % exclude_recipe) diff --git a/meta/lib/oeqa/selftest/base.py b/meta/lib/oeqa/selftest/base.py index b5a52fe57a..47a8ea8271 100644 --- a/meta/lib/oeqa/selftest/base.py +++ b/meta/lib/oeqa/selftest/base.py @@ -28,17 +28,47 @@ class oeSelfTest(unittest.TestCase): def __init__(self, methodName="runTest"): self.builddir = os.environ.get("BUILDDIR") self.localconf_path = os.path.join(self.builddir, "conf/local.conf") + self.localconf_backup = os.path.join(self.builddir, "conf/local.bk") self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc") self.local_bblayers_path = os.path.join(self.builddir, "conf/bblayers.conf") + self.local_bblayers_backup = os.path.join(self.builddir, + "conf/bblayers.bk") self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc") self.machineinc_path = os.path.join(self.builddir, "conf/machine.inc") self.testlayer_path = oeSelfTest.testlayer_path self._extra_tear_down_commands = [] - self._track_for_cleanup = [self.testinc_path, self.testinc_bblayers_path, self.machineinc_path] + self._track_for_cleanup = [ + self.testinc_path, self.testinc_bblayers_path, + self.machineinc_path, self.localconf_backup, + self.local_bblayers_backup] super(oeSelfTest, self).__init__(methodName) def setUp(self): os.chdir(self.builddir) + # Check if local.conf or bblayers.conf files backup exists + # from a previous failed test and restore them + if os.path.isfile(self.localconf_backup) or os.path.isfile( + self.local_bblayers_backup): + self.log.debug("Found a local.conf and/or bblayers.conf backup \ +from a previously aborted test. Restoring these files now, but tests should \ +be re-executed from a clean environment to ensure accurate results.") + try: + shutil.copyfile(self.localconf_backup, self.localconf_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + try: + shutil.copyfile(self.local_bblayers_backup, + self.local_bblayers_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + else: + # backup local.conf and bblayers.conf + shutil.copyfile(self.localconf_path, self.localconf_backup) + shutil.copyfile(self.local_bblayers_path, + self.local_bblayers_backup) + self.log.debug("Creating local.conf and bblayers.conf backups.") # we don't know what the previous test left around in config or inc files # if it failed so we need a fresh start try: @@ -133,7 +163,7 @@ class oeSelfTest(unittest.TestCase): # remove data from <builddir>/conf/selftest.inc def remove_config(self, data): - self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data)) + self.log.debug("Removing from: %s\n%s\n" % (self.testinc_path, data)) ftools.remove_from_file(self.testinc_path, data) # write to meta-sefltest/recipes-test/<recipe>/test_recipe.inc @@ -176,7 +206,7 @@ class oeSelfTest(unittest.TestCase): # remove data from <builddir>/conf/bblayers.inc def remove_bblayers_config(self, data): - self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_bblayers_path, data)) + self.log.debug("Removing from: %s\n%s\n" % (self.testinc_bblayers_path, data)) ftools.remove_from_file(self.testinc_bblayers_path, data) # write to <builddir>/conf/machine.inc diff --git a/meta/lib/oeqa/selftest/bblayers.py b/meta/lib/oeqa/selftest/bblayers.py index d23675e84a..cd658c5d4e 100644 --- a/meta/lib/oeqa/selftest/bblayers.py +++ b/meta/lib/oeqa/selftest/bblayers.py @@ -71,17 +71,12 @@ class BitbakeLayers(oeSelfTest): result = runCmd('bitbake-layers show-recipes') self.assertIn('aspell:', result.output) self.assertIn('mtd-utils:', result.output) - self.assertIn('linux-yocto:', result.output) self.assertIn('core-image-minimal:', result.output) result = runCmd('bitbake-layers show-recipes mtd-utils') self.assertIn('mtd-utils:', result.output) self.assertNotIn('aspell:', result.output) - result = runCmd('bitbake-layers show-recipes -i kernel') - self.assertIn('linux-yocto:', result.output) - self.assertNotIn('mtd-utils:', result.output) result = runCmd('bitbake-layers show-recipes -i image') self.assertIn('core-image-minimal', result.output) - self.assertNotIn('linux-yocto:', result.output) self.assertNotIn('mtd-utils:', result.output) result = runCmd('bitbake-layers show-recipes -i cmake,pkgconfig') self.assertIn('libproxy:', result.output) diff --git a/meta/lib/oeqa/selftest/bbtests.py b/meta/lib/oeqa/selftest/bbtests.py index 26728a4b41..46e09f509f 100644 --- a/meta/lib/oeqa/selftest/bbtests.py +++ b/meta/lib/oeqa/selftest/bbtests.py @@ -3,7 +3,7 @@ import re import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars from oeqa.utils.decorators import testcase class BitbakeTests(oeSelfTest): @@ -29,7 +29,7 @@ class BitbakeTests(oeSelfTest): def test_event_handler(self): self.write_config("INHERIT += \"test_events\"") result = bitbake('m4-native') - find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Preparing RunQueue", result.output) + find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Executing RunQueue Tasks", result.output) find_build_completed = re.search("Tasks Summary:.*(\n.*)*NOTE: Test for bb\.event\.BuildCompleted", result.output) self.assertTrue(find_build_started, msg = "Match failed in:\n%s" % result.output) self.assertTrue(find_build_completed, msg = "Match failed in:\n%s" % result.output) @@ -37,7 +37,6 @@ class BitbakeTests(oeSelfTest): @testcase(103) def test_local_sstate(self): - bitbake('m4-native -ccleansstate') bitbake('m4-native') bitbake('m4-native -cclean') result = bitbake('m4-native') @@ -64,24 +63,28 @@ class BitbakeTests(oeSelfTest): @testcase(108) def test_invalid_patch(self): + # This patch already exists in SRC_URI so adding it again will cause the + # patch to fail. self.write_recipeinc('man', 'SRC_URI += "file://man-1.5h1-make.patch"') + self.write_config("INHERIT_remove = \"report-error\"") result = bitbake('man -c patch', ignore_status=True) self.delete_recipeinc('man') bitbake('-cclean man') line = self.getline(result, "Function failed: patch_do_patch") - self.assertTrue(line and line.startswith("ERROR:"), msg = "Though no man-1.5h1-make.patch file exists, bitbake didn't output any err. message. bitbake output: %s" % result.output) + self.assertTrue(line and line.startswith("ERROR:"), msg = "Repeated patch application didn't fail. bitbake output: %s" % result.output) @testcase(1354) def test_force_task_1(self): # test 1 from bug 5875 test_recipe = 'zlib' test_data = "Microsoft Made No Profit From Anyone's Zunes Yo" - image_dir = get_bb_var('D', test_recipe) - pkgsplit_dir = get_bb_var('PKGDEST', test_recipe) - man_dir = get_bb_var('mandir', test_recipe) + bb_vars = get_bb_vars(['D', 'PKGDEST', 'mandir'], test_recipe) + image_dir = bb_vars['D'] + pkgsplit_dir = bb_vars['PKGDEST'] + man_dir = bb_vars['mandir'] - bitbake('-c cleansstate %s' % test_recipe) - bitbake(test_recipe) + bitbake('-c clean %s' % test_recipe) + bitbake('-c package -f %s' % test_recipe) self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) man_file = os.path.join(image_dir + man_dir, 'man3/zlib.3') @@ -100,7 +103,6 @@ class BitbakeTests(oeSelfTest): # test 2 from bug 5875 test_recipe = 'zlib' - bitbake('-c cleansstate %s' % test_recipe) bitbake(test_recipe) self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) @@ -111,17 +113,18 @@ class BitbakeTests(oeSelfTest): @testcase(167) def test_bitbake_g(self): - result = bitbake('-g core-image-full-cmdline') - for f in ['pn-buildlist', 'pn-depends.dot', 'package-depends.dot', 'task-depends.dot']: + result = bitbake('-g core-image-minimal') + for f in ['pn-buildlist', 'recipe-depends.dot', 'task-depends.dot']: self.addCleanup(os.remove, f) - self.assertTrue('NOTE: PN build list saved to \'pn-buildlist\'' in result.output, msg = "No dependency \"pn-buildlist\" file was generated for the given task target. bitbake output: %s" % result.output) - self.assertTrue('openssh' in ftools.read_file(os.path.join(self.builddir, 'pn-buildlist')), msg = "No \"openssh\" dependency found in pn-buildlist file.") + self.assertTrue('Task dependencies saved to \'task-depends.dot\'' in result.output, msg = "No task dependency \"task-depends.dot\" file was generated for the given task target. bitbake output: %s" % result.output) + self.assertTrue('busybox' in ftools.read_file(os.path.join(self.builddir, 'task-depends.dot')), msg = "No \"busybox\" dependency found in task-depends.dot file.") @testcase(899) def test_image_manifest(self): bitbake('core-image-minimal') - deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") - imagename = get_bb_var("IMAGE_LINK_NAME", target="core-image-minimal") + bb_vars = get_bb_vars(["DEPLOY_DIR_IMAGE", "IMAGE_LINK_NAME"], "core-image-minimal") + deploydir = bb_vars["DEPLOY_DIR_IMAGE"] + imagename = bb_vars["IMAGE_LINK_NAME"] manifest = os.path.join(deploydir, imagename + ".manifest") self.assertTrue(os.path.islink(manifest), msg="No manifest file created for image. It should have been created in %s" % manifest) @@ -131,6 +134,7 @@ class BitbakeTests(oeSelfTest): self.write_recipeinc('man', data) self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" +INHERIT_remove = \"report-error\" """) self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) @@ -141,25 +145,27 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertEqual(result.status, 1, msg="Command succeded when it should have failed. bitbake output: %s" % result.output) self.assertTrue('Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output, msg = "\"invalid\" file \ doesn't exist, yet no error message encountered. bitbake output: %s" % result.output) - line = self.getline(result, 'Function failed: Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.') + line = self.getline(result, 'Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.') self.assertTrue(line and line.startswith("ERROR:"), msg = "\"invalid\" file \ doesn't exist, yet fetcher didn't report any error. bitbake output: %s" % result.output) @testcase(171) def test_rename_downloaded_file(self): + # TODO unique dldir instead of using cleanall + # TODO: need to set sstatedir? self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" """) self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) - data = 'SRC_URI_append = ";downloadfilename=test-aspell.tar.gz"' + data = 'SRC_URI = "${GNU_MIRROR}/aspell/aspell-${PV}.tar.gz;downloadfilename=test-aspell.tar.gz"' self.write_recipeinc('aspell', data) - bitbake('-ccleanall aspell') - result = bitbake('-c fetch aspell', ignore_status=True) + result = bitbake('-f -c fetch aspell', ignore_status=True) self.delete_recipeinc('aspell') self.assertEqual(result.status, 0, msg = "Couldn't fetch aspell. %s" % result.output) - self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz')), msg = "File rename failed. No corresponding test-aspell.tar.gz file found under %s" % str(get_bb_var("DL_DIR"))) - self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz.done')), "File rename failed. No corresponding test-aspell.tar.gz.done file found under %s" % str(get_bb_var("DL_DIR"))) + dl_dir = get_bb_var("DL_DIR") + self.assertTrue(os.path.isfile(os.path.join(dl_dir, 'test-aspell.tar.gz')), msg = "File rename failed. No corresponding test-aspell.tar.gz file found under %s" % dl_dir) + self.assertTrue(os.path.isfile(os.path.join(dl_dir, 'test-aspell.tar.gz.done')), "File rename failed. No corresponding test-aspell.tar.gz.done file found under %s" % dl_dir) @testcase(1028) def test_environment(self): @@ -212,11 +218,12 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" def test_continue(self): self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" +INHERIT_remove = \"report-error\" """) self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) self.write_recipeinc('man',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" ) runCmd('bitbake -c cleanall man xcursor-transparent-theme') - result = runCmd('bitbake man xcursor-transparent-theme -k', ignore_status=True) + result = runCmd('bitbake -c unpack -k man xcursor-transparent-theme', ignore_status=True) errorpos = result.output.find('ERROR: Function failed: do_fail_task') manver = re.search("NOTE: recipe xcursor-transparent-theme-(.*?): task do_unpack: Started", result.output) continuepos = result.output.find('NOTE: recipe xcursor-transparent-theme-%s: task do_unpack: Started' % manver.group(1)) @@ -224,14 +231,12 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" @testcase(1119) def test_non_gplv3(self): - data = 'INCOMPATIBLE_LICENSE = "GPLv3"' - conf = os.path.join(self.builddir, 'conf/local.conf') - ftools.append_file(conf ,data) - self.addCleanup(ftools.remove_from_file, conf ,data) - result = bitbake('readline', ignore_status=True) + self.write_config('INCOMPATIBLE_LICENSE = "GPLv3"') + result = bitbake('selftest-ed', ignore_status=True) self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output)) - self.assertFalse(os.path.isfile(os.path.join(self.builddir, 'tmp/deploy/licenses/readline/generic_GPLv3'))) - self.assertTrue(os.path.isfile(os.path.join(self.builddir, 'tmp/deploy/licenses/readline/generic_GPLv2'))) + lic_dir = get_bb_var('LICENSE_DIRECTORY') + self.assertFalse(os.path.isfile(os.path.join(lic_dir, 'selftest-ed/generic_GPLv3'))) + self.assertTrue(os.path.isfile(os.path.join(lic_dir, 'selftest-ed/generic_GPLv2'))) @testcase(1422) def test_setscene_only(self): @@ -252,8 +257,9 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" def test_bbappend_order(self): """ Bitbake should bbappend to recipe in a predictable order """ test_recipe = 'ed' - test_recipe_summary_before = get_bb_var('SUMMARY', test_recipe) - test_recipe_pv = get_bb_var('PV', test_recipe) + bb_vars = get_bb_vars(['SUMMARY', 'PV'], test_recipe) + test_recipe_summary_before = bb_vars['SUMMARY'] + test_recipe_pv = bb_vars['PV'] recipe_append_file = test_recipe + '_' + test_recipe_pv + '.bbappend' expected_recipe_summary = test_recipe_summary_before diff --git a/meta/lib/oeqa/selftest/buildhistory.py b/meta/lib/oeqa/selftest/buildhistory.py index 674da6205a..008c39c956 100644 --- a/meta/lib/oeqa/selftest/buildhistory.py +++ b/meta/lib/oeqa/selftest/buildhistory.py @@ -3,14 +3,15 @@ import re import datetime from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import bitbake, get_bb_var +from oeqa.utils.commands import bitbake, get_bb_vars from oeqa.utils.decorators import testcase class BuildhistoryBase(oeSelfTest): def config_buildhistory(self, tmp_bh_location=False): - if (not 'buildhistory' in get_bb_var('USER_CLASSES')) and (not 'buildhistory' in get_bb_var('INHERIT')): + bb_vars = get_bb_vars(['USER_CLASSES', 'INHERIT']) + if (not 'buildhistory' in bb_vars['USER_CLASSES']) and (not 'buildhistory' in bb_vars['INHERIT']): add_buildhistory_config = 'INHERIT += "buildhistory"\nBUILDHISTORY_COMMIT = "1"' self.append_config(add_buildhistory_config) diff --git a/meta/lib/oeqa/selftest/buildoptions.py b/meta/lib/oeqa/selftest/buildoptions.py index b6151d7d82..a6e0203f5a 100644 --- a/meta/lib/oeqa/selftest/buildoptions.py +++ b/meta/lib/oeqa/selftest/buildoptions.py @@ -5,7 +5,7 @@ import shutil import tempfile from oeqa.selftest.base import oeSelfTest from oeqa.selftest.buildhistory import BuildhistoryBase -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import oeqa.utils.ftools as ftools from oeqa.utils.decorators import testcase @@ -16,49 +16,38 @@ class ImageOptionsTests(oeSelfTest): image_pkgtype = get_bb_var("IMAGE_PKGTYPE") if image_pkgtype != 'rpm': self.skipTest('Not using RPM as main package format') - bitbake("-c cleanall core-image-minimal") + bitbake("-c clean core-image-minimal") self.write_config('INC_RPM_IMAGE_GEN = "1"') self.append_config('IMAGE_FEATURES += "ssh-server-openssh"') bitbake("core-image-minimal") log_data_file = os.path.join(get_bb_var("WORKDIR", "core-image-minimal"), "temp/log.do_rootfs") log_data_created = ftools.read_file(log_data_file) - incremental_created = re.search("NOTE: load old install solution for incremental install\nNOTE: old install solution not exist\nNOTE: creating new install solution for incremental install(\n.*)*NOTE: Installing the following packages:.*packagegroup-core-ssh-openssh", log_data_created) + incremental_created = re.search("Installing : packagegroup-core-ssh-openssh", log_data_created) self.remove_config('IMAGE_FEATURES += "ssh-server-openssh"') self.assertTrue(incremental_created, msg = "Match failed in:\n%s" % log_data_created) bitbake("core-image-minimal") log_data_removed = ftools.read_file(log_data_file) - incremental_removed = re.search("NOTE: load old install solution for incremental install\nNOTE: creating new install solution for incremental install(\n.*)*NOTE: incremental removed:.*openssh-sshd-.*", log_data_removed) + incremental_removed = re.search("Erasing : packagegroup-core-ssh-openssh", log_data_removed) self.assertTrue(incremental_removed, msg = "Match failed in:\n%s" % log_data_removed) - @testcase(925) - def test_rm_old_image(self): - bitbake("core-image-minimal") - deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") - imagename = get_bb_var("IMAGE_LINK_NAME", target="core-image-minimal") - deploydir_files = os.listdir(deploydir) - track_original_files = [] - for image_file in deploydir_files: - if imagename in image_file and os.path.islink(os.path.join(deploydir, image_file)): - track_original_files.append(os.path.realpath(os.path.join(deploydir, image_file))) - self.write_config("RM_OLD_IMAGE = \"1\"") - bitbake("-C rootfs core-image-minimal") - deploydir_files = os.listdir(deploydir) - remaining_not_expected = [path for path in track_original_files if os.path.basename(path) in deploydir_files] - self.assertFalse(remaining_not_expected, msg="\nThe following image files were not removed: %s" % ', '.join(map(str, remaining_not_expected))) - @testcase(286) def test_ccache_tool(self): bitbake("ccache-native") - self.assertTrue(os.path.isfile(os.path.join(get_bb_var('STAGING_BINDIR_NATIVE', 'ccache-native'), "ccache")), msg = "No ccache found under %s" % str(get_bb_var('STAGING_BINDIR_NATIVE', 'ccache-native'))) + bb_vars = get_bb_vars(['SYSROOT_DESTDIR', 'bindir'], 'ccache-native') + p = bb_vars['SYSROOT_DESTDIR'] + bb_vars['bindir'] + "/" + "ccache" + self.assertTrue(os.path.isfile(p), msg = "No ccache found (%s)" % p) self.write_config('INHERIT += "ccache"') - bitbake("m4 -c cleansstate") - bitbake("m4 -c compile") - self.addCleanup(bitbake, 'ccache-native -ccleansstate') - res = runCmd("grep ccache %s" % (os.path.join(get_bb_var("WORKDIR","m4"),"temp/log.do_compile")), ignore_status=True) - self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % os.path.join(get_bb_var("WORKDIR","m4"),"temp/log.do_compile")) + self.add_command_to_tearDown('bitbake -c clean m4') + bitbake("m4 -f -c compile") + log_compile = os.path.join(get_bb_var("WORKDIR","m4"), "temp/log.do_compile") + res = runCmd("grep ccache %s" % log_compile, ignore_status=True) + self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile) @testcase(1435) def test_read_only_image(self): + distro_features = get_bb_var('DISTRO_FEATURES') + if not ('x11' in distro_features and 'opengl' in distro_features): + self.skipTest('core-image-sato requires x11 and opengl in distro features') self.write_config('IMAGE_FEATURES += "read-only-rootfs"') bitbake("core-image-sato") # do_image will fail if there are any pending postinsts @@ -87,13 +76,14 @@ class SanityOptionsTest(oeSelfTest): @testcase(927) def test_options_warnqa_errorqa_switch(self): - bitbake("xcursor-transparent-theme -ccleansstate") + self.write_config("INHERIT_remove = \"report-error\"") if "packages-list" not in get_bb_var("ERROR_QA"): - self.write_config("ERROR_QA_append = \" packages-list\"") + self.append_config("ERROR_QA_append = \" packages-list\"") self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') - res = bitbake("xcursor-transparent-theme", ignore_status=True) + self.add_command_to_tearDown('bitbake -c clean xcursor-transparent-theme') + res = bitbake("xcursor-transparent-theme -f -c package", ignore_status=True) self.delete_recipeinc('xcursor-transparent-theme') line = self.getline(res, "QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors.") self.assertTrue(line and line.startswith("ERROR:"), msg=res.output) @@ -101,8 +91,7 @@ class SanityOptionsTest(oeSelfTest): self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') self.append_config('ERROR_QA_remove = "packages-list"') self.append_config('WARN_QA_append = " packages-list"') - bitbake("xcursor-transparent-theme -ccleansstate") - res = bitbake("xcursor-transparent-theme") + res = bitbake("xcursor-transparent-theme -f -c package") self.delete_recipeinc('xcursor-transparent-theme') line = self.getline(res, "QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors.") self.assertTrue(line and line.startswith("WARNING:"), msg=res.output) @@ -111,8 +100,8 @@ class SanityOptionsTest(oeSelfTest): def test_sanity_unsafe_script_references(self): self.write_config('WARN_QA_append = " unsafe-references-in-scripts"') - bitbake("-ccleansstate gzip") - res = bitbake("gzip") + self.add_command_to_tearDown('bitbake -c clean gzip') + res = bitbake("gzip -f -c package_qa") line = self.getline(res, "QA Issue: gzip") self.assertFalse(line, "WARNING: QA Issue: gzip message is present in bitbake's output and shouldn't be: %s" % res.output) @@ -121,29 +110,10 @@ do_install_append_pn-gzip () { echo "\n${bindir}/test" >> ${D}${bindir}/zcat } """) - res = bitbake("gzip") + res = bitbake("gzip -f -c package_qa") line = self.getline(res, "QA Issue: gzip") self.assertTrue(line and line.startswith("WARNING:"), "WARNING: QA Issue: gzip message is not present in bitbake's output: %s" % res.output) - @testcase(1434) - def test_sanity_unsafe_binary_references(self): - self.write_config('WARN_QA_append = " unsafe-references-in-binaries"') - - bitbake("-ccleansstate nfs-utils") - #res = bitbake("nfs-utils") - # FIXME when nfs-utils passes this test - #line = self.getline(res, "QA Issue: nfs-utils") - #self.assertFalse(line, "WARNING: QA Issue: nfs-utils message is present in bitbake's output and shouldn't be: %s" % res.output) - -# self.append_config(""" -#do_install_append_pn-nfs-utils () { -# echo "\n${bindir}/test" >> ${D}${base_sbindir}/osd_login -#} -#""") - res = bitbake("nfs-utils") - line = self.getline(res, "QA Issue: nfs-utils") - self.assertTrue(line and line.startswith("WARNING:"), "WARNING: QA Issue: nfs-utils message is not present in bitbake's output: %s" % res.output) - @testcase(1421) def test_layer_without_git_dir(self): """ @@ -193,36 +163,22 @@ class BuildhistoryTests(BuildhistoryBase): @testcase(294) def test_buildhistory_buildtime_pr_backwards(self): - self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' error = "ERROR:.*QA Issue: Package version for package %s went backwards which would break package feeds from (.*-r1.* to .*-r0.*)" % target self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error) - -class BuildImagesTest(oeSelfTest): - @testcase(563) - def test_directfb(self): - """ - This method is used to test the build of directfb image for arm arch. - In essence we build a coreimagedirectfb and test the exitcode of bitbake that in case of success is 0. - """ - self.add_command_to_tearDown('cleanup-workdir') - self.write_config("DISTRO_FEATURES_remove = \"x11\"\nDISTRO_FEATURES_append = \" directfb\"\nMACHINE ??= \"qemuarm\"") - res = bitbake("core-image-directfb", ignore_status=True) - self.assertEqual(res.status, 0, "\ncoreimagedirectfb failed to build. Please check logs for further details.\nbitbake output %s" % res.output) - class ArchiverTest(oeSelfTest): @testcase(926) def test_arch_work_dir_and_export_source(self): """ Test for archiving the work directory and exporting the source files. """ - self.add_command_to_tearDown('cleanup-workdir') self.write_config("INHERIT += \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") res = bitbake("xcursor-transparent-theme", ignore_status=True) self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output) - pkgs_path = g.glob(str(self.builddir) + "/tmp/deploy/sources/allarch*/xcurs*") + deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC') + pkgs_path = g.glob(str(deploy_dir_src) + "/allarch*/xcurs*") src_file_glob = str(pkgs_path[0]) + "/xcursor*.src.rpm" tar_file_glob = str(pkgs_path[0]) + "/xcursor*.tar.gz" - self.assertTrue((g.glob(src_file_glob) and g.glob(tar_file_glob)), "Couldn't find .src.rpm and .tar.gz files under tmp/deploy/sources/allarch*/xcursor*") + self.assertTrue((g.glob(src_file_glob) and g.glob(tar_file_glob)), "Couldn't find .src.rpm and .tar.gz files under %s/allarch*/xcursor*" % deploy_dir_src) diff --git a/meta/lib/oeqa/selftest/containerimage.py b/meta/lib/oeqa/selftest/containerimage.py new file mode 100644 index 0000000000..def481f144 --- /dev/null +++ b/meta/lib/oeqa/selftest/containerimage.py @@ -0,0 +1,83 @@ +import os + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake, get_bb_vars, runCmd + +# This test builds an image with using the "container" IMAGE_FSTYPE, and +# ensures that then files in the image are only the ones expected. +# +# The only package added to the image is container_image_testpkg, which +# contains one file. However, due to some other things not cleaning up during +# rootfs creation, there is some cruft. Ideally bugs will be filed and the +# cruft removed, but for now we whitelist some known set. +# +# Also for performance reasons we're only checking the cruft when using ipk. +# When using deb, and rpm it is a bit different and we could test all +# of them, but this test is more to catch if other packages get added by +# default other than what is in ROOTFS_BOOTSTRAP_INSTALL. +# +class ContainerImageTests(oeSelfTest): + + # Verify that when specifying a IMAGE_TYPEDEP_ of the form "foo.bar" that + # the conversion type bar gets added as a dep as well + def test_expected_files(self): + + def get_each_path_part(path): + if path: + part = [ '.' + path + '/' ] + result = get_each_path_part(path.rsplit('/', 1)[0]) + if result: + return part + result + else: + return part + else: + return None + + self.write_config("""PREFERRED_PROVIDER_virtual/kernel = "linux-dummy" +IMAGE_FSTYPES = "container" +PACKAGE_CLASSES = "package_ipk" +IMAGE_FEATURES = "" +""") + + bbvars = get_bb_vars(['bindir', 'sysconfdir', 'localstatedir', + 'DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'], + target='container-test-image') + expected_files = [ + './', + '.{bindir}/theapp', + '.{sysconfdir}/default/', + '.{sysconfdir}/default/postinst', + '.{sysconfdir}/ld.so.cache', + '.{sysconfdir}/timestamp', + '.{sysconfdir}/version', + './run/', + '.{localstatedir}/cache/', + '.{localstatedir}/cache/ldconfig/', + '.{localstatedir}/cache/ldconfig/aux-cache', + '.{localstatedir}/cache/opkg/', + '.{localstatedir}/lib/', + '.{localstatedir}/lib/opkg/' + ] + + expected_files = [ x.format(bindir=bbvars['bindir'], + sysconfdir=bbvars['sysconfdir'], + localstatedir=bbvars['localstatedir']) + for x in expected_files ] + + # Since tar lists all directories individually, make sure each element + # from bindir, sysconfdir, etc is added + expected_files += get_each_path_part(bbvars['bindir']) + expected_files += get_each_path_part(bbvars['sysconfdir']) + expected_files += get_each_path_part(bbvars['localstatedir']) + + expected_files = sorted(expected_files) + + # Build the image of course + bitbake('container-test-image') + + image = os.path.join(bbvars['DEPLOY_DIR_IMAGE'], + bbvars['IMAGE_LINK_NAME'] + '.tar.bz2') + + # Ensure the files in the image are what we expect + result = runCmd("tar tf {} | sort".format(image), shell=True) + self.assertEqual(result.output.split('\n'), expected_files) diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py index 0b305c893e..8e7642c32b 100644 --- a/meta/lib/oeqa/selftest/devtool.py +++ b/meta/lib/oeqa/selftest/devtool.py @@ -5,10 +5,12 @@ import re import shutil import tempfile import glob +import fnmatch import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer +from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer from oeqa.utils.decorators import testcase class DevtoolBase(oeSelfTest): @@ -43,9 +45,12 @@ class DevtoolBase(oeSelfTest): if var and var in checkvars: needvalue = checkvars.pop(var) if needvalue is None: - self.fail('Variable %s should not appear in recipe') + self.fail('Variable %s should not appear in recipe, but value is being set to "%s"' % (var, value)) if isinstance(needvalue, set): - value = set(value.split()) + if var == 'LICENSE': + value = set(value.split(' & ')) + else: + value = set(value.split()) self.assertEqual(value, needvalue, 'values for %s do not match' % var) @@ -113,6 +118,20 @@ class DevtoolBase(oeSelfTest): class DevtoolTests(DevtoolBase): + @classmethod + def setUpClass(cls): + bb_vars = get_bb_vars(['TOPDIR', 'SSTATE_DIR']) + cls.original_sstate = bb_vars['SSTATE_DIR'] + cls.devtool_sstate = os.path.join(bb_vars['TOPDIR'], 'sstate_devtool') + cls.sstate_conf = 'SSTATE_DIR = "%s"\n' % cls.devtool_sstate + cls.sstate_conf += ('SSTATE_MIRRORS += "file://.* file:///%s/PATH"\n' + % cls.original_sstate) + + @classmethod + def tearDownClass(cls): + cls.log.debug('Deleting devtool sstate cache on %s' % cls.devtool_sstate) + runCmd('rm -rf %s' % cls.devtool_sstate) + def setUp(self): """Test case setup function""" super(DevtoolTests, self).setUp() @@ -120,6 +139,7 @@ class DevtoolTests(DevtoolBase): self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory ' 'under the build directory') + self.append_config(self.sstate_conf) def _check_src_repo(self, repo_dir): """Check srctree git repository""" @@ -193,9 +213,10 @@ class DevtoolTests(DevtoolBase): bitbake('pv -c cleansstate') # Test devtool build result = runCmd('devtool build pv') - installdir = get_bb_var('D', 'pv') + bb_vars = get_bb_vars(['D', 'bindir'], 'pv') + installdir = bb_vars['D'] self.assertTrue(installdir, 'Could not query installdir variable') - bindir = get_bb_var('bindir', 'pv') + bindir = bb_vars['bindir'] self.assertTrue(bindir, 'Could not query bindir variable') if bindir[0] == '/': bindir = bindir[1:] @@ -243,8 +264,6 @@ class DevtoolTests(DevtoolBase): @testcase(1162) def test_devtool_add_library(self): - # We don't have the ability to pick up this dependency automatically yet... - bitbake('libusb1') # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -273,13 +292,17 @@ class DevtoolTests(DevtoolBase): result = runCmd('recipetool setvar %s EXTRA_OECMAKE -- \'-DPYTHON_BINDINGS=OFF -DLIBFTDI_CMAKE_CONFIG_DIR=${datadir}/cmake/Modules\'' % recipefile) with open(recipefile, 'a') as f: f.write('\nFILES_${PN}-dev += "${datadir}/cmake/Modules"\n') + # We don't have the ability to pick up this dependency automatically yet... + f.write('\nDEPENDS += "libusb1"\n') + f.write('\nTESTLIBOUTPUT = "${STAGING_DIR}-components/${TUNE_PKGARCH}/${PN}/${libdir}"\n') # Test devtool build result = runCmd('devtool build libftdi') - staging_libdir = get_bb_var('STAGING_LIBDIR', 'libftdi') - self.assertTrue(staging_libdir, 'Could not query STAGING_LIBDIR variable') + bb_vars = get_bb_vars(['TESTLIBOUTPUT', 'STAMP'], 'libftdi') + staging_libdir = bb_vars['TESTLIBOUTPUT'] + self.assertTrue(staging_libdir, 'Could not query TESTLIBOUTPUT variable') self.assertTrue(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), "libftdi binary not found in STAGING_LIBDIR. Output of devtool build libftdi %s" % result.output) # Test devtool reset - stampprefix = get_bb_var('STAMP', 'libftdi') + stampprefix = bb_vars['STAMP'] result = runCmd('devtool reset libftdi') result = runCmd('devtool status') self.assertNotIn('libftdi', result.output) @@ -336,12 +359,11 @@ class DevtoolTests(DevtoolBase): @testcase(1161) def test_devtool_add_fetch_git(self): - # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - url = 'git://git.yoctoproject.org/libmatchbox' - checkrev = '462f0652055d89c648ddd54fd7b03f175c2c6973' - testrecipe = 'libmatchbox2' + url = 'gitsm://git.yoctoproject.org/mraa' + checkrev = 'ae127b19a50aa54255e4330ccfdd9a5d058e581d' + testrecipe = 'mraa' srcdir = os.path.join(tempdir, testrecipe) # Test devtool add self.track_for_cleanup(self.workspacedir) @@ -349,7 +371,7 @@ class DevtoolTests(DevtoolBase): self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool add %s %s -a -f %s' % (testrecipe, srcdir, url)) self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output) - self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory') + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'imraa', 'imraa.c')), 'Unable to find imraa/imraa.c in source directory') # Test devtool status result = runCmd('devtool status') self.assertIn(testrecipe, result.output) @@ -359,7 +381,7 @@ class DevtoolTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.11+git${SRCPV}' + checkvars['PV'] = '1.0+git${SRCPV}' checkvars['SRC_URI'] = url checkvars['SRCREV'] = '${AUTOREV}' self._test_recipe_contents(recipefile, checkvars, []) @@ -368,7 +390,7 @@ class DevtoolTests(DevtoolBase): shutil.rmtree(srcdir) url_rev = '%s;rev=%s' % (url, checkrev) result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev)) - self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory') + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'imraa', 'imraa.c')), 'Unable to find imraa/imraa.c in source directory') # Test devtool status result = runCmd('devtool status') self.assertIn(testrecipe, result.output) @@ -413,9 +435,8 @@ class DevtoolTests(DevtoolBase): @testcase(1164) def test_devtool_modify(self): - # Clean up anything in the workdir/sysroot/sstate cache - bitbake('mdadm -c cleansstate') - # Try modifying a recipe + import oe.path + tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(self.workspacedir) @@ -426,35 +447,95 @@ class DevtoolTests(DevtoolBase): self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'mdadm_*.bbappend')) self.assertTrue(matches, 'bbappend not created %s' % result.output) + # Test devtool status result = runCmd('devtool status') self.assertIn('mdadm', result.output) self.assertIn(tempdir, result.output) - # Check git repo self._check_src_repo(tempdir) - # Try building - bitbake('mdadm') - # Try making (minor) modifications to the source - result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % os.path.join(tempdir, 'mdadm.8.in')) - bitbake('mdadm -c package') - pkgd = get_bb_var('PKGD', 'mdadm') + + bitbake('mdadm -C unpack') + + def check_line(checkfile, expected, message, present=True): + # Check for $expected, on a line on its own, in checkfile. + with open(checkfile, 'r') as f: + if present: + self.assertIn(expected + '\n', f, message) + else: + self.assertNotIn(expected + '\n', f, message) + + modfile = os.path.join(tempdir, 'mdadm.8.in') + bb_vars = get_bb_vars(['PKGD', 'mandir'], 'mdadm') + pkgd = bb_vars['PKGD'] self.assertTrue(pkgd, 'Could not query PKGD variable') - mandir = get_bb_var('mandir', 'mdadm') + mandir = bb_vars['mandir'] self.assertTrue(mandir, 'Could not query mandir variable') - if mandir[0] == '/': - mandir = mandir[1:] - with open(os.path.join(pkgd, mandir, 'man8', 'mdadm.8'), 'r') as f: - for line in f: - if line.startswith('.TH'): - self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified. man searched file path: %s' % os.path.join(pkgd, mandir, 'man8', 'mdadm.8')) - # Test devtool reset - stampprefix = get_bb_var('STAMP', 'mdadm') + manfile = oe.path.join(pkgd, mandir, 'man8', 'mdadm.8') + + check_line(modfile, 'Linux Software RAID', 'Could not find initial string') + check_line(modfile, 'antique pin sardine', 'Unexpectedly found replacement string', present=False) + + result = runCmd("sed -i 's!^Linux Software RAID$!antique pin sardine!' %s" % modfile) + check_line(modfile, 'antique pin sardine', 'mdadm.8.in file not modified (sed failed)') + + bitbake('mdadm -c package') + check_line(manfile, 'antique pin sardine', 'man file not modified. man searched file path: %s' % manfile) + + result = runCmd('git checkout -- %s' % modfile, cwd=tempdir) + check_line(modfile, 'Linux Software RAID', 'man .in file not restored (git failed)') + + bitbake('mdadm -c package') + check_line(manfile, 'Linux Software RAID', 'man file not updated. man searched file path: %s' % manfile) + result = runCmd('devtool reset mdadm') result = runCmd('devtool status') self.assertNotIn('mdadm', result.output) - self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe mdadm') - matches = glob.glob(stampprefix + '*') - self.assertFalse(matches, 'Stamp files exist for recipe mdadm that should have been cleaned') + + def test_devtool_buildclean(self): + def assertFile(path, *paths): + f = os.path.join(path, *paths) + self.assertTrue(os.path.exists(f), "%r does not exist" % f) + def assertNoFile(path, *paths): + f = os.path.join(path, *paths) + self.assertFalse(os.path.exists(os.path.join(f)), "%r exists" % f) + + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('mdadm m4 -c cleansstate') + # Try modifying a recipe + tempdir_mdadm = tempfile.mkdtemp(prefix='devtoolqa') + tempdir_m4 = tempfile.mkdtemp(prefix='devtoolqa') + builddir_m4 = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir_mdadm) + self.track_for_cleanup(tempdir_m4) + self.track_for_cleanup(builddir_m4) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean mdadm m4') + self.write_recipeinc('m4', 'EXTERNALSRC_BUILD = "%s"\ndo_clean() {\n\t:\n}\n' % builddir_m4) + try: + runCmd('devtool modify mdadm -x %s' % tempdir_mdadm) + runCmd('devtool modify m4 -x %s' % tempdir_m4) + assertNoFile(tempdir_mdadm, 'mdadm') + assertNoFile(builddir_m4, 'src/m4') + result = bitbake('m4 -e') + result = bitbake('mdadm m4 -c compile') + self.assertEqual(result.status, 0) + assertFile(tempdir_mdadm, 'mdadm') + assertFile(builddir_m4, 'src/m4') + # Check that buildclean task exists and does call make clean + bitbake('mdadm m4 -c buildclean') + assertNoFile(tempdir_mdadm, 'mdadm') + assertNoFile(builddir_m4, 'src/m4') + bitbake('mdadm m4 -c compile') + assertFile(tempdir_mdadm, 'mdadm') + assertFile(builddir_m4, 'src/m4') + bitbake('mdadm m4 -c clean') + # Check that buildclean task is run before clean for B == S + assertNoFile(tempdir_mdadm, 'mdadm') + # Check that buildclean task is not run before clean for B != S + assertFile(builddir_m4, 'src/m4') + finally: + self.delete_recipeinc('m4') @testcase(1166) def test_devtool_modify_invalid(self): @@ -577,8 +658,8 @@ class DevtoolTests(DevtoolBase): @testcase(1378) def test_devtool_modify_virtual(self): # Try modifying a virtual recipe - virtrecipe = 'virtual/libx11' - realrecipe = 'libx11' + virtrecipe = 'virtual/make' + realrecipe = 'make' tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(self.workspacedir) @@ -601,8 +682,9 @@ class DevtoolTests(DevtoolBase): def test_devtool_update_recipe(self): # Check preconditions testrecipe = 'minicom' - recipefile = get_bb_var('FILE', testrecipe) - src_uri = get_bb_var('SRC_URI', testrecipe) + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe) self._check_repo_status(os.path.dirname(recipefile), []) # First, modify a recipe @@ -633,8 +715,9 @@ class DevtoolTests(DevtoolBase): def test_devtool_update_recipe_git(self): # Check preconditions testrecipe = 'mtd-utils' - recipefile = get_bb_var('FILE', testrecipe) - src_uri = get_bb_var('SRC_URI', testrecipe) + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) patches = [] for entry in src_uri.split(): @@ -653,7 +736,7 @@ class DevtoolTests(DevtoolBase): self._check_src_repo(tempdir) # Add a couple of commits # FIXME: this only tests adding, need to also test update and remove - result = runCmd('echo "# Additional line" >> Makefile', cwd=tempdir) + result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempdir) result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempdir) result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir) result = runCmd('git add devtool-new-file', cwd=tempdir) @@ -702,8 +785,9 @@ class DevtoolTests(DevtoolBase): def test_devtool_update_recipe_append(self): # Check preconditions testrecipe = 'mdadm' - recipefile = get_bb_var('FILE', testrecipe) - src_uri = get_bb_var('SRC_URI', testrecipe) + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe) self._check_repo_status(os.path.dirname(recipefile), []) # First, modify a recipe @@ -770,8 +854,9 @@ class DevtoolTests(DevtoolBase): def test_devtool_update_recipe_append_git(self): # Check preconditions testrecipe = 'mtd-utils' - recipefile = get_bb_var('FILE', testrecipe) - src_uri = get_bb_var('SRC_URI', testrecipe) + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) for entry in src_uri.split(): if entry.startswith('git://'): @@ -790,7 +875,7 @@ class DevtoolTests(DevtoolBase): # Check git repo self._check_src_repo(tempsrcdir) # Add a commit - result = runCmd('echo "# Additional line" >> Makefile', cwd=tempsrcdir) + result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempsrcdir) result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe)) # Create a temporary layer @@ -870,6 +955,8 @@ class DevtoolTests(DevtoolBase): result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) # Check git repo self._check_src_repo(tempdir) + # Try building just to ensure we haven't broken that + bitbake("%s" % testrecipe) # Edit / commit local source runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir) runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) @@ -926,6 +1013,78 @@ class DevtoolTests(DevtoolBase): ('??', '.*/0001-Add-new-file.patch$')] self._check_repo_status(os.path.dirname(recipefile), expected_status) + def test_devtool_update_recipe_local_files_3(self): + # First, modify the recipe + testrecipe = 'devtool-test-localonly' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + # Modify one file + runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe, 'oe-local-files')) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s/file2$' % testrecipe)] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + def test_devtool_update_recipe_local_patch_gz(self): + # First, modify the recipe + testrecipe = 'devtool-test-patch-gz' + if get_bb_var('DISTRO') == 'poky-tiny': + self.skipTest("The DISTRO 'poky-tiny' does not provide the dependencies needed by %s" % testrecipe) + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + # Modify one file + srctree = os.path.join(self.workspacedir, 'sources', testrecipe) + runCmd('echo "Another line" >> README', cwd=srctree) + runCmd('git commit -a --amend --no-edit', cwd=srctree) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s/readme.patch.gz$' % testrecipe)] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + patch_gz = os.path.join(os.path.dirname(recipefile), testrecipe, 'readme.patch.gz') + result = runCmd('file %s' % patch_gz) + if 'gzip compressed data' not in result.output: + self.fail('New patch file is not gzipped - file reports:\n%s' % result.output) + + def test_devtool_update_recipe_local_files_subdir(self): + # Try devtool extract on a recipe that has a file with subdir= set in + # SRC_URI such that it overwrites a file that was in an archive that + # was also in SRC_URI + # First, modify the recipe + testrecipe = 'devtool-test-subdir' + bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + recipefile = bb_vars['FILE'] + src_uri = bb_vars['SRC_URI'] + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + testfile = os.path.join(self.workspacedir, 'sources', testrecipe, 'testfile') + self.assertTrue(os.path.exists(testfile), 'Extracted source could not be found') + with open(testfile, 'r') as f: + contents = f.read().rstrip() + self.assertEqual(contents, 'Modified version', 'File has apparently not been overwritten as it should have been') + # Test devtool update-recipe without modifying any files + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + @testcase(1163) def test_devtool_extract(self): tempdir = tempfile.mkdtemp(prefix='devtoolqa') @@ -943,7 +1102,7 @@ class DevtoolTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract self.track_for_cleanup(tempdir) - result = runCmd('devtool extract virtual/libx11 %s' % tempdir) + result = runCmd('devtool extract virtual/make %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') # devtool extract shouldn't create the workspace self.assertFalse(os.path.exists(self.workspacedir)) @@ -1037,9 +1196,10 @@ class DevtoolTests(DevtoolBase): result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand)) # Check if it deployed all of the files with the right ownership/perms # First look on the host - need to do this under pseudo to get the correct ownership/perms - installdir = get_bb_var('D', testrecipe) - fakerootenv = get_bb_var('FAKEROOTENV', testrecipe) - fakerootcmd = get_bb_var('FAKEROOTCMD', testrecipe) + bb_vars = get_bb_vars(['D', 'FAKEROOTENV', 'FAKEROOTCMD'], testrecipe) + installdir = bb_vars['D'] + fakerootenv = bb_vars['FAKEROOTENV'] + fakerootcmd = bb_vars['FAKEROOTCMD'] result = runCmd('%s %s find . -type f -exec ls -l {} \;' % (fakerootenv, fakerootcmd), cwd=installdir) filelist1 = self._process_ls_output(result.output) @@ -1189,3 +1349,348 @@ class DevtoolTests(DevtoolBase): s = "Microsoft Made No Profit From Anyone's Zunes Yo" result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s) self.assertEqual(result.output, s[::-1]) + + def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): + dstdir = basedstdir + self.assertTrue(os.path.exists(dstdir)) + for p in paths: + dstdir = os.path.join(dstdir, p) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + self.track_for_cleanup(dstdir) + dstfile = os.path.join(dstdir, os.path.basename(srcfile)) + if srcfile != dstfile: + shutil.copy(srcfile, dstfile) + self.track_for_cleanup(dstfile) + + def test_devtool_load_plugin(self): + """Test that devtool loads only the first found plugin in BBPATH.""" + + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + devtool = runCmd("which devtool") + fromname = runCmd("devtool --quiet pluginfile") + srcfile = fromname.output + bbpath = get_bb_var('BBPATH') + searchpath = bbpath.split(':') + [os.path.dirname(devtool.output)] + plugincontent = [] + with open(srcfile) as fh: + plugincontent = fh.readlines() + try: + self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') + for path in searchpath: + self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool') + result = runCmd("devtool --quiet count") + self.assertEqual(result.output, '1') + result = runCmd("devtool --quiet multiloaded") + self.assertEqual(result.output, "no") + for path in searchpath: + result = runCmd("devtool --quiet bbdir") + self.assertEqual(result.output, path) + os.unlink(os.path.join(result.output, 'lib', 'devtool', 'bbpath.py')) + finally: + with open(srcfile, 'w') as fh: + fh.writelines(plugincontent) + + def _setup_test_devtool_finish_upgrade(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Use a "real" recipe from meta-selftest + recipe = 'devtool-upgrade-test1' + oldversion = '1.5.3' + newversion = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Do the upgrade + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, newversion)) + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "pv.h"/a \\/* Here is a new comment *\\/\' src/pv/number.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M src/pv/number.c', result.output) + result = runCmd('git commit src/pv/number.c -m "Add a comment to the code"', cwd=tempdir) + # Check if patch is there + recipedir = os.path.dirname(oldrecipefile) + olddir = os.path.join(recipedir, recipe + '-' + oldversion) + patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch' + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Original patch file does not exist') + return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn + + def test_devtool_finish_upgrade_origlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertFalse(os.path.exists(oldrecipefile), 'Old recipe file should have been deleted but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should have been deleted but wasn\'t') + newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion)) + newdir = os.path.join(recipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def test_devtool_finish_upgrade_otherlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to a different layer - should create a bbappend + # This cleanup isn't strictly necessary but do it anyway just in case it goes wrong and writes to here + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + oe_core_dir = os.path.join(get_bb_var('COREBASE'), 'meta') + newrecipedir = os.path.join(oe_core_dir, 'recipes-test', 'devtool') + newrecipefile = os.path.join(newrecipedir, '%s_%s.bb' % (recipe, newversion)) + self.track_for_cleanup(newrecipedir) + result = runCmd('devtool finish %s oe-core' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertTrue(os.path.exists(oldrecipefile), 'Old recipe file should not have been deleted') + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should not have been deleted') + newdir = os.path.join(newrecipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def _setup_test_devtool_finish_modify(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying a recipe + self.track_for_cleanup(self.workspacedir) + recipe = 'mdadm' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "mdadm.h"/a \\/* Here is a new comment *\\/\' maps.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M maps.c', result.output) + result = runCmd('git commit maps.c -m "Add a comment to the code"', cwd=tempdir) + for entry in os.listdir(recipedir): + filesdir = os.path.join(recipedir, entry) + if os.path.isdir(filesdir): + break + else: + self.fail('Unable to find recipe files directory for %s' % recipe) + return recipe, oldrecipefile, recipedir, filesdir + + def test_devtool_finish_modify_origlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)), + ('??', '.*/.*-Add-a-comment-to-the-code.patch$')] + self._check_repo_status(recipedir, expected_status) + + def test_devtool_finish_modify_otherlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + relpth = os.path.relpath(recipedir, os.path.join(get_bb_var('COREBASE'), 'meta')) + appenddir = os.path.join(get_test_layer(), relpth) + self.track_for_cleanup(appenddir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains the following unexpected changes after finish:\n%s' % (recipe, result.output.strip())) + recipefn = os.path.splitext(os.path.basename(oldrecipefile))[0] + recipefn = recipefn.split('_')[0] + '_%' + appendfile = os.path.join(appenddir, recipefn + '.bbappend') + self.assertTrue(os.path.exists(appendfile), 'bbappend %s should have been created but wasn\'t' % appendfile) + newdir = os.path.join(appenddir, recipe) + files = os.listdir(newdir) + foundpatch = None + for fn in files: + if fnmatch.fnmatch(fn, '*-Add-a-comment-to-the-code.patch'): + foundpatch = fn + if not foundpatch: + self.fail('No patch file created next to bbappend') + files.remove(foundpatch) + if files: + self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files)) + + def test_devtool_rename(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + # First run devtool add + # We already have this recipe in OE-Core, but that doesn't matter + recipename = 'i2c-tools' + recipever = '3.1.2' + recipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, recipever)) + url = 'http://downloads.yoctoproject.org/mirror/sources/i2c-tools-%s.tar.bz2' % recipever + def add_recipe(): + result = runCmd('devtool add %s' % url) + self.assertTrue(os.path.exists(recipefile), 'Expected recipe file not created') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'sources', recipename)), 'Source directory not created') + checkvars = {} + checkvars['S'] = None + checkvars['SRC_URI'] = url.replace(recipever, '${PV}') + self._test_recipe_contents(recipefile, checkvars, []) + add_recipe() + # Now rename it - change both name and version + newrecipename = 'mynewrecipe' + newrecipever = '456' + newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, newrecipever)) + result = runCmd('devtool rename %s %s -V %s' % (recipename, newrecipename, newrecipever)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file not renamed') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipename)), 'Old recipe directory still exists') + newsrctree = os.path.join(self.workspacedir, 'sources', newrecipename) + self.assertTrue(os.path.exists(newsrctree), 'Source directory not renamed') + checkvars = {} + checkvars['S'] = '${WORKDIR}/%s-%s' % (recipename, recipever) + checkvars['SRC_URI'] = url + self._test_recipe_contents(newrecipefile, checkvars, []) + # Try again - change just name this time + result = runCmd('devtool reset -n %s' % newrecipename) + shutil.rmtree(newsrctree) + add_recipe() + newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, recipever)) + result = runCmd('devtool rename %s %s' % (recipename, newrecipename)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file not renamed') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipename)), 'Old recipe directory still exists') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'sources', newrecipename)), 'Source directory not renamed') + checkvars = {} + checkvars['S'] = '${WORKDIR}/%s-${PV}' % recipename + checkvars['SRC_URI'] = url.replace(recipever, '${PV}') + self._test_recipe_contents(newrecipefile, checkvars, []) + # Try again - change just version this time + result = runCmd('devtool reset -n %s' % newrecipename) + shutil.rmtree(newsrctree) + add_recipe() + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, newrecipever)) + result = runCmd('devtool rename %s -V %s' % (recipename, newrecipever)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file not renamed') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'sources', recipename)), 'Source directory no longer exists') + checkvars = {} + checkvars['S'] = '${WORKDIR}/${BPN}-%s' % recipever + checkvars['SRC_URI'] = url + self._test_recipe_contents(newrecipefile, checkvars, []) + + @testcase(1577) + def test_devtool_virtual_kernel_modify(self): + """ + Summary: The purpose of this test case is to verify that + devtool modify works correctly when building + the kernel. + Dependencies: NA + Steps: 1. Build kernel with bitbake. + 2. Save the config file generated. + 3. Clean the environment. + 4. Use `devtool modify virtual/kernel` to validate following: + 4.1 The source is checked out correctly. + 4.2 The resulting configuration is the same as + what was get on step 2. + 4.3 The Kernel can be build correctly. + 4.4 Changes made on the source are reflected on the + subsequent builds. + 4.5 Changes on the configuration are reflected on the + subsequent builds + Expected: devtool modify is able to checkout the source of the kernel + and modification to the source and configurations are reflected + when building the kernel. + """ + #Set machine to qemxu86 to be able to modify the kernel and + #verify the modification. + features = 'MACHINE = "qemux86"\n' + self.write_config(features) + kernel_provider = get_bb_var('PREFERRED_PROVIDER_virtual/kernel') + # Clean up the enviroment + bitbake('%s -c clean' % kernel_provider) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.add_command_to_tearDown('bitbake -c clean %s' % kernel_provider) + #Step 1 + #Here is just generated the config file instead of all the kernel to optimize the + #time of executing this test case. + bitbake('%s -c configure' % kernel_provider) + bbconfig = os.path.join(get_bb_var('B', kernel_provider),'.config') + buildir= get_bb_var('TOPDIR') + #Step 2 + runCmd('cp %s %s' % (bbconfig, buildir)) + self.assertTrue(os.path.exists(os.path.join(buildir, '.config')), + 'Could not copy .config file from kernel') + + tmpconfig = os.path.join(buildir, '.config') + #Step 3 + bitbake('%s -c clean' % kernel_provider) + #Step 4.1 + runCmd('devtool modify virtual/kernel -x %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), + 'Extracted source could not be found') + #Step 4.2 + configfile = os.path.join(tempdir,'.config') + diff = runCmd('diff %s %s' % (tmpconfig, configfile)) + self.assertEqual(0,diff.status,'Kernel .config file is not the same using bitbake and devtool') + #Step 4.3 + #NOTE: virtual/kernel is mapped to kernel_provider + result = runCmd('devtool build %s' % kernel_provider) + self.assertEqual(0,result.status,'Cannot build kernel using `devtool build`') + kernelfile = os.path.join(get_bb_var('KBUILD_OUTPUT', kernel_provider), 'vmlinux') + self.assertTrue(os.path.exists(kernelfile),'Kernel was not build correctly') + + #Modify the kernel source, this is specific for qemux86 + modfile = os.path.join(tempdir,'arch/x86/boot/header.S') + modstring = "use a boot loader - Devtool kernel testing" + modapplied = runCmd("sed -i 's/boot loader/%s/' %s" % (modstring, modfile)) + self.assertEqual(0,modapplied.status,'Modification to %s on kernel source failed' % modfile) + #Modify the configuration + codeconfigfile = os.path.join(tempdir,'.config.new') + modconfopt = "CONFIG_SG_POOL=n" + modconf = runCmd("sed -i 's/CONFIG_SG_POOL=y/%s/' %s" % (modconfopt, codeconfigfile)) + self.assertEqual(0,modconf.status,'Modification to %s failed' % codeconfigfile) + #Build again kernel with devtool + rebuild = runCmd('devtool build %s' % kernel_provider) + self.assertEqual(0,rebuild.status,'Fail to build kernel after modification of source and config') + #Step 4.4 + bzimagename = 'bzImage-' + get_bb_var('KERNEL_VERSION_NAME', kernel_provider) + bzimagefile = os.path.join(get_bb_var('D', kernel_provider),'boot', bzimagename) + checkmodcode = runCmd("grep '%s' %s" % (modstring, bzimagefile)) + self.assertEqual(0,checkmodcode.status,'Modification on kernel source failed') + #Step 4.5 + checkmodconfg = runCmd("grep %s %s" % (modconfopt, codeconfigfile)) + self.assertEqual(0,checkmodconfg.status,'Modification to configuration file failed') diff --git a/meta/lib/oeqa/selftest/eSDK.py b/meta/lib/oeqa/selftest/eSDK.py new file mode 100644 index 0000000000..1596c6e9d6 --- /dev/null +++ b/meta/lib/oeqa/selftest/eSDK.py @@ -0,0 +1,115 @@ +import unittest +import tempfile +import shutil +import os +import glob +import logging +import subprocess +import oeqa.utils.ftools as ftools +from oeqa.utils.decorators import testcase +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars + +class oeSDKExtSelfTest(oeSelfTest): + """ + # Bugzilla Test Plan: 6033 + # This code is planned to be part of the automation for eSDK containig + # Install libraries and headers, image generation binary feeds, sdk-update. + """ + + @staticmethod + def get_esdk_environment(env_eSDK, tmpdir_eSDKQA): + # XXX: at this time use the first env need to investigate + # what environment load oe-selftest, i586, x86_64 + pattern = os.path.join(tmpdir_eSDKQA, 'environment-setup-*') + return glob.glob(pattern)[0] + + @staticmethod + def run_esdk_cmd(env_eSDK, tmpdir_eSDKQA, cmd, postconfig=None, **options): + if postconfig: + esdk_conf_file = os.path.join(tmpdir_eSDKQA, 'conf', 'local.conf') + with open(esdk_conf_file, 'a+') as f: + f.write(postconfig) + if not options: + options = {} + if not 'shell' in options: + options['shell'] = True + + runCmd("cd %s; . %s; %s" % (tmpdir_eSDKQA, env_eSDK, cmd), **options) + + @staticmethod + def generate_eSDK(image): + pn_task = '%s -c populate_sdk_ext' % image + bitbake(pn_task) + + @staticmethod + def get_eSDK_toolchain(image): + pn_task = '%s -c populate_sdk_ext' % image + + bb_vars = get_bb_vars(['SDK_DEPLOY', 'TOOLCHAINEXT_OUTPUTNAME'], pn_task) + sdk_deploy = bb_vars['SDK_DEPLOY'] + toolchain_name = bb_vars['TOOLCHAINEXT_OUTPUTNAME'] + return os.path.join(sdk_deploy, toolchain_name + '.sh') + + @staticmethod + def update_configuration(cls, image, tmpdir_eSDKQA, env_eSDK, ext_sdk_path): + sstate_dir = os.path.join(os.environ['BUILDDIR'], 'sstate-cache') + + oeSDKExtSelfTest.generate_eSDK(cls.image) + + cls.ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(cls.image) + runCmd("%s -y -d \"%s\"" % (cls.ext_sdk_path, cls.tmpdir_eSDKQA)) + + cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA) + + sstate_config=""" +SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS" +SSTATE_MIRRORS = "file://.* file://%s/PATH" +CORE_IMAGE_EXTRA_INSTALL = "perl" + """ % sstate_dir + + with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f: + f.write(sstate_config) + + @classmethod + def setUpClass(cls): + cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA') + + sstate_dir = get_bb_var('SSTATE_DIR') + + cls.image = 'core-image-minimal' + oeSDKExtSelfTest.generate_eSDK(cls.image) + + # Install eSDK + cls.ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(cls.image) + runCmd("%s -y -d \"%s\"" % (cls.ext_sdk_path, cls.tmpdir_eSDKQA)) + + cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA) + + # Configure eSDK to use sstate mirror from poky + sstate_config=""" +SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS" +SSTATE_MIRRORS = "file://.* file://%s/PATH" + """ % sstate_dir + with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f: + f.write(sstate_config) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdir_eSDKQA) + + @testcase (1602) + def test_install_libraries_headers(self): + pn_sstate = 'bc' + bitbake(pn_sstate) + cmd = "devtool sdk-install %s " % pn_sstate + oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd) + + @testcase(1603) + def test_image_generation_binary_feeds(self): + image = 'core-image-minimal' + cmd = "devtool build-image %s" % image + oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd) + +if __name__ == '__main__': + unittest.main() diff --git a/meta/lib/oeqa/selftest/esdk_prepare.py b/meta/lib/oeqa/selftest/esdk_prepare.py deleted file mode 100755 index 406089787d..0000000000 --- a/meta/lib/oeqa/selftest/esdk_prepare.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -import shutil, tempfile -import sys -import os -import imp -import unittest -try: - from oeqa.utils.commands import get_bb_var -except ImportError: - pass - -# module under test -module_file_name = "ext-sdk-prepare.py" -module_path = "" - -class ExtSdkPrepareTest(unittest.TestCase): - - """ unit test for fix for Yocto #9019 """ - - @classmethod - def setUpClass(self): - # copy module under test to temp dir - self.test_dir = tempfile.mkdtemp() - module_dest_path = os.path.join(self.test_dir, module_file_name) - try: - shutil.copy(module_path, self.test_dir) - # load module under test - self.test_mod = imp.load_source("", module_dest_path) - except: - print("error: unable to copy or load %s [src: %s, dst: %s]" % \ - (module_file_name, module_path, module_dest_path)) - sys.exit(1) - - def test_prepare_unexpected(self): - # test data - # note: pathnames have been truncated from the actual bitbake - # output as they are not important for the test. - test_data = ( - 'NOTE: Running noexec task 9 of 6539 (ID: 28, quilt/quilt-native_0.64.bb, do_build)\n' - 'NOTE: Running task 10 of 6539 (ID: 29, quilt/quilt-native_0.64.bb, do_package)\n' - 'NOTE: Running task 11 of 6539 (ID: 30, quilt/quilt-native_0.64.bb, do_rm_work)\n' - 'NOTE: Running noexec task 6402 of 6539 (ID: 1, images/core-image-sato.bb, do_patch)\n' - 'NOTE: Running task 6538 of 6539 (ID: 14, images/core-image-sato.bb, do_rm_work)\n' - ) - # expected warning output - expected = [ (' task 10 of 6539 (ID: 29, quilt/quilt-native_0.64.bb, do_package)') ] - # recipe to test, matching test input data - recipes = [ "core-image-sato.bb" ] - - # run the test - output = self.test_mod.check_unexpected(test_data, recipes) - self.assertEqual(output, expected) - - @classmethod - def tearDownClass(self): - # remove temp dir - shutil.rmtree(self.test_dir) - -if __name__ == '__main__': - # running from command line - i.e., not under oe-selftest - # directory containing module under test comes from command line - if len(sys.argv) == 2 and os.path.isdir(sys.argv[1]): - module_path = os.path.join(sys.argv[1], module_file_name) - suite = unittest.TestLoader().loadTestsFromTestCase(ExtSdkPrepareTest) - unittest.TextTestRunner().run(suite) - else: - progname = os.path.basename(sys.argv[0]) - print("%s: missing directory path" % progname) - print("usage: %s /path/to/directory-of(ext-sdk-prepare.py)" % progname) - sys.exit(1) -else: - # running under oe-selftest - # determine module source dir from COREBASE and expected path - module_path = os.path.join(get_bb_var("COREBASE"), "meta", "files", module_file_name) diff --git a/meta/lib/oeqa/selftest/image_typedep.py b/meta/lib/oeqa/selftest/image_typedep.py new file mode 100644 index 0000000000..256142d255 --- /dev/null +++ b/meta/lib/oeqa/selftest/image_typedep.py @@ -0,0 +1,51 @@ +import os + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake + +class ImageTypeDepTests(oeSelfTest): + + # Verify that when specifying a IMAGE_TYPEDEP_ of the form "foo.bar" that + # the conversion type bar gets added as a dep as well + def test_conversion_typedep_added(self): + + self.write_recipeinc('emptytest', """ +# Try to empty out the default dependency list +PACKAGE_INSTALL = "" +DISTRO_EXTRA_RDEPENDS="" + +LICENSE = "MIT" +IMAGE_FSTYPES = "testfstype" + +IMAGE_TYPES_MASKED += "testfstype" +IMAGE_TYPEDEP_testfstype = "tar.bz2" + +inherit image + +""") + # First get the dependency that should exist for bz2, it will look + # like CONVERSION_DEPENDS_bz2="somedep" + result = bitbake('-e emptytest') + + for line in result.output.split('\n'): + if line.startswith('CONVERSION_DEPENDS_bz2'): + dep = line.split('=')[1].strip('"') + break + + # Now get the dependency task list and check for the expected task + # dependency + bitbake('-g emptytest') + + taskdependsfile = os.path.join(self.builddir, 'task-depends.dot') + dep = dep + ".do_populate_sysroot" + depfound = False + expectedline = '"emptytest.do_rootfs" -> "{}"'.format(dep) + + with open(taskdependsfile, "r") as f: + for line in f: + if line.strip() == expectedline: + depfound = True + break + + if not depfound: + raise AssertionError("\"{}\" not found".format(expectedline)) diff --git a/meta/lib/oeqa/selftest/imagefeatures.py b/meta/lib/oeqa/selftest/imagefeatures.py index 08e382f2b7..76896c7981 100644 --- a/meta/lib/oeqa/selftest/imagefeatures.py +++ b/meta/lib/oeqa/selftest/imagefeatures.py @@ -91,9 +91,9 @@ class ImageFeatures(oeSelfTest): AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> """ - features = 'DISTRO_FEATURES_append = " wayland"\n' - features += 'CORE_IMAGE_EXTRA_INSTALL += "wayland weston"' - self.write_config(features) + distro_features = get_bb_var('DISTRO_FEATURES') + if not ('opengl' in distro_features and 'wayland' in distro_features): + self.skipTest('neither opengl nor wayland present on DISTRO_FEATURES so core-image-weston cannot be built') # Build a core-image-weston bitbake('core-image-weston') @@ -113,9 +113,9 @@ class ImageFeatures(oeSelfTest): image_name = 'core-image-minimal' bitbake(image_name) - deploy_dir = get_bb_var('DEPLOY_DIR_IMAGE') + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') link_name = get_bb_var('IMAGE_LINK_NAME', image_name) - image_path = os.path.join(deploy_dir, "%s.ext4" % link_name) + image_path = os.path.join(deploy_dir_image, "%s.ext4" % link_name) bmap_path = "%s.bmap" % image_path # check if result image and bmap file are in deploy directory diff --git a/meta/lib/oeqa/selftest/layerappend.py b/meta/lib/oeqa/selftest/layerappend.py index 4de5034a94..37bb32cd1d 100644 --- a/meta/lib/oeqa/selftest/layerappend.py +++ b/meta/lib/oeqa/selftest/layerappend.py @@ -55,7 +55,7 @@ SRC_URI_append += "file://appendtest.txt" @testcase(1196) def test_layer_appends(self): corebase = get_bb_var("COREBASE") - stagingdir = get_bb_var("STAGING_DIR_TARGET") + for l in ["0", "1", "2"]: layer = os.path.join(corebase, "meta-layertest" + l) self.assertFalse(os.path.exists(layer)) @@ -83,6 +83,7 @@ SRC_URI_append += "file://appendtest.txt" self.layerappend = "BBLAYERS += \"{0}/meta-layertest0 {0}/meta-layertest1 {0}/meta-layertest2\"".format(corebase) ftools.append_file(self.builddir + "/conf/bblayers.conf", self.layerappend) + stagingdir = get_bb_var("SYSROOT_DESTDIR", "layerappendtest") bitbake("layerappendtest") data = ftools.read_file(stagingdir + "/appendtest.txt") self.assertEqual(data, "Layer 2 test") diff --git a/meta/lib/oeqa/selftest/liboe.py b/meta/lib/oeqa/selftest/liboe.py new file mode 100644 index 0000000000..0b0301def6 --- /dev/null +++ b/meta/lib/oeqa/selftest/liboe.py @@ -0,0 +1,99 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import get_bb_var, get_bb_vars, bitbake, runCmd +import oe.path +import glob +import os +import os.path + +class LibOE(oeSelfTest): + + @classmethod + def setUpClass(cls): + cls.tmp_dir = get_bb_var('TMPDIR') + + def test_copy_tree_special(self): + """ + Summary: oe.path.copytree() should copy files with special character + Expected: 'test file with sp£c!al @nd spaces' should exist in + copy destination + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + testloc = oe.path.join(self.tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfilename = 'test file with sp£c!al @nd spaces' + + # create the test file and copy it + open(oe.path.join(src, testfilename), 'w+b').close() + oe.path.copytree(src, dst) + + # ensure path exists in dest + fileindst = os.path.isfile(oe.path.join(dst, testfilename)) + self.assertTrue(fileindst, "File with spaces doesn't exist in dst") + + oe.path.remove(testloc) + + def test_copy_tree_xattr(self): + """ + Summary: oe.path.copytree() should preserve xattr on copied files + Expected: testxattr file in destination should have user.oetest + extended attribute + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + testloc = oe.path.join(self.tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfilename = 'testxattr' + + # ensure we have setfattr available + bitbake("attr-native") + + bb_vars = get_bb_vars(['SYSROOT_DESTDIR', 'bindir'], 'attr-native') + destdir = bb_vars['SYSROOT_DESTDIR'] + bindir = bb_vars['bindir'] + bindir = destdir + bindir + + # create a file with xattr and copy it + open(oe.path.join(src, testfilename), 'w+b').close() + runCmd('%s/setfattr -n user.oetest -v "testing liboe" %s' % (bindir, oe.path.join(src, testfilename))) + oe.path.copytree(src, dst) + + # ensure file in dest has user.oetest xattr + result = runCmd('%s/getfattr -n user.oetest %s' % (bindir, oe.path.join(dst, testfilename))) + self.assertIn('user.oetest="testing liboe"', result.output, 'Extended attribute not sert in dst') + + oe.path.remove(testloc) + + def test_copy_hardlink_tree_count(self): + """ + Summary: oe.path.copyhardlinktree() shouldn't miss out files + Expected: src and dst should have the same number of files + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + testloc = oe.path.join(self.tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfiles = ['foo', 'bar', '.baz', 'quux'] + + def touchfile(tf): + open(oe.path.join(src, tf), 'w+b').close() + + for f in testfiles: + touchfile(f) + + oe.path.copyhardlinktree(src, dst) + + dstcnt = len(os.listdir(dst)) + srccnt = len(os.listdir(src)) + self.assertEquals(dstcnt, len(testfiles), "Number of files in dst (%s) differs from number of files in src(%s)." % (dstcnt, srccnt)) + + oe.path.remove(testloc) diff --git a/meta/lib/oeqa/selftest/lic-checksum.py b/meta/lib/oeqa/selftest/lic-checksum.py index df44c97e80..2e81373ae4 100644 --- a/meta/lib/oeqa/selftest/lic-checksum.py +++ b/meta/lib/oeqa/selftest/lic-checksum.py @@ -29,6 +29,7 @@ SRC_URI = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" with open(lic_path, "w") as f: f.write("data") + self.write_config("INHERIT_remove = \"report-error\"") result = bitbake(bitbake_cmd, ignore_status=True) if error_msg not in result.output: raise AssertionError(result.output) diff --git a/meta/lib/oeqa/selftest/manifest.py b/meta/lib/oeqa/selftest/manifest.py index 44d0404c5d..fe6f949644 100644 --- a/meta/lib/oeqa/selftest/manifest.py +++ b/meta/lib/oeqa/selftest/manifest.py @@ -2,7 +2,7 @@ import unittest import os from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import get_bb_var, bitbake +from oeqa.utils.commands import get_bb_var, get_bb_vars, bitbake from oeqa.utils.decorators import testcase class ManifestEntry: @@ -84,9 +84,10 @@ class VerifyManifest(oeSelfTest): try: mdir = self.get_dir_from_bb_var('SDK_DEPLOY', self.buildtarget) for k in d_target.keys(): + bb_vars = get_bb_vars(['SDK_NAME', 'SDK_VERSION'], self.buildtarget) mfilename[k] = "{}-toolchain-{}.{}.manifest".format( - get_bb_var("SDK_NAME", self.buildtarget), - get_bb_var("SDK_VERSION", self.buildtarget), + bb_vars['SDK_NAME'], + bb_vars['SDK_VERSION'], k) mpath[k] = os.path.join(mdir, mfilename[k]) if not os.path.isfile(mpath[k]): diff --git a/meta/lib/oeqa/selftest/oelib/__init__.py b/meta/lib/oeqa/selftest/oelib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/meta/lib/oeqa/selftest/oelib/__init__.py diff --git a/meta/lib/oeqa/selftest/oelib/buildhistory.py b/meta/lib/oeqa/selftest/oelib/buildhistory.py new file mode 100644 index 0000000000..5ed4b026fe --- /dev/null +++ b/meta/lib/oeqa/selftest/oelib/buildhistory.py @@ -0,0 +1,88 @@ +import os +import unittest +import tempfile +from git import Repo +from oeqa.utils.commands import get_bb_var +from oe.buildhistory_analysis import blob_to_dict, compare_dict_blobs + +class TestBlobParsing(unittest.TestCase): + + def setUp(self): + import time + self.repo_path = tempfile.mkdtemp(prefix='selftest-buildhistory', + dir=get_bb_var('TOPDIR')) + + self.repo = Repo.init(self.repo_path) + self.test_file = "test" + self.var_map = {} + + def tearDown(self): + import shutil + shutil.rmtree(self.repo_path) + + def commit_vars(self, to_add={}, to_remove = [], msg="A commit message"): + if len(to_add) == 0 and len(to_remove) == 0: + return + + for k in to_remove: + self.var_map.pop(x,None) + for k in to_add: + self.var_map[k] = to_add[k] + + with open(os.path.join(self.repo_path, self.test_file), 'w') as repo_file: + for k in self.var_map: + repo_file.write("%s = %s\n" % (k, self.var_map[k])) + + self.repo.git.add("--all") + self.repo.git.commit(message=msg) + + def test_blob_to_dict(self): + """ + Test convertion of git blobs to dictionary + """ + valuesmap = { "foo" : "1", "bar" : "2" } + self.commit_vars(to_add = valuesmap) + + blob = self.repo.head.commit.tree.blobs[0] + self.assertEqual(valuesmap, blob_to_dict(blob), + "commit was not translated correctly to dictionary") + + def test_compare_dict_blobs(self): + """ + Test comparisson of dictionaries extracted from git blobs + """ + changesmap = { "foo-2" : ("2", "8"), "bar" : ("","4"), "bar-2" : ("","5")} + + self.commit_vars(to_add = { "foo" : "1", "foo-2" : "2", "foo-3" : "3" }) + blob1 = self.repo.heads.master.commit.tree.blobs[0] + + self.commit_vars(to_add = { "foo-2" : "8", "bar" : "4", "bar-2" : "5" }) + blob2 = self.repo.heads.master.commit.tree.blobs[0] + + change_records = compare_dict_blobs(os.path.join(self.repo_path, self.test_file), + blob1, blob2, False, False) + + var_changes = { x.fieldname : (x.oldvalue, x.newvalue) for x in change_records} + self.assertEqual(changesmap, var_changes, "Changes not reported correctly") + + def test_compare_dict_blobs_default(self): + """ + Test default values for comparisson of git blob dictionaries + """ + defaultmap = { x : ("default", "1") for x in ["PKG", "PKGE", "PKGV", "PKGR"]} + + self.commit_vars(to_add = { "foo" : "1" }) + blob1 = self.repo.heads.master.commit.tree.blobs[0] + + self.commit_vars(to_add = { "PKG" : "1", "PKGE" : "1", "PKGV" : "1", "PKGR" : "1" }) + blob2 = self.repo.heads.master.commit.tree.blobs[0] + + change_records = compare_dict_blobs(os.path.join(self.repo_path, self.test_file), + blob1, blob2, False, False) + + var_changes = {} + for x in change_records: + oldvalue = "default" if ("default" in x.oldvalue) else x.oldvalue + var_changes[x.fieldname] = (oldvalue, x.newvalue) + + self.assertEqual(defaultmap, var_changes, "Defaults not set properly") diff --git a/meta/lib/oeqa/selftest/oelib/elf.py b/meta/lib/oeqa/selftest/oelib/elf.py new file mode 100644 index 0000000000..1f59037ed9 --- /dev/null +++ b/meta/lib/oeqa/selftest/oelib/elf.py @@ -0,0 +1,21 @@ +import unittest +import oe.qa + +class TestElf(unittest.TestCase): + def test_machine_name(self): + """ + Test elf_machine_to_string() + """ + self.assertEqual(oe.qa.elf_machine_to_string(0x02), "SPARC") + self.assertEqual(oe.qa.elf_machine_to_string(0x03), "x86") + self.assertEqual(oe.qa.elf_machine_to_string(0x08), "MIPS") + self.assertEqual(oe.qa.elf_machine_to_string(0x14), "PowerPC") + self.assertEqual(oe.qa.elf_machine_to_string(0x28), "ARM") + self.assertEqual(oe.qa.elf_machine_to_string(0x2A), "SuperH") + self.assertEqual(oe.qa.elf_machine_to_string(0x32), "IA-64") + self.assertEqual(oe.qa.elf_machine_to_string(0x3E), "x86-64") + self.assertEqual(oe.qa.elf_machine_to_string(0xB7), "AArch64") + + self.assertEqual(oe.qa.elf_machine_to_string(0x00), "Unknown (0)") + self.assertEqual(oe.qa.elf_machine_to_string(0xDEADBEEF), "Unknown (3735928559)") + self.assertEqual(oe.qa.elf_machine_to_string("foobar"), "Unknown ('foobar')") diff --git a/meta/lib/oe/tests/test_license.py b/meta/lib/oeqa/selftest/oelib/license.py index c388886184..c388886184 100644 --- a/meta/lib/oe/tests/test_license.py +++ b/meta/lib/oeqa/selftest/oelib/license.py diff --git a/meta/lib/oe/tests/test_path.py b/meta/lib/oeqa/selftest/oelib/path.py index 44d068143e..44d068143e 100644 --- a/meta/lib/oe/tests/test_path.py +++ b/meta/lib/oeqa/selftest/oelib/path.py diff --git a/meta/lib/oe/tests/test_types.py b/meta/lib/oeqa/selftest/oelib/types.py index 367cc30e45..4fe2746a3b 100644 --- a/meta/lib/oe/tests/test_types.py +++ b/meta/lib/oeqa/selftest/oelib/types.py @@ -1,19 +1,7 @@ import unittest -from oe.maketype import create, factory +from oe.maketype import create -class TestTypes(unittest.TestCase): - def assertIsInstance(self, obj, cls): - return self.assertTrue(isinstance(obj, cls)) - - def assertIsNot(self, obj, other): - return self.assertFalse(obj is other) - - def assertFactoryCreated(self, value, type, **flags): - cls = factory(type) - self.assertIsNot(cls, None) - self.assertIsInstance(create(value, type, **flags), cls) - -class TestBooleanType(TestTypes): +class TestBooleanType(unittest.TestCase): def test_invalid(self): self.assertRaises(ValueError, create, '', 'boolean') self.assertRaises(ValueError, create, 'foo', 'boolean') @@ -43,7 +31,7 @@ class TestBooleanType(TestTypes): self.assertEqual(create('y', 'boolean'), True) self.assertNotEqual(create('y', 'boolean'), False) -class TestList(TestTypes): +class TestList(unittest.TestCase): def assertListEqual(self, value, valid, sep=None): obj = create(value, 'list', separator=sep) self.assertEqual(obj, valid) diff --git a/meta/lib/oe/tests/test_utils.py b/meta/lib/oeqa/selftest/oelib/utils.py index 5d9ac52e7d..7deb10f3c8 100644 --- a/meta/lib/oe/tests/test_utils.py +++ b/meta/lib/oeqa/selftest/oelib/utils.py @@ -1,5 +1,5 @@ import unittest -from oe.utils import packages_filter_out_system +from oe.utils import packages_filter_out_system, trim_version class TestPackagesFilterOutSystem(unittest.TestCase): def test_filter(self): diff --git a/meta/lib/oeqa/selftest/oescripts.py b/meta/lib/oeqa/selftest/oescripts.py index 31cd50809c..29547f56a9 100644 --- a/meta/lib/oeqa/selftest/oescripts.py +++ b/meta/lib/oeqa/selftest/oescripts.py @@ -10,42 +10,10 @@ from oeqa.selftest.buildhistory import BuildhistoryBase from oeqa.utils.commands import Command, runCmd, bitbake, get_bb_var, get_test_layer from oeqa.utils.decorators import testcase -class TestScripts(oeSelfTest): - - @testcase(300) - def test_cleanup_workdir(self): - path = os.path.dirname(get_bb_var('WORKDIR', 'gzip')) - old_version_recipe = os.path.join(get_bb_var('COREBASE'), 'meta/recipes-extended/gzip/gzip_1.3.12.bb') - old_version = '1.3.12' - bitbake("-ccleansstate gzip") - bitbake("-ccleansstate -b %s" % old_version_recipe) - if os.path.exists(get_bb_var('WORKDIR', "-b %s" % old_version_recipe)): - shutil.rmtree(get_bb_var('WORKDIR', "-b %s" % old_version_recipe)) - if os.path.exists(get_bb_var('WORKDIR', 'gzip')): - shutil.rmtree(get_bb_var('WORKDIR', 'gzip')) - - if os.path.exists(path): - initial_contents = os.listdir(path) - else: - initial_contents = [] - - bitbake('gzip') - intermediary_contents = os.listdir(path) - bitbake("-b %s" % old_version_recipe) - runCmd('cleanup-workdir') - remaining_contents = os.listdir(path) - - expected_contents = [x for x in intermediary_contents if x not in initial_contents] - remaining_not_expected = [x for x in remaining_contents if x not in expected_contents] - self.assertFalse(remaining_not_expected, msg="Not all necessary content has been deleted from %s: %s" % (path, ', '.join(map(str, remaining_not_expected)))) - expected_not_remaining = [x for x in expected_contents if x not in remaining_contents] - self.assertFalse(expected_not_remaining, msg="The script removed extra contents from %s: %s" % (path, ', '.join(map(str, expected_not_remaining)))) - class BuildhistoryDiffTests(BuildhistoryBase): @testcase(295) def test_buildhistory_diff(self): - self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True) diff --git a/meta/lib/oeqa/selftest/pkgdata.py b/meta/lib/oeqa/selftest/pkgdata.py index 5a63f89ff2..d69c3c800a 100644 --- a/meta/lib/oeqa/selftest/pkgdata.py +++ b/meta/lib/oeqa/selftest/pkgdata.py @@ -6,7 +6,7 @@ import fnmatch import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars from oeqa.utils.decorators import testcase class OePkgdataUtilTests(oeSelfTest): @@ -16,21 +16,21 @@ class OePkgdataUtilTests(oeSelfTest): # Ensure we have the right data in pkgdata logger = logging.getLogger("selftest") logger.info('Running bitbake to generate pkgdata') - bitbake('glibc busybox zlib bash') + bitbake('busybox zlib m4') @testcase(1203) def test_lookup_pkg(self): # Forward tests - result = runCmd('oe-pkgdata-util lookup-pkg "glibc busybox"') - self.assertEqual(result.output, 'libc6\nbusybox') + result = runCmd('oe-pkgdata-util lookup-pkg "zlib busybox"') + self.assertEqual(result.output, 'libz1\nbusybox') result = runCmd('oe-pkgdata-util lookup-pkg zlib-dev') self.assertEqual(result.output, 'libz-dev') result = runCmd('oe-pkgdata-util lookup-pkg nonexistentpkg', ignore_status=True) self.assertEqual(result.status, 1, "Status different than 1. output: %s" % result.output) self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') # Reverse tests - result = runCmd('oe-pkgdata-util lookup-pkg -r "libc6 busybox"') - self.assertEqual(result.output, 'glibc\nbusybox') + result = runCmd('oe-pkgdata-util lookup-pkg -r "libz1 busybox"') + self.assertEqual(result.output, 'zlib\nbusybox') result = runCmd('oe-pkgdata-util lookup-pkg -r libz-dev') self.assertEqual(result.output, 'zlib-dev') result = runCmd('oe-pkgdata-util lookup-pkg -r nonexistentpkg', ignore_status=True) @@ -41,24 +41,26 @@ class OePkgdataUtilTests(oeSelfTest): def test_read_value(self): result = runCmd('oe-pkgdata-util read-value PN libz1') self.assertEqual(result.output, 'zlib') - result = runCmd('oe-pkgdata-util read-value PKGSIZE bash') + result = runCmd('oe-pkgdata-util read-value PKG libz1') + self.assertEqual(result.output, 'libz1') + result = runCmd('oe-pkgdata-util read-value PKGSIZE m4') pkgsize = int(result.output.strip()) self.assertGreater(pkgsize, 1, "Size should be greater than 1. %s" % result.output) @testcase(1198) def test_find_path(self): - result = runCmd('oe-pkgdata-util find-path /lib/libc.so.6') - self.assertEqual(result.output, 'glibc: /lib/libc.so.6') - result = runCmd('oe-pkgdata-util find-path /bin/bash') - self.assertEqual(result.output, 'bash: /bin/bash') + result = runCmd('oe-pkgdata-util find-path /lib/libz.so.1') + self.assertEqual(result.output, 'zlib: /lib/libz.so.1') + result = runCmd('oe-pkgdata-util find-path /usr/bin/m4') + self.assertEqual(result.output, 'm4: /usr/bin/m4') result = runCmd('oe-pkgdata-util find-path /not/exist', ignore_status=True) self.assertEqual(result.status, 1, "Status different than 1. output: %s" % result.output) self.assertEqual(result.output, 'ERROR: Unable to find any package producing path /not/exist') @testcase(1204) def test_lookup_recipe(self): - result = runCmd('oe-pkgdata-util lookup-recipe "libc6-staticdev busybox"') - self.assertEqual(result.output, 'glibc\nbusybox') + result = runCmd('oe-pkgdata-util lookup-recipe "libz-staticdev busybox"') + self.assertEqual(result.output, 'zlib\nbusybox') result = runCmd('oe-pkgdata-util lookup-recipe libz-dbg') self.assertEqual(result.output, 'zlib') result = runCmd('oe-pkgdata-util lookup-recipe nonexistentpkg', ignore_status=True) @@ -70,12 +72,11 @@ class OePkgdataUtilTests(oeSelfTest): # No arguments result = runCmd('oe-pkgdata-util list-pkgs') pkglist = result.output.split() - self.assertIn('glibc-utils', pkglist, "Listed packages: %s" % result.output) + self.assertIn('zlib', pkglist, "Listed packages: %s" % result.output) self.assertIn('zlib-dev', pkglist, "Listed packages: %s" % result.output) # No pkgspec, runtime result = runCmd('oe-pkgdata-util list-pkgs -r') pkglist = result.output.split() - self.assertIn('libc6-utils', pkglist, "Listed packages: %s" % result.output) self.assertIn('libz-dev', pkglist, "Listed packages: %s" % result.output) # With recipe specified result = runCmd('oe-pkgdata-util list-pkgs -p zlib') @@ -124,10 +125,11 @@ class OePkgdataUtilTests(oeSelfTest): curpkg = line.split(':')[0] files[curpkg] = [] return files - base_libdir = get_bb_var('base_libdir') - libdir = get_bb_var('libdir') - includedir = get_bb_var('includedir') - mandir = get_bb_var('mandir') + bb_vars = get_bb_vars(['base_libdir', 'libdir', 'includedir', 'mandir']) + base_libdir = bb_vars['base_libdir'] + libdir = bb_vars['libdir'] + includedir = bb_vars['includedir'] + mandir = bb_vars['mandir'] # Test recipe-space package name result = runCmd('oe-pkgdata-util list-pkg-files zlib-dev zlib-doc') files = splitoutput(result.output) @@ -205,11 +207,10 @@ class OePkgdataUtilTests(oeSelfTest): self.track_for_cleanup(tempdir) pkglistfile = os.path.join(tempdir, 'pkglist') with open(pkglistfile, 'w') as f: - f.write('libc6\n') f.write('libz1\n') f.write('busybox\n') result = runCmd('oe-pkgdata-util glob %s "*-dev"' % pkglistfile) - desiredresult = ['libc6-dev', 'libz-dev', 'busybox-dev'] + desiredresult = ['libz-dev', 'busybox-dev'] self.assertEqual(sorted(result.output.split()), sorted(desiredresult)) # The following should not error (because when we use this during rootfs construction, sometimes the complementary package won't exist) result = runCmd('oe-pkgdata-util glob %s "*-nonexistent"' % pkglistfile) @@ -222,5 +223,5 @@ class OePkgdataUtilTests(oeSelfTest): @testcase(1206) def test_specify_pkgdatadir(self): - result = runCmd('oe-pkgdata-util -p %s lookup-pkg glibc' % get_bb_var('PKGDATA_DIR')) - self.assertEqual(result.output, 'libc6') + result = runCmd('oe-pkgdata-util -p %s lookup-pkg zlib' % get_bb_var('PKGDATA_DIR')) + self.assertEqual(result.output, 'libz1') diff --git a/meta/lib/oeqa/selftest/prservice.py b/meta/lib/oeqa/selftest/prservice.py index 1b9a510fd4..34d419762c 100644 --- a/meta/lib/oeqa/selftest/prservice.py +++ b/meta/lib/oeqa/selftest/prservice.py @@ -12,10 +12,13 @@ from oeqa.utils.decorators import testcase from oeqa.utils.network import get_free_port class BitbakePrTests(oeSelfTest): - + + @classmethod + def setUpClass(cls): + cls.pkgdata_dir = get_bb_var('PKGDATA_DIR') + def get_pr_version(self, package_name): - pkgdata_dir = get_bb_var('PKGDATA_DIR') - package_data_file = os.path.join(pkgdata_dir, 'runtime', package_name) + package_data_file = os.path.join(self.pkgdata_dir, 'runtime', package_name) package_data = ftools.read_file(package_data_file) find_pr = re.search("PKGR: r[0-9]+\.([0-9]+)", package_data) self.assertTrue(find_pr, "No PKG revision found in %s" % package_data_file) @@ -37,7 +40,6 @@ class BitbakePrTests(oeSelfTest): def increment_package_pr(self, package_name): inc_data = "do_package_append() {\n bb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\"\n}" % datetime.datetime.now() self.write_recipeinc(package_name, inc_data) - bitbake("-ccleansstate %s" % package_name) res = bitbake(package_name, ignore_status=True) self.delete_recipeinc(package_name) self.assertEqual(res.status, 0, msg=res.output) @@ -60,7 +62,6 @@ class BitbakePrTests(oeSelfTest): pr_2 = self.get_pr_version(package_name) stamp_2 = self.get_task_stamp(package_name, track_task) - bitbake("-ccleansstate %s" % package_name) self.assertTrue(pr_2 - pr_1 == 1, "Step between same pkg. revision is greater than 1") self.assertTrue(stamp_1 != stamp_2, "Different pkg rev. but same stamp: %s" % stamp_1) @@ -86,7 +87,6 @@ class BitbakePrTests(oeSelfTest): self.increment_package_pr(package_name) pr_2 = self.get_pr_version(package_name) - bitbake("-ccleansstate %s" % package_name) self.assertTrue(pr_2 - pr_1 == 1, "Step between same pkg. revision is greater than 1") @testcase(930) diff --git a/meta/lib/oeqa/selftest/recipetool.py b/meta/lib/oeqa/selftest/recipetool.py index 9436301937..7bfb02f161 100644 --- a/meta/lib/oeqa/selftest/recipetool.py +++ b/meta/lib/oeqa/selftest/recipetool.py @@ -1,9 +1,11 @@ import os import logging +import shutil import tempfile import urllib.parse -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import get_bb_vars, create_temp_layer from oeqa.utils.decorators import testcase from oeqa.selftest import devtool @@ -24,6 +26,7 @@ def tearDownModule(): class RecipetoolBase(devtool.DevtoolBase): + def setUpLocal(self): self.templayerdir = templayerdir self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa') @@ -64,17 +67,16 @@ class RecipetoolBase(devtool.DevtoolBase): class RecipetoolTests(RecipetoolBase): + @classmethod def setUpClass(cls): # Ensure we have the right data in shlibs/pkgdata logger = logging.getLogger("selftest") logger.info('Running bitbake to generate pkgdata') bitbake('-c packagedata base-files coreutils busybox selftest-recipetool-appendfile') - - @classmethod - def tearDownClass(cls): - # Shouldn't leave any traces of this artificial recipe behind - bitbake('-c cleansstate selftest-recipetool-appendfile') + bb_vars = get_bb_vars(['COREBASE', 'BBPATH']) + cls.corebase = bb_vars['COREBASE'] + cls.bbpath = bb_vars['BBPATH'] def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles): cmd = 'recipetool appendfile %s %s %s %s' % (self.templayerdir, destfile, newfile, options) @@ -108,9 +110,8 @@ class RecipetoolTests(RecipetoolBase): # Now try with a file we know should be an alternative # (this is very much a fake example, but one we know is reliably an alternative) self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox']) - corebase = get_bb_var('COREBASE') # Need a test file - should be executable - testfile2 = os.path.join(corebase, 'oe-init-build-env') + testfile2 = os.path.join(self.corebase, 'oe-init-build-env') testfile2name = os.path.basename(testfile2) expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', '\n', @@ -139,7 +140,6 @@ class RecipetoolTests(RecipetoolBase): @testcase(1173) def test_recipetool_appendfile_add(self): - corebase = get_bb_var('COREBASE') # Try arbitrary file add to a recipe expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', '\n', @@ -152,7 +152,7 @@ class RecipetoolTests(RecipetoolBase): self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile']) # Try adding another file, this time where the source file is executable # (so we're testing that, plus modifying an existing bbappend) - testfile2 = os.path.join(corebase, 'oe-init-build-env') + testfile2 = os.path.join(self.corebase, 'oe-init-build-env') testfile2name = os.path.basename(testfile2) expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', '\n', @@ -369,27 +369,29 @@ class RecipetoolTests(RecipetoolBase): tempsrc = os.path.join(self.tempdir, 'srctree') os.makedirs(tempsrc) recipefile = os.path.join(self.tempdir, 'logrotate_3.8.7.bb') - srcuri = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-3.8.7.tar.gz' + srcuri = 'https://github.com/logrotate/logrotate/archive/r3-8-7.tar.gz' result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) self.assertTrue(os.path.isfile(recipefile)) checkvars = {} checkvars['LICENSE'] = 'GPLv2' checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=18810669f13b87348459e611d31ab760' - checkvars['SRC_URI'] = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-${PV}.tar.gz' - checkvars['SRC_URI[md5sum]'] = '99e08503ef24c3e2e3ff74cc5f3be213' - checkvars['SRC_URI[sha256sum]'] = 'f6ba691f40e30e640efa2752c1f9499a3f9738257660994de70a45fe00d12b64' + checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/archive/r3-8-7.tar.gz' + checkvars['SRC_URI[md5sum]'] = '6b1aa0e0d07eda3c9a2526520850397a' + checkvars['SRC_URI[sha256sum]'] = 'dece4bfeb9d8374a0ecafa34be139b5a697db5c926dcc69a9b8715431a22e733' self._test_recipe_contents(recipefile, checkvars, []) @testcase(1194) def test_recipetool_create_git(self): + if 'x11' not in get_bb_var('DISTRO_FEATURES'): + self.skipTest('Test requires x11 as distro feature') # Ensure we have the right data in shlibs/pkgdata - bitbake('libpng pango libx11 libxext jpeg libxsettings-client libcheck') + bitbake('libpng pango libx11 libxext jpeg libcheck') # Try adding a recipe tempsrc = os.path.join(self.tempdir, 'srctree') os.makedirs(tempsrc) recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') srcuri = 'git://git.yoctoproject.org/libmatchbox' - result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) + result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) checkvars = {} checkvars['LICENSE'] = 'LGPLv2.1' @@ -397,7 +399,7 @@ class RecipetoolTests(RecipetoolBase): checkvars['S'] = '${WORKDIR}/git' checkvars['PV'] = '1.11+git${SRCPV}' checkvars['SRC_URI'] = srcuri - checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxsettings-client', 'libxext', 'pango']) + checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) inherits = ['autotools', 'pkgconfig'] self._test_recipe_contents(recipefile, checkvars, inherits) @@ -442,6 +444,89 @@ class RecipetoolTests(RecipetoolBase): inherits = ['cmake', 'python-dir', 'gettext', 'pkgconfig'] self._test_recipe_contents(recipefile, checkvars, inherits) + def test_recipetool_create_github(self): + # Basic test to see if github URL mangling works + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'meson_git.bb') + srcuri = 'https://github.com/mesonbuild/meson' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Apache-2.0']) + checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https' + inherits = ['setuptools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_github_tarball(self): + # Basic test to ensure github URL mangling doesn't apply to release tarballs + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pv = '0.32.0' + recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv) + srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Apache-2.0']) + checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz' + inherits = ['setuptools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_git_http(self): + # Basic test to check http git URL mangling works + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'matchbox-terminal_git.bb') + srcuri = 'http://git.yoctoproject.org/git/matchbox-terminal' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['GPLv2']) + checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/matchbox-terminal;protocol=http' + inherits = ['pkgconfig', 'autotools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): + dstdir = basedstdir + self.assertTrue(os.path.exists(dstdir)) + for p in paths: + dstdir = os.path.join(dstdir, p) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + self.track_for_cleanup(dstdir) + dstfile = os.path.join(dstdir, os.path.basename(srcfile)) + if srcfile != dstfile: + shutil.copy(srcfile, dstfile) + self.track_for_cleanup(dstfile) + + def test_recipetool_load_plugin(self): + """Test that recipetool loads only the first found plugin in BBPATH.""" + + recipetool = runCmd("which recipetool") + fromname = runCmd("recipetool --quiet pluginfile") + srcfile = fromname.output + searchpath = self.bbpath.split(':') + [os.path.dirname(recipetool.output)] + plugincontent = [] + with open(srcfile) as fh: + plugincontent = fh.readlines() + try: + self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') + for path in searchpath: + self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') + result = runCmd("recipetool --quiet count") + self.assertEqual(result.output, '1') + result = runCmd("recipetool --quiet multiloaded") + self.assertEqual(result.output, "no") + for path in searchpath: + result = runCmd("recipetool --quiet bbdir") + self.assertEqual(result.output, path) + os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py')) + finally: + with open(srcfile, 'w') as fh: + fh.writelines(plugincontent) + + class RecipetoolAppendsrcBase(RecipetoolBase): def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile) @@ -517,20 +602,23 @@ class RecipetoolAppendsrcBase(RecipetoolBase): self._try_recipetool_appendsrcfiles(testrecipe, newfiles, expectedfiles=expectedfiles, destdir=destdir, options=options) - src_uri = get_bb_var('SRC_URI', testrecipe).split() + bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'FILESEXTRAPATHS'], testrecipe) + src_uri = bb_vars['SRC_URI'].split() for f in expectedfiles: if destdir: self.assertIn('file://%s;subdir=%s' % (f, destdir), src_uri) else: self.assertIn('file://%s' % f, src_uri) - recipefile = get_bb_var('FILE', testrecipe) + recipefile = bb_vars['FILE'] bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) - filesextrapaths = get_bb_var('FILESEXTRAPATHS', testrecipe).split(':') + filesextrapaths = bb_vars['FILESEXTRAPATHS'].split(':') self.assertIn(filesdir, filesextrapaths) + + class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): @testcase(1273) @@ -556,8 +644,9 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): @testcase(1280) def test_recipetool_appendsrcfile_srcdir_basic(self): testrecipe = 'bash' - srcdir = get_bb_var('S', testrecipe) - workdir = get_bb_var('WORKDIR', testrecipe) + bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) + srcdir = bb_vars['S'] + workdir = bb_vars['WORKDIR'] subdir = os.path.relpath(srcdir, workdir) self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir) @@ -582,8 +671,9 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): def test_recipetool_appendsrcfile_replace_file_srcdir(self): testrecipe = 'bash' filepath = 'Makefile.in' - srcdir = get_bb_var('S', testrecipe) - workdir = get_bb_var('WORKDIR', testrecipe) + bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) + srcdir = bb_vars['S'] + workdir = bb_vars['WORKDIR'] subdir = os.path.relpath(srcdir, workdir) self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir) diff --git a/meta/lib/oeqa/selftest/runqemu.py b/meta/lib/oeqa/selftest/runqemu.py new file mode 100644 index 0000000000..58c6f96f98 --- /dev/null +++ b/meta/lib/oeqa/selftest/runqemu.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2017 Wind River Systems, Inc. +# + +import re +import logging + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake, runqemu, get_bb_var +from oeqa.utils.decorators import testcase + +class RunqemuTests(oeSelfTest): + """Runqemu test class""" + + image_is_ready = False + deploy_dir_image = '' + + def setUpLocal(self): + self.recipe = 'core-image-minimal' + self.machine = 'qemux86-64' + self.fstypes = "ext4 iso hddimg vmdk qcow2 vdi" + self.cmd_common = "runqemu nographic" + + # Avoid emit the same record multiple times. + mainlogger = logging.getLogger("BitBake.Main") + mainlogger.propagate = False + + self.write_config( +""" +MACHINE = "%s" +IMAGE_FSTYPES = "%s" +# 10 means 1 second +SYSLINUX_TIMEOUT = "10" +""" +% (self.machine, self.fstypes) + ) + + if not RunqemuTests.image_is_ready: + RunqemuTests.deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + bitbake(self.recipe) + RunqemuTests.image_is_ready = True + + @testcase(2001) + def test_boot_machine(self): + """Test runqemu machine""" + cmd = "%s %s" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2002) + def test_boot_machine_ext4(self): + """Test runqemu machine ext4""" + cmd = "%s %s ext4" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('rootfs.ext4' in f.read(), "Failed: %s" % cmd) + + @testcase(2003) + def test_boot_machine_iso(self): + """Test runqemu machine iso""" + cmd = "%s %s iso" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue(' -cdrom ' in f.read(), "Failed: %s" % cmd) + + @testcase(2004) + def test_boot_recipe_image(self): + """Test runqemu recipe-image""" + cmd = "%s %s" % (self.cmd_common, self.recipe) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2005) + def test_boot_recipe_image_vmdk(self): + """Test runqemu recipe-image vmdk""" + cmd = "%s %s vmdk" % (self.cmd_common, self.recipe) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('format=vmdk' in f.read(), "Failed: %s" % cmd) + + @testcase(2006) + def test_boot_recipe_image_vdi(self): + """Test runqemu recipe-image vdi""" + cmd = "%s %s vdi" % (self.cmd_common, self.recipe) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('format=vdi' in f.read(), "Failed: %s" % cmd) + + @testcase(2007) + def test_boot_deploy(self): + """Test runqemu deploy_dir_image""" + cmd = "%s %s" % (self.cmd_common, self.deploy_dir_image) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2008) + def test_boot_deploy_hddimg(self): + """Test runqemu deploy_dir_image hddimg""" + cmd = "%s %s hddimg" % (self.cmd_common, self.deploy_dir_image) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s" % cmd) + + @testcase(2009) + def test_boot_machine_slirp(self): + """Test runqemu machine slirp""" + cmd = "%s slirp %s" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue(' -netdev user' in f.read(), "Failed: %s" % cmd) + + @testcase(2009) + def test_boot_machine_slirp_qcow2(self): + """Test runqemu machine slirp qcow2""" + cmd = "%s slirp qcow2 %s" % (self.cmd_common, self.machine) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + with open(qemu.qemurunnerlog) as f: + self.assertTrue('format=qcow2' in f.read(), "Failed: %s" % cmd) + + @testcase(2010) + def test_boot_qemu_boot(self): + """Test runqemu /path/to/image.qemuboot.conf""" + qemuboot_conf = "%s-%s.qemuboot.conf" % (self.recipe, self.machine) + qemuboot_conf = os.path.join(self.deploy_dir_image, qemuboot_conf) + if not os.path.exists(qemuboot_conf): + self.skipTest("%s not found" % qemuboot_conf) + cmd = "%s %s" % (self.cmd_common, qemuboot_conf) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + + @testcase(2011) + def test_boot_rootfs(self): + """Test runqemu /path/to/rootfs.ext4""" + rootfs = "%s-%s.ext4" % (self.recipe, self.machine) + rootfs = os.path.join(self.deploy_dir_image, rootfs) + if not os.path.exists(rootfs): + self.skipTest("%s not found" % rootfs) + cmd = "%s %s" % (self.cmd_common, rootfs) + with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: + self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) diff --git a/meta/lib/oeqa/selftest/runtime-test.py b/meta/lib/oeqa/selftest/runtime-test.py new file mode 100644 index 0000000000..e498d046cf --- /dev/null +++ b/meta/lib/oeqa/selftest/runtime-test.py @@ -0,0 +1,237 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu +from oeqa.utils.decorators import testcase +import os +import re + +class TestExport(oeSelfTest): + + @classmethod + def tearDownClass(cls): + runCmd("rm -rf /tmp/sdk") + + def test_testexport_basic(self): + """ + Summary: Check basic testexport functionality with only ping test enabled. + Expected: 1. testexport directory must be created. + 2. runexported.py must run without any error/exception. + 3. ping test must succeed. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testexport"\n' + # These aren't the actual IP addresses but testexport class needs something defined + features += 'TEST_SERVER_IP = "192.168.7.1"\n' + features += 'TEST_TARGET_IP = "192.168.7.1"\n' + features += 'TEST_SUITES = "ping"\n' + self.write_config(features) + + # Build tesexport for core-image-minimal + bitbake('core-image-minimal') + bitbake('-c testexport core-image-minimal') + + testexport_dir = get_bb_var('TEST_EXPORT_DIR', 'core-image-minimal') + + # Verify if TEST_EXPORT_DIR was created + isdir = os.path.isdir(testexport_dir) + self.assertEqual(True, isdir, 'Failed to create testexport dir: %s' % testexport_dir) + + with runqemu('core-image-minimal') as qemu: + # Attempt to run runexported.py to perform ping test + test_path = os.path.join(testexport_dir, "oe-test") + data_file = os.path.join(testexport_dir, 'data', 'testdata.json') + manifest = os.path.join(testexport_dir, 'data', 'manifest') + cmd = ("%s runtime --test-data-file %s --packages-manifest %s " + "--target-ip %s --server-ip %s --quiet" + % (test_path, data_file, manifest, qemu.ip, qemu.server_ip)) + result = runCmd(cmd) + # Verify ping test was succesful + self.assertEqual(0, result.status, 'oe-test runtime returned a non 0 status') + + def test_testexport_sdk(self): + """ + Summary: Check sdk functionality for testexport. + Expected: 1. testexport directory must be created. + 2. SDK tarball must exists. + 3. Uncompressing of tarball must succeed. + 4. Check if the SDK directory is added to PATH. + 5. Run tar from the SDK directory. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testexport"\n' + # These aren't the actual IP addresses but testexport class needs something defined + features += 'TEST_SERVER_IP = "192.168.7.1"\n' + features += 'TEST_TARGET_IP = "192.168.7.1"\n' + features += 'TEST_SUITES = "ping"\n' + features += 'TEST_EXPORT_SDK_ENABLED = "1"\n' + features += 'TEST_EXPORT_SDK_PACKAGES = "nativesdk-tar"\n' + self.write_config(features) + + # Build tesexport for core-image-minimal + bitbake('core-image-minimal') + bitbake('-c testexport core-image-minimal') + + needed_vars = ['TEST_EXPORT_DIR', 'TEST_EXPORT_SDK_DIR', 'TEST_EXPORT_SDK_NAME'] + bb_vars = get_bb_vars(needed_vars, 'core-image-minimal') + testexport_dir = bb_vars['TEST_EXPORT_DIR'] + sdk_dir = bb_vars['TEST_EXPORT_SDK_DIR'] + sdk_name = bb_vars['TEST_EXPORT_SDK_NAME'] + + # Check for SDK + tarball_name = "%s.sh" % sdk_name + tarball_path = os.path.join(testexport_dir, sdk_dir, tarball_name) + msg = "Couldn't find SDK tarball: %s" % tarball_path + self.assertEqual(os.path.isfile(tarball_path), True, msg) + + # Extract SDK and run tar from SDK + result = runCmd("%s -y -d /tmp/sdk" % tarball_path) + self.assertEqual(0, result.status, "Couldn't extract SDK") + + env_script = result.output.split()[-1] + result = runCmd(". %s; which tar" % env_script, shell=True) + self.assertEqual(0, result.status, "Couldn't setup SDK environment") + is_sdk_tar = True if "/tmp/sdk" in result.output else False + self.assertTrue(is_sdk_tar, "Couldn't setup SDK environment") + + tar_sdk = result.output + result = runCmd("%s --version" % tar_sdk) + self.assertEqual(0, result.status, "Couldn't run tar from SDK") + + +class TestImage(oeSelfTest): + + def test_testimage_install(self): + """ + Summary: Check install packages functionality for testimage/testexport. + Expected: 1. Import tests from a directory other than meta. + 2. Check install/uninstall of socat. + 3. Check that remote package feeds can be accessed + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + Author: Alexander Kanavin <alexander.kanavin@intel.com> + """ + if get_bb_var('DISTRO') == 'poky-tiny': + self.skipTest('core-image-full-cmdline not buildable for poky-tiny') + + features = 'INHERIT += "testimage"\n' + features += 'TEST_SUITES = "ping ssh selftest"\n' + # We don't yet know what the server ip and port will be - they will be patched + # in at the start of the on-image test + features += 'PACKAGE_FEED_URIS = "http://bogus_ip:bogus_port"\n' + features += 'EXTRA_IMAGE_FEATURES += "package-management"\n' + features += 'PACKAGE_CLASSES = "package_rpm"' + self.write_config(features) + + # Build core-image-sato and testimage + bitbake('core-image-full-cmdline socat') + bitbake('-c testimage core-image-full-cmdline') + +class Postinst(oeSelfTest): + @testcase(1540) + def test_verify_postinst(self): + """ + Summary: The purpose of this test is to verify the execution order of postinst Bugzilla ID: [5319] + Expected : + 1. Compile a minimal image. + 2. The compiled image will add the created layer with the recipes postinst[ abdpt] + 3. Run qemux86 + 4. Validate the task execution order + Author: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com> + """ + features = 'INHERIT += "testimage"\n' + features += 'CORE_IMAGE_EXTRA_INSTALL += "postinst-at-rootfs \ +postinst-delayed-a \ +postinst-delayed-b \ +postinst-delayed-d \ +postinst-delayed-p \ +postinst-delayed-t \ +"\n' + self.write_config(features) + + bitbake('core-image-minimal -f ') + + postinst_list = ['100-postinst-at-rootfs', + '101-postinst-delayed-a', + '102-postinst-delayed-b', + '103-postinst-delayed-d', + '104-postinst-delayed-p', + '105-postinst-delayed-t'] + path_workdir = get_bb_var('WORKDIR','core-image-minimal') + workspacedir = 'testimage/qemu_boot_log' + workspacedir = os.path.join(path_workdir, workspacedir) + rexp = re.compile("^Running postinst .*/(?P<postinst>.*)\.\.\.$") + with runqemu('core-image-minimal') as qemu: + with open(workspacedir) as f: + found = False + idx = 0 + for line in f.readlines(): + line = line.strip().replace("^M","") + if not line: # To avoid empty lines + continue + m = rexp.search(line) + if m: + self.assertEqual(postinst_list[idx], m.group('postinst'), "Fail") + idx = idx+1 + found = True + elif found: + self.assertEqual(idx, len(postinst_list), "Not found all postinsts") + break + + @testcase(1545) + def test_postinst_rootfs_and_boot(self): + """ + Summary: The purpose of this test case is to verify Post-installation + scripts are called when rootfs is created and also test + that script can be delayed to run at first boot. + Dependencies: NA + Steps: 1. Add proper configuration to local.conf file + 2. Build a "core-image-minimal" image + 3. Verify that file created by postinst_rootfs recipe is + present on rootfs dir. + 4. Boot the image created on qemu and verify that the file + created by postinst_boot recipe is present on image. + Expected: The files are successfully created during rootfs and boot + time for 3 different package managers: rpm,ipk,deb and + for initialization managers: sysvinit and systemd. + + """ + file_rootfs_name = "this-was-created-at-rootfstime" + fileboot_name = "this-was-created-at-first-boot" + rootfs_pkg = 'postinst-at-rootfs' + boot_pkg = 'postinst-delayed-a' + #Step 1 + features = 'MACHINE = "qemux86"\n' + features += 'CORE_IMAGE_EXTRA_INSTALL += "%s %s "\n'% (rootfs_pkg, boot_pkg) + features += 'IMAGE_FEATURES += "ssh-server-openssh"\n' + for init_manager in ("sysvinit", "systemd"): + #for sysvinit no extra configuration is needed, + if (init_manager is "systemd"): + features += 'DISTRO_FEATURES_append = " systemd"\n' + features += 'VIRTUAL-RUNTIME_init_manager = "systemd"\n' + features += 'DISTRO_FEATURES_BACKFILL_CONSIDERED = "sysvinit"\n' + features += 'VIRTUAL-RUNTIME_initscripts = ""\n' + for classes in ("package_rpm package_deb package_ipk", + "package_deb package_rpm package_ipk", + "package_ipk package_deb package_rpm"): + features += 'PACKAGE_CLASSES = "%s"\n' % classes + self.write_config(features) + + #Step 2 + bitbake('core-image-minimal') + + #Step 3 + file_rootfs_created = os.path.join(get_bb_var('IMAGE_ROOTFS',"core-image-minimal"), + file_rootfs_name) + found = os.path.isfile(file_rootfs_created) + self.assertTrue(found, "File %s was not created at rootfs time by %s" % \ + (file_rootfs_name, rootfs_pkg)) + + #Step 4 + testcommand = 'ls /etc/'+fileboot_name + with runqemu('core-image-minimal') as qemu: + sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand)) + self.assertEqual(result.status, 0, 'File %s was not created at firts boot'% fileboot_name) diff --git a/meta/lib/oeqa/selftest/signing.py b/meta/lib/oeqa/selftest/signing.py index 1babca07df..0ac3d1fac9 100644 --- a/meta/lib/oeqa/selftest/signing.py +++ b/meta/lib/oeqa/selftest/signing.py @@ -1,5 +1,5 @@ from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import os import glob import re @@ -12,38 +12,32 @@ from oeqa.utils.ftools import write_file class Signing(oeSelfTest): gpg_dir = "" - pub_key_name = 'key.pub' - secret_key_name = 'key.secret' + pub_key_path = "" + secret_key_path = "" @classmethod def setUpClass(cls): - # Import the gpg keys + # Check that we can find the gpg binary and fail early if we can't + if not shutil.which("gpg"): + raise AssertionError("This test needs GnuPG") - cls.gpg_dir = os.path.join(cls.testlayer_path, 'files/signing/') + cls.gpg_home_dir = tempfile.TemporaryDirectory(prefix="oeqa-signing-") + cls.gpg_dir = cls.gpg_home_dir.name - # key.secret key.pub are located in gpg_dir - pub_key_location = cls.gpg_dir + cls.pub_key_name - secret_key_location = cls.gpg_dir + cls.secret_key_name - runCmd('gpg --homedir %s --import %s %s' % (cls.gpg_dir, pub_key_location, secret_key_location)) + cls.pub_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.pub") + cls.secret_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.secret") - @classmethod - def tearDownClass(cls): - # Delete the files generated by 'gpg --import' - - gpg_files = glob.glob(cls.gpg_dir + '*.gpg*') - random_seed_file = cls.gpg_dir + 'random_seed' - gpg_files.append(random_seed_file) - - for gpg_file in gpg_files: - runCmd('rm -f ' + gpg_file) + runCmd('gpg --batch --homedir %s --import %s %s' % (cls.gpg_dir, cls.pub_key_path, cls.secret_key_path)) @testcase(1362) def test_signing_packages(self): """ Summary: Test that packages can be signed in the package feed Expected: Package should be signed with the correct key + Expected: Images can be created from signed packages Product: oe-core Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + Author: Alexander Kanavin <alexander.kanavin@intel.com> AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> """ import oe.packagedata @@ -57,39 +51,47 @@ class Signing(oeSelfTest): feature = 'INHERIT += "sign_rpm"\n' feature += 'RPM_GPG_PASSPHRASE = "test123"\n' feature += 'RPM_GPG_NAME = "testuser"\n' - feature += 'RPM_GPG_PUBKEY = "%s%s"\n' % (self.gpg_dir, self.pub_key_name) feature += 'GPG_PATH = "%s"\n' % self.gpg_dir self.write_config(feature) - bitbake('-c cleansstate %s' % test_recipe) - bitbake(test_recipe) + bitbake('-c clean %s' % test_recipe) + bitbake('-f -c package_write_rpm %s' % test_recipe) + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) - pkgdatadir = get_bb_var('PKGDATA_DIR', test_recipe) + needed_vars = ['PKGDATA_DIR', 'DEPLOY_DIR_RPM', 'PACKAGE_ARCH', 'STAGING_BINDIR_NATIVE'] + bb_vars = get_bb_vars(needed_vars, test_recipe) + pkgdatadir = bb_vars['PKGDATA_DIR'] pkgdata = oe.packagedata.read_pkgdatafile(pkgdatadir + "/runtime/ed") if 'PKGE' in pkgdata: pf = pkgdata['PN'] + "-" + pkgdata['PKGE'] + pkgdata['PKGV'] + '-' + pkgdata['PKGR'] else: pf = pkgdata['PN'] + "-" + pkgdata['PKGV'] + '-' + pkgdata['PKGR'] - deploy_dir_rpm = get_bb_var('DEPLOY_DIR_RPM', test_recipe) - package_arch = get_bb_var('PACKAGE_ARCH', test_recipe).replace('-', '_') - staging_bindir_native = get_bb_var('STAGING_BINDIR_NATIVE') + deploy_dir_rpm = bb_vars['DEPLOY_DIR_RPM'] + package_arch = bb_vars['PACKAGE_ARCH'].replace('-', '_') + staging_bindir_native = bb_vars['STAGING_BINDIR_NATIVE'] pkg_deploy = os.path.join(deploy_dir_rpm, package_arch, '.'.join((pf, package_arch, 'rpm'))) # Use a temporary rpmdb rpmdb = tempfile.mkdtemp(prefix='oeqa-rpmdb') - runCmd('%s/rpm --define "_dbpath %s" --import %s%s' % - (staging_bindir_native, rpmdb, self.gpg_dir, self.pub_key_name)) + runCmd('%s/rpmkeys --define "_dbpath %s" --import %s' % + (staging_bindir_native, rpmdb, self.pub_key_path)) - ret = runCmd('%s/rpm --define "_dbpath %s" --checksig %s' % + ret = runCmd('%s/rpmkeys --define "_dbpath %s" --checksig %s' % (staging_bindir_native, rpmdb, pkg_deploy)) # tmp/deploy/rpm/i586/ed-1.9-r0.i586.rpm: rsa sha1 md5 OK - self.assertIn('rsa sha1 md5 OK', ret.output, 'Package signed incorrectly.') + self.assertIn('rsa sha1 (md5) pgp md5 OK', ret.output, 'Package signed incorrectly.') shutil.rmtree(rpmdb) + #Check that an image can be built from signed packages + self.add_command_to_tearDown('bitbake -c clean core-image-minimal') + bitbake('-c clean core-image-minimal') + bitbake('core-image-minimal') + + @testcase(1382) def test_signing_sstate_archive(self): """ @@ -106,24 +108,19 @@ class Signing(oeSelfTest): sstatedir = os.path.join(builddir, 'test-sstate') self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) - self.add_command_to_tearDown('bitbake -c cleansstate %s' % test_recipe) self.add_command_to_tearDown('rm -rf %s' % sstatedir) - # Determine the pub key signature - ret = runCmd('gpg --homedir %s --list-keys' % self.gpg_dir) - pub_key = re.search(r'^pub\s+\S+/(\S+)\s+', ret.output, re.M) - self.assertIsNotNone(pub_key, 'Failed to determine the public key signature.') - pub_key = pub_key.group(1) - - feature = 'SSTATE_SIG_KEY ?= "%s"\n' % pub_key + feature = 'SSTATE_SIG_KEY ?= "testuser"\n' feature += 'SSTATE_SIG_PASSPHRASE ?= "test123"\n' feature += 'SSTATE_VERIFY_SIG ?= "1"\n' feature += 'GPG_PATH = "%s"\n' % self.gpg_dir feature += 'SSTATE_DIR = "%s"\n' % sstatedir + # Any mirror might have partial sstate without .sig files, triggering failures + feature += 'SSTATE_MIRRORS_forcevariable = ""\n' self.write_config(feature) - bitbake('-c cleansstate %s' % test_recipe) + bitbake('-c clean %s' % test_recipe) bitbake(test_recipe) recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_package.tgz.sig') diff --git a/meta/lib/oeqa/selftest/sstate.py b/meta/lib/oeqa/selftest/sstate.py index 5989724432..f54bc41465 100644 --- a/meta/lib/oeqa/selftest/sstate.py +++ b/meta/lib/oeqa/selftest/sstate.py @@ -6,16 +6,24 @@ import shutil import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer +from oeqa.utils.commands import runCmd, bitbake, get_bb_vars, get_test_layer class SStateBase(oeSelfTest): def setUpLocal(self): self.temp_sstate_location = None - self.sstate_path = get_bb_var('SSTATE_DIR') - self.distro = get_bb_var('NATIVELSBSTRING') - self.distro_specific_sstate = os.path.join(self.sstate_path, self.distro) + needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', + 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] + bb_vars = get_bb_vars(needed_vars) + self.sstate_path = bb_vars['SSTATE_DIR'] + self.hostdistro = bb_vars['NATIVELSBSTRING'] + self.tclibc = bb_vars['TCLIBC'] + self.tune_arch = bb_vars['TUNE_ARCH'] + self.topdir = bb_vars['TOPDIR'] + self.target_vendor = bb_vars['TARGET_VENDOR'] + self.target_os = bb_vars['TARGET_OS'] + self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) # Creates a special sstate configuration with the option to add sstate mirrors def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): @@ -26,9 +34,10 @@ class SStateBase(oeSelfTest): config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path self.append_config(config_temp_sstate) self.track_for_cleanup(temp_sstate_path) - self.sstate_path = get_bb_var('SSTATE_DIR') - self.distro = get_bb_var('NATIVELSBSTRING') - self.distro_specific_sstate = os.path.join(self.sstate_path, self.distro) + bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) + self.sstate_path = bb_vars['SSTATE_DIR'] + self.hostdistro = bb_vars['NATIVELSBSTRING'] + self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) if add_local_mirrors: config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' @@ -42,7 +51,7 @@ class SStateBase(oeSelfTest): def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): result = [] for root, dirs, files in os.walk(self.sstate_path): - if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.distro, root): + if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.hostdistro, root): for f in files: if re.search(filename_regex, f): result.append(f) diff --git a/meta/lib/oeqa/selftest/sstatetests.py b/meta/lib/oeqa/selftest/sstatetests.py index 07212ac4f1..e35ddfff5f 100644 --- a/meta/lib/oeqa/selftest/sstatetests.py +++ b/meta/lib/oeqa/selftest/sstatetests.py @@ -16,7 +16,7 @@ class SStateTests(SStateBase): # Test sstate files creation and their location def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): - self.config_sstate(temp_sstate_location) + self.config_sstate(temp_sstate_location, [self.sstate_path]) if self.temp_sstate_location: bitbake(['-cclean'] + targets) @@ -41,26 +41,23 @@ class SStateTests(SStateBase): @testcase(975) def test_sstate_creation_distro_specific_pass(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) @testcase(1374) def test_sstate_creation_distro_specific_fail(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) + self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) @testcase(976) def test_sstate_creation_distro_nonspecific_pass(self): - self.run_test_sstate_creation(['glibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) @testcase(1375) def test_sstate_creation_distro_nonspecific_fail(self): - self.run_test_sstate_creation(['glibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) - + self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) # Test the sstate files deletion part of the do_cleansstate task def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): - self.config_sstate(temp_sstate_location) + self.config_sstate(temp_sstate_location, [self.sstate_path]) bitbake(['-ccleansstate'] + targets) @@ -77,22 +74,24 @@ class SStateTests(SStateBase): @testcase(977) def test_cleansstate_task_distro_specific_nonspecific(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_cleansstate_task(['binutils-cross-' + targetarch, 'binutils-native', 'glibc-initial'], distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) + targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] + targets.append('linux-libc-headers') + self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) @testcase(1376) def test_cleansstate_task_distro_nonspecific(self): - self.run_test_cleansstate_task(['glibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) + self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) @testcase(1377) def test_cleansstate_task_distro_specific(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_cleansstate_task(['binutils-cross-'+ targetarch, 'binutils-native', 'glibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) + targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] + targets.append('linux-libc-headers') + self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) # Test rebuilding of distro-specific sstate files def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): - self.config_sstate(temp_sstate_location) + self.config_sstate(temp_sstate_location, [self.sstate_path]) bitbake(['-ccleansstate'] + targets) @@ -124,13 +123,11 @@ class SStateTests(SStateBase): @testcase(175) def test_rebuild_distro_specific_sstate_cross_native_targets(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch, 'binutils-native'], temp_sstate_location=True) + self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) @testcase(1372) def test_rebuild_distro_specific_sstate_cross_target(self): - targetarch = get_bb_var('TUNE_ARCH') - self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch], temp_sstate_location=True) + self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch], temp_sstate_location=True) @testcase(1373) def test_rebuild_distro_specific_sstate_native_target(self): @@ -145,10 +142,9 @@ class SStateTests(SStateBase): self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) - # If buildhistory is enabled, we need to disable version-going-backwards QA checks for this test. It may report errors otherwise. - if ('buildhistory' in get_bb_var('USER_CLASSES')) or ('buildhistory' in get_bb_var('INHERIT')): - remove_errors_config = 'ERROR_QA_remove = "version-going-backwards"' - self.append_config(remove_errors_config) + # If buildhistory is enabled, we need to disable version-going-backwards + # QA checks for this test. It may report errors otherwise. + self.append_config('ERROR_QA_remove = "version-going-backwards"') # For not this only checks if random sstate tasks are handled correctly as a group. # In the future we should add control over what tasks we check for. @@ -229,16 +225,15 @@ class SStateTests(SStateBase): manually and check using bitbake -S. """ - topdir = get_bb_var('TOPDIR') - targetvendor = get_bb_var('TARGET_VENDOR') self.write_config(""" MACHINE = "qemux86" TMPDIR = "${TOPDIR}/tmp-sstatesamehash" BUILD_ARCH = "x86_64" BUILD_OS = "linux" SDKMACHINE = "x86_64" +PACKAGE_CLASSES = "package_rpm package_ipk package_deb" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") bitbake("core-image-sato -S none") self.write_config(""" MACHINE = "qemux86" @@ -246,8 +241,9 @@ TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" BUILD_ARCH = "i686" BUILD_OS = "linux" SDKMACHINE = "i686" +PACKAGE_CLASSES = "package_rpm package_ipk package_deb" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") bitbake("core-image-sato -S none") def get_files(d): @@ -260,9 +256,9 @@ SDKMACHINE = "i686" continue f.extend(os.path.join(root, name) for name in files) return f - files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") - files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") - files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + targetvendor + "-linux", "x86_64" + targetvendor + "-linux", ) for x in files2] + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + self.target_vendor + "-linux", "x86_64" + self.target_vendor + "-linux", ) for x in files2] self.maxDiff = None self.assertCountEqual(files1, files2) @@ -275,18 +271,17 @@ SDKMACHINE = "i686" builds, override the variables manually and check using bitbake -S. """ - topdir = get_bb_var('TOPDIR') self.write_config(""" TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" NATIVELSBSTRING = \"DistroA\" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") bitbake("core-image-sato -S none") self.write_config(""" TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" NATIVELSBSTRING = \"DistroB\" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") bitbake("core-image-sato -S none") def get_files(d): @@ -294,8 +289,8 @@ NATIVELSBSTRING = \"DistroB\" for root, dirs, files in os.walk(d): f.extend(os.path.join(root, name) for name in files) return f - files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") - files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None self.assertCountEqual(files1, files2) @@ -309,20 +304,46 @@ NATIVELSBSTRING = \"DistroB\" the two MACHINE values. """ - topdir = get_bb_var('TOPDIR') - targetos = get_bb_var('TARGET_OS') - targetvendor = get_bb_var('TARGET_VENDOR') - self.write_config(""" + configA = """ TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" -MACHINE = \"qemux86\" -""") - self.track_for_cleanup(topdir + "/tmp-sstatesamehash") - bitbake("world meta-toolchain -S none") - self.write_config(""" +MACHINE = \"qemux86-64\" +""" + configB = """ TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" MACHINE = \"qemuarm\" -""") - self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") +""" + self.sstate_allarch_samesigs(configA, configB) + + def test_sstate_allarch_samesigs_multilib(self): + """ + The sstate checksums of allarch multilib packages should be independent of whichever + MACHINE is set. Check this using bitbake -S. + Also, rather than duplicate the test, check nativesdk stamps are the same between + the two MACHINE values. + """ + + configA = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86-64\" +require conf/multilib.conf +MULTILIBS = \"multilib:lib32\" +DEFAULTTUNE_virtclass-multilib-lib32 = \"x86\" +""" + configB = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemuarm\" +require conf/multilib.conf +MULTILIBS = \"\" +""" + self.sstate_allarch_samesigs(configA, configB) + + def sstate_allarch_samesigs(self, configA, configB): + + self.write_config(configA) + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("world meta-toolchain -S none") + self.write_config(configB) + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") bitbake("world meta-toolchain -S none") def get_files(d): @@ -336,15 +357,15 @@ MACHINE = \"qemuarm\" (_, task, _, shash) = name.rsplit(".", 3) f[os.path.join(os.path.basename(root), task)] = shash return f - files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/all" + targetvendor + "-" + targetos) - files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/all" + targetvendor + "-" + targetos) + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/all" + self.target_vendor + "-" + self.target_os) + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/all" + self.target_vendor + "-" + self.target_os) self.maxDiff = None self.assertEqual(files1, files2) - nativesdkdir = os.path.basename(glob.glob(topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) + nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) - files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) - files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) self.maxDiff = None self.assertEqual(files1, files2) @@ -356,9 +377,6 @@ MACHINE = \"qemuarm\" qemux86copy machine to test this. Also include multilibs in the test. """ - topdir = get_bb_var('TOPDIR') - targetos = get_bb_var('TARGET_OS') - targetvendor = get_bb_var('TARGET_VENDOR') self.write_config(""" TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" MACHINE = \"qemux86\" @@ -366,7 +384,7 @@ require conf/multilib.conf MULTILIBS = "multilib:lib32" DEFAULTTUNE_virtclass-multilib-lib32 = "x86" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") bitbake("world meta-toolchain -S none") self.write_config(""" TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" @@ -375,7 +393,7 @@ require conf/multilib.conf MULTILIBS = "multilib:lib32" DEFAULTTUNE_virtclass-multilib-lib32 = "x86" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") bitbake("world meta-toolchain -S none") def get_files(d): @@ -389,8 +407,8 @@ DEFAULTTUNE_virtclass-multilib-lib32 = "x86" if "do_build" not in name and "do_populate_sdk" not in name: f.append(os.path.join(root, name)) return f - files1 = get_files(topdir + "/tmp-sstatesamehash/stamps") - files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps") + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None self.assertCountEqual(files1, files2) @@ -402,8 +420,6 @@ DEFAULTTUNE_virtclass-multilib-lib32 = "x86" classes inherits should be the same. """ - topdir = get_bb_var('TOPDIR') - targetvendor = get_bb_var('TARGET_VENDOR') self.write_config(""" TMPDIR = "${TOPDIR}/tmp-sstatesamehash" BB_NUMBER_THREADS = "1" @@ -414,8 +430,8 @@ DATE = "20161111" INHERIT_remove = "buildstats-summary buildhistory uninative" http_proxy = "" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash") - self.track_for_cleanup(topdir + "/download1") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + self.track_for_cleanup(self.topdir + "/download1") bitbake("world meta-toolchain -S none") self.write_config(""" TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" @@ -429,8 +445,8 @@ INHERIT_remove = "uninative" INHERIT += "buildstats-summary buildhistory" http_proxy = "http://example.com/" """) - self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") - self.track_for_cleanup(topdir + "/download2") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + self.track_for_cleanup(self.topdir + "/download2") bitbake("world meta-toolchain -S none") def get_files(d): @@ -442,8 +458,8 @@ http_proxy = "http://example.com/" base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name]) f[base] = shash return f - files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") - files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") # Remove items that are identical in both sets for k,v in files1.items() & files2.items(): del files1[k] @@ -456,8 +472,8 @@ http_proxy = "http://example.com/" if k in files1 and k in files2: print("%s differs:" % k) print(subprocess.check_output(("bitbake-diffsigs", - topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k], - topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k]))) + self.topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k], + self.topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k]))) elif k in files1 and k not in files2: print("%s in files1" % k) elif k not in files1 and k in files2: diff --git a/meta/lib/oeqa/selftest/tinfoil.py b/meta/lib/oeqa/selftest/tinfoil.py new file mode 100644 index 0000000000..73a0c3bac0 --- /dev/null +++ b/meta/lib/oeqa/selftest/tinfoil.py @@ -0,0 +1,190 @@ +import unittest +import os +import re +import bb.tinfoil + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd +from oeqa.utils.decorators import testcase + +class TinfoilTests(oeSelfTest): + """ Basic tests for the tinfoil API """ + + @testcase(1568) + def test_getvar(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(True) + machine = tinfoil.config_data.getVar('MACHINE') + if not machine: + self.fail('Unable to get MACHINE value - returned %s' % machine) + + @testcase(1569) + def test_expand(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(True) + expr = '${@os.getpid()}' + pid = tinfoil.config_data.expand(expr) + if not pid: + self.fail('Unable to expand "%s" - returned %s' % (expr, pid)) + + @testcase(1570) + def test_getvar_bb_origenv(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(True) + origenv = tinfoil.config_data.getVar('BB_ORIGENV', False) + if not origenv: + self.fail('Unable to get BB_ORIGENV value - returned %s' % origenv) + self.assertEqual(origenv.getVar('HOME', False), os.environ['HOME']) + + @testcase(1571) + def test_parse_recipe(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3]) + self.assertEqual(testrecipe, rd.getVar('PN')) + + @testcase(1572) + def test_parse_recipe_copy_expand(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + rd = tinfoil.parse_recipe_file(best[3]) + # Check we can get variable values + self.assertEqual(testrecipe, rd.getVar('PN')) + # Check that expanding a value that includes a variable reference works + self.assertEqual(testrecipe, rd.getVar('BPN')) + # Now check that changing the referenced variable's value in a copy gives that + # value when expanding + localdata = bb.data.createCopy(rd) + localdata.setVar('PN', 'hello') + self.assertEqual('hello', localdata.getVar('BPN')) + + @testcase(1573) + def test_parse_recipe_initial_datastore(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + testrecipe = 'mdadm' + best = tinfoil.find_best_provider(testrecipe) + if not best: + self.fail('Unable to find recipe providing %s' % testrecipe) + dcopy = bb.data.createCopy(tinfoil.config_data) + dcopy.setVar('MYVARIABLE', 'somevalue') + rd = tinfoil.parse_recipe_file(best[3], config_data=dcopy) + # Check we can get variable values + self.assertEqual('somevalue', rd.getVar('MYVARIABLE')) + + @testcase(1574) + def test_list_recipes(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False, quiet=2) + # Check pkg_pn + checkpns = ['tar', 'automake', 'coreutils', 'm4-native', 'nativesdk-gcc'] + pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn + for pn in checkpns: + self.assertIn(pn, pkg_pn) + # Check pkg_fn + checkfns = {'nativesdk-gcc': '^virtual:nativesdk:.*', 'coreutils': '.*/coreutils_.*.bb'} + for fn, pn in tinfoil.cooker.recipecaches[''].pkg_fn.items(): + if pn in checkpns: + if pn in checkfns: + self.assertTrue(re.match(checkfns[pn], fn), 'Entry for %s: %s did not match %s' % (pn, fn, checkfns[pn])) + checkpns.remove(pn) + if checkpns: + self.fail('Unable to find pkg_fn entries for: %s' % ', '.join(checkpns)) + + @testcase(1575) + def test_wait_event(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + # Need to drain events otherwise events that will be masked will still be in the queue + while tinfoil.wait_event(0.25): + pass + tinfoil.set_event_mask(['bb.event.FilesMatchingFound', 'bb.command.CommandCompleted']) + pattern = 'conf' + res = tinfoil.run_command('findFilesMatchingInDir', pattern, 'conf/machine') + self.assertTrue(res) + + eventreceived = False + waitcount = 5 + while waitcount > 0: + event = tinfoil.wait_event(1) + if event: + if isinstance(event, bb.command.CommandCompleted): + break + elif isinstance(event, bb.event.FilesMatchingFound): + self.assertEqual(pattern, event._pattern) + self.assertIn('qemuarm.conf', event._matches) + eventreceived = True + else: + self.fail('Unexpected event: %s' % event) + + waitcount = waitcount - 1 + + self.assertNotEqual(waitcount, 0, 'Timed out waiting for CommandCompleted event from bitbake server') + self.assertTrue(eventreceived, 'Did not receive FilesMatchingFound event from bitbake server') + + @testcase(1576) + def test_setvariable_clean(self): + # First check that setVariable affects the datastore + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + tinfoil.run_command('setVariable', 'TESTVAR', 'specialvalue') + self.assertEqual(tinfoil.config_data.getVar('TESTVAR'), 'specialvalue', 'Value set using setVariable is not reflected in client-side getVar()') + + # Now check that the setVariable's effects are no longer present + # (this may legitimately break in future if we stop reinitialising + # the datastore, in which case we'll have to reconsider use of + # setVariable entirely) + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + self.assertNotEqual(tinfoil.config_data.getVar('TESTVAR'), 'specialvalue', 'Value set using setVariable is still present!') + + # Now check that setVar on the main datastore works (uses setVariable internally) + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + tinfoil.config_data.setVar('TESTVAR', 'specialvalue') + value = tinfoil.run_command('getVariable', 'TESTVAR') + self.assertEqual(value, 'specialvalue', 'Value set using config_data.setVar() is not reflected in config_data.getVar()') + + def test_datastore_operations(self): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + # Test setVarFlag() / getVarFlag() + tinfoil.config_data.setVarFlag('TESTVAR', 'flagname', 'flagval') + value = tinfoil.config_data.getVarFlag('TESTVAR', 'flagname') + self.assertEqual(value, 'flagval', 'Value set using config_data.setVarFlag() is not reflected in config_data.getVarFlag()') + # Test delVarFlag() + tinfoil.config_data.setVarFlag('TESTVAR', 'otherflag', 'othervalue') + tinfoil.config_data.delVarFlag('TESTVAR', 'flagname') + value = tinfoil.config_data.getVarFlag('TESTVAR', 'flagname') + self.assertEqual(value, None, 'Varflag deleted using config_data.delVarFlag() is not reflected in config_data.getVarFlag()') + value = tinfoil.config_data.getVarFlag('TESTVAR', 'otherflag') + self.assertEqual(value, 'othervalue', 'Varflag deleted using config_data.delVarFlag() caused unrelated flag to be removed') + # Test delVar() + tinfoil.config_data.setVar('TESTVAR', 'varvalue') + value = tinfoil.config_data.getVar('TESTVAR') + self.assertEqual(value, 'varvalue', 'Value set using config_data.setVar() is not reflected in config_data.getVar()') + tinfoil.config_data.delVar('TESTVAR') + value = tinfoil.config_data.getVar('TESTVAR') + self.assertEqual(value, None, 'Variable deleted using config_data.delVar() appears to still have a value') + # Test renameVar() + tinfoil.config_data.setVar('TESTVAROLD', 'origvalue') + tinfoil.config_data.renameVar('TESTVAROLD', 'TESTVARNEW') + value = tinfoil.config_data.getVar('TESTVAROLD') + self.assertEqual(value, None, 'Variable renamed using config_data.renameVar() still seems to exist') + value = tinfoil.config_data.getVar('TESTVARNEW') + self.assertEqual(value, 'origvalue', 'Variable renamed using config_data.renameVar() does not appear with new name') + # Test overrides + tinfoil.config_data.setVar('TESTVAR', 'original') + tinfoil.config_data.setVar('TESTVAR_overrideone', 'one') + tinfoil.config_data.setVar('TESTVAR_overridetwo', 'two') + tinfoil.config_data.appendVar('OVERRIDES', ':overrideone') + value = tinfoil.config_data.getVar('TESTVAR') + self.assertEqual(value, 'one', 'Variable overrides not functioning correctly') diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py index 895c6d6d01..726af19e9d 100644 --- a/meta/lib/oeqa/selftest/wic.py +++ b/meta/lib/oeqa/selftest/wic.py @@ -24,41 +24,84 @@ """Test cases for wic.""" import os +import sys +import unittest from glob import glob from shutil import rmtree +from functools import wraps, lru_cache +from tempfile import NamedTemporaryFile from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars, runqemu from oeqa.utils.decorators import testcase +@lru_cache(maxsize=32) +def get_host_arch(recipe): + """A cached call to get_bb_var('HOST_ARCH', <recipe>)""" + return get_bb_var('HOST_ARCH', recipe) + + +def only_for_arch(archs, image='core-image-minimal'): + """Decorator for wrapping test cases that can be run only for specific target + architectures. A list of compatible architectures is passed in `archs`. + Current architecture will be determined by parsing bitbake output for + `image` recipe. + """ + def wrapper(func): + @wraps(func) + def wrapped_f(*args, **kwargs): + arch = get_host_arch(image) + if archs and arch not in archs: + raise unittest.SkipTest("Testcase arch dependency not met: %s" % arch) + return func(*args, **kwargs) + wrapped_f.__name__ = func.__name__ + return wrapped_f + return wrapper + + class Wic(oeSelfTest): """Wic test class.""" - resultdir = "/var/tmp/wic/build/" + resultdir = "/var/tmp/wic.oe-selftest/" image_is_ready = False + native_sysroot = None + wicenv_cache = {} def setUpLocal(self): """This code is executed before each test method.""" - self.write_config('IMAGE_FSTYPES += " hddimg"\n' - 'MACHINE_FEATURES_append = " efi"\n') + if not self.native_sysroot: + Wic.native_sysroot = get_bb_var('STAGING_DIR_NATIVE', 'wic-tools') # Do this here instead of in setUpClass as the base setUp does some # clean up which can result in the native tools built earlier in # setUpClass being unavailable. if not Wic.image_is_ready: - bitbake('syslinux syslinux-native parted-native gptfdisk-native ' - 'dosfstools-native mtools-native') + if get_bb_var('USE_NLS') == 'yes': + bitbake('wic-tools') + else: + self.skipTest('wic-tools cannot be built due its (intltool|gettext)-native dependency and NLS disable') + bitbake('core-image-minimal') Wic.image_is_ready = True rmtree(self.resultdir, ignore_errors=True) + def tearDownLocal(self): + """Remove resultdir as it may contain images.""" + rmtree(self.resultdir, ignore_errors=True) + + @testcase(1552) + def test_version(self): + """Test wic --version""" + self.assertEqual(0, runCmd('wic --version').status) + @testcase(1208) def test_help(self): - """Test wic --help""" + """Test wic --help and wic -h""" self.assertEqual(0, runCmd('wic --help').status) + self.assertEqual(0, runCmd('wic -h').status) @testcase(1209) def test_createhelp(self): @@ -70,44 +113,15 @@ class Wic(oeSelfTest): """Test wic list --help""" self.assertEqual(0, runCmd('wic list --help').status) - @testcase(1211) - def test_build_image_name(self): - """Test wic create directdisk --image-name core-image-minimal""" - self.assertEqual(0, runCmd("wic create directdisk " - "--image-name core-image-minimal").status) - self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) - - @testcase(1212) - def test_build_artifacts(self): - """Test wic create directdisk providing all artifacts.""" - bbvars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ - for var in ('STAGING_DATADIR', 'DEPLOY_DIR_IMAGE', - 'STAGING_DIR_NATIVE', 'IMAGE_ROOTFS')) - status = runCmd("wic create directdisk " - "-b %(staging_datadir)s " - "-k %(deploy_dir_image)s " - "-n %(staging_dir_native)s " - "-r %(image_rootfs)s" % bbvars).status - self.assertEqual(0, status) - self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) - - @testcase(1157) - def test_gpt_image(self): - """Test creation of core-image-minimal with gpt table and UUID boot""" - self.assertEqual(0, runCmd("wic create directdisk-gpt " - "--image-name core-image-minimal").status) - self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) - - @testcase(1213) - def test_unsupported_subcommand(self): - """Test unsupported subcommand""" - self.assertEqual(1, runCmd('wic unsupported', - ignore_status=True).status) + @testcase(1553) + def test_help_create(self): + """Test wic help create""" + self.assertEqual(0, runCmd('wic help create').status) - @testcase(1214) - def test_no_command(self): - """Test wic without command""" - self.assertEqual(1, runCmd('wic', ignore_status=True).status) + @testcase(1554) + def test_help_list(self): + """Test wic help list""" + self.assertEqual(0, runCmd('wic help list').status) @testcase(1215) def test_help_overview(self): @@ -124,94 +138,418 @@ class Wic(oeSelfTest): """Test wic help kickstart""" self.assertEqual(0, runCmd('wic help kickstart').status) + @testcase(1555) + def test_list_images(self): + """Test wic list images""" + self.assertEqual(0, runCmd('wic list images').status) + + @testcase(1556) + def test_list_source_plugins(self): + """Test wic list source-plugins""" + self.assertEqual(0, runCmd('wic list source-plugins').status) + + @testcase(1557) + def test_listed_images_help(self): + """Test wic listed images help""" + output = runCmd('wic list images').output + imagelist = [line.split()[0] for line in output.splitlines()] + for image in imagelist: + self.assertEqual(0, runCmd('wic list %s help' % image).status) + + @testcase(1213) + def test_unsupported_subcommand(self): + """Test unsupported subcommand""" + self.assertEqual(1, runCmd('wic unsupported', + ignore_status=True).status) + + @testcase(1214) + def test_no_command(self): + """Test wic without command""" + self.assertEqual(1, runCmd('wic', ignore_status=True).status) + + @testcase(1211) + def test_build_image_name(self): + """Test wic create wictestdisk --image-name=core-image-minimal""" + cmd = "wic create wictestdisk --image-name=core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1157) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_gpt_image(self): + """Test creation of core-image-minimal with gpt table and UUID boot""" + cmd = "wic create directdisk-gpt --image-name core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) + + @testcase(1346) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_iso_image(self): + """Test creation of hybrid iso image with legacy and EFI boot""" + config = 'INITRAMFS_IMAGE = "core-image-minimal-initramfs"\n'\ + 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) + cmd = "wic create mkhybridiso --image-name core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.direct"))) + self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.iso"))) + + @testcase(1348) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_qemux86_directdisk(self): + """Test creation of qemux-86-directdisk image""" + cmd = "wic create qemux86-directdisk -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "qemux86-directdisk-*direct"))) + + @testcase(1350) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_mkefidisk(self): + """Test creation of mkefidisk image""" + cmd = "wic create mkefidisk -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "mkefidisk-*direct"))) + + @testcase(1385) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_bootloader_config(self): + """Test creation of directdisk-bootloader-config image""" + cmd = "wic create directdisk-bootloader-config -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-bootloader-config-*direct"))) + + @testcase(1560) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_systemd_bootdisk(self): + """Test creation of systemd-bootdisk image""" + config = 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) + cmd = "wic create systemd-bootdisk -e core-image-minimal -o %s" % self.resultdir + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "systemd-bootdisk-*direct"))) + + @testcase(1561) + def test_sdimage_bootpart(self): + """Test creation of sdimage-bootpart image""" + cmd = "wic create sdimage-bootpart -e core-image-minimal -o %s" % self.resultdir + kimgtype = get_bb_var('KERNEL_IMAGETYPE', 'core-image-minimal') + self.write_config('IMAGE_BOOT_FILES = "%s"\n' % kimgtype) + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob(self.resultdir + "sdimage-bootpart-*direct"))) + + @testcase(1562) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_default_output_dir(self): + """Test default output location""" + for fname in glob("directdisk-*.direct"): + os.remove(fname) + cmd = "wic create directdisk -e core-image-minimal" + self.assertEqual(0, runCmd(cmd).status) + self.assertEqual(1, len(glob("directdisk-*.direct"))) + + @testcase(1212) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_build_artifacts(self): + """Test wic create directdisk providing all artifacts.""" + bb_vars = get_bb_vars(['STAGING_DATADIR', 'RECIPE_SYSROOT_NATIVE'], + 'wic-tools') + bb_vars.update(get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_ROOTFS'], + 'core-image-minimal')) + bbvars = {key.lower(): value for key, value in bb_vars.items()} + bbvars['resultdir'] = self.resultdir + status = runCmd("wic create directdisk " + "-b %(staging_datadir)s " + "-k %(deploy_dir_image)s " + "-n %(recipe_sysroot_native)s " + "-r %(image_rootfs)s " + "-o %(resultdir)s" % bbvars).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) + @testcase(1264) def test_compress_gzip(self): """Test compressing an image with gzip""" - self.assertEqual(0, runCmd("wic create directdisk " + self.assertEqual(0, runCmd("wic create wictestdisk " "--image-name core-image-minimal " - "-c gzip").status) - self.assertEqual(1, len(glob(self.resultdir + \ - "directdisk-*.direct.gz"))) + "-c gzip -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct.gz"))) @testcase(1265) def test_compress_bzip2(self): """Test compressing an image with bzip2""" - self.assertEqual(0, runCmd("wic create directdisk " - "--image-name core-image-minimal " - "-c bzip2").status) - self.assertEqual(1, len(glob(self.resultdir + \ - "directdisk-*.direct.bz2"))) + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-c bzip2 -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct.bz2"))) @testcase(1266) def test_compress_xz(self): """Test compressing an image with xz""" - self.assertEqual(0, runCmd("wic create directdisk " - "--image-name core-image-minimal " - "-c xz").status) - self.assertEqual(1, len(glob(self.resultdir + \ - "directdisk-*.direct.xz"))) + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--compress-with=xz -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct.xz"))) @testcase(1267) def test_wrong_compressor(self): """Test how wic breaks if wrong compressor is provided""" - self.assertEqual(2, runCmd("wic create directdisk " - "--image-name core-image-minimal " - "-c wrong", ignore_status=True).status) + self.assertEqual(2, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-c wrong -o %s" % self.resultdir, + ignore_status=True).status) + + @testcase(1558) + def test_debug_short(self): + """Test -D option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-D -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + def test_debug_long(self): + """Test --debug option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--debug -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1563) + def test_skip_build_check_short(self): + """Test -s option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-s -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + def test_skip_build_check_long(self): + """Test --skip-build-check option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--skip-build-check " + "--outdir %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + @testcase(1564) + def test_build_rootfs_short(self): + """Test -f option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-f -o %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) + + def test_build_rootfs_long(self): + """Test --build-rootfs option""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "--build-rootfs " + "--outdir %s" % self.resultdir).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*.direct"))) @testcase(1268) + @only_for_arch(['i586', 'i686', 'x86_64']) def test_rootfs_indirect_recipes(self): """Test usage of rootfs plugin with rootfs recipes""" - wks = "directdisk-multi-rootfs" - self.assertEqual(0, runCmd("wic create %s " - "--image-name core-image-minimal " - "--rootfs rootfs1=core-image-minimal " - "--rootfs rootfs2=core-image-minimal" \ - % wks).status) - self.assertEqual(1, len(glob(self.resultdir + "%s*.direct" % wks))) + status = runCmd("wic create directdisk-multi-rootfs " + "--image-name=core-image-minimal " + "--rootfs rootfs1=core-image-minimal " + "--rootfs rootfs2=core-image-minimal " + "--outdir %s" % self.resultdir).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "directdisk-multi-rootfs*.direct"))) @testcase(1269) + @only_for_arch(['i586', 'i686', 'x86_64']) def test_rootfs_artifacts(self): """Test usage of rootfs plugin with rootfs paths""" - bbvars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ - for var in ('STAGING_DATADIR', 'DEPLOY_DIR_IMAGE', - 'STAGING_DIR_NATIVE', 'IMAGE_ROOTFS')) + bb_vars = get_bb_vars(['STAGING_DATADIR', 'RECIPE_SYSROOT_NATIVE'], + 'wic-tools') + bb_vars.update(get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_ROOTFS'], + 'core-image-minimal')) + bbvars = {key.lower(): value for key, value in bb_vars.items()} bbvars['wks'] = "directdisk-multi-rootfs" + bbvars['resultdir'] = self.resultdir status = runCmd("wic create %(wks)s " - "-b %(staging_datadir)s " - "-k %(deploy_dir_image)s " - "-n %(staging_dir_native)s " + "--bootimg-dir=%(staging_datadir)s " + "--kernel-dir=%(deploy_dir_image)s " + "--native-sysroot=%(recipe_sysroot_native)s " "--rootfs-dir rootfs1=%(image_rootfs)s " - "--rootfs-dir rootfs2=%(image_rootfs)s" \ - % bbvars).status + "--rootfs-dir rootfs2=%(image_rootfs)s " + "--outdir %(resultdir)s" % bbvars).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "%(wks)s-*.direct" % bbvars))) + + def test_exclude_path(self): + """Test --exclude-path wks option.""" + + oldpath = os.environ['PATH'] + os.environ['PATH'] = get_bb_var("PATH", "wic-tools") + + try: + wks_file = 'temp.wks' + with open(wks_file, 'w') as wks: + rootfs_dir = get_bb_var('IMAGE_ROOTFS', 'core-image-minimal') + wks.write(""" +part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path usr +part /usr --source rootfs --ondisk mmcblk0 --fstype=ext4 --rootfs-dir %s/usr +part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --rootfs-dir %s/usr""" + % (rootfs_dir, rootfs_dir)) + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir)).status) + + os.remove(wks_file) + wicout = glob(self.resultdir + "%s-*direct" % 'temp') + self.assertEqual(1, len(wicout)) + + wicimg = wicout[0] + + # verify partition size with wic + res = runCmd("parted -m %s unit b p 2>/dev/null" % wicimg) + self.assertEqual(0, res.status) + + # parse parted output which looks like this: + # BYT;\n + # /var/tmp/wic/build/tmpfwvjjkf_-201611101222-hda.direct:200MiB:file:512:512:msdos::;\n + # 1:0.00MiB:200MiB:200MiB:ext4::;\n + partlns = res.output.splitlines()[2:] + + self.assertEqual(3, len(partlns)) + + for part in [1, 2, 3]: + part_file = os.path.join(self.resultdir, "selftest_img.part%d" % part) + partln = partlns[part-1].split(":") + self.assertEqual(7, len(partln)) + start = int(partln[1].rstrip("B")) / 512 + length = int(partln[3].rstrip("B")) / 512 + self.assertEqual(0, runCmd("dd if=%s of=%s skip=%d count=%d" % + (wicimg, part_file, start, length)).status) + + def extract_files(debugfs_output): + """ + extract file names from the output of debugfs -R 'ls -p', + which looks like this: + + /2/040755/0/0/.//\n + /2/040755/0/0/..//\n + /11/040700/0/0/lost+found^M//\n + /12/040755/1002/1002/run//\n + /13/040755/1002/1002/sys//\n + /14/040755/1002/1002/bin//\n + /80/040755/1002/1002/var//\n + /92/040755/1002/1002/tmp//\n + """ + # NOTE the occasional ^M in file names + return [line.split('/')[5].strip() for line in \ + debugfs_output.strip().split('/\n')] + + # Test partition 1, should contain the normal root directories, except + # /usr. + res = runCmd("debugfs -R 'ls -p' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part1")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertIn("etc", files) + self.assertNotIn("usr", files) + + # Partition 2, should contain common directories for /usr, not root + # directories. + res = runCmd("debugfs -R 'ls -p' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part2")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertNotIn("etc", files) + self.assertNotIn("usr", files) + self.assertIn("share", files) + + # Partition 3, should contain the same as partition 2, including the bin + # directory, but not the files inside it. + res = runCmd("debugfs -R 'ls -p' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part3")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertNotIn("etc", files) + self.assertNotIn("usr", files) + self.assertIn("share", files) + self.assertIn("bin", files) + res = runCmd("debugfs -R 'ls -p bin' %s 2>/dev/null" % \ + os.path.join(self.resultdir, "selftest_img.part3")) + self.assertEqual(0, res.status) + files = extract_files(res.output) + self.assertIn(".", files) + self.assertIn("..", files) + self.assertEqual(2, len(files)) + + for part in [1, 2, 3]: + part_file = os.path.join(self.resultdir, "selftest_img.part%d" % part) + os.remove(part_file) + + finally: + os.environ['PATH'] = oldpath + + def test_exclude_path_errors(self): + """Test --exclude-path wks option error handling.""" + wks_file = 'temp.wks' + + # Absolute argument. + with open(wks_file, 'w') as wks: + wks.write("part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path /usr") + self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir), ignore_status=True).status) + os.remove(wks_file) + + # Argument pointing to parent directory. + with open(wks_file, 'w') as wks: + wks.write("part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path ././..") + self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wks_file, self.resultdir), ignore_status=True).status) + os.remove(wks_file) + + @testcase(1496) + def test_bmap_short(self): + """Test generation of .bmap file -m option""" + cmd = "wic create wictestdisk -e core-image-minimal -m -o %s" % self.resultdir + status = runCmd(cmd).status self.assertEqual(0, status) - self.assertEqual(1, len(glob(self.resultdir + \ - "%(wks)s-*.direct" % bbvars))) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct.bmap"))) - @testcase(1346) - def test_iso_image(self): - """Test creation of hybrid iso image with legacy and EFI boot""" - self.assertEqual(0, runCmd("wic create mkhybridiso " - "--image-name core-image-minimal").status) - self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.direct"))) - self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.iso"))) + def test_bmap_long(self): + """Test generation of .bmap file --bmap option""" + cmd = "wic create wictestdisk -e core-image-minimal --bmap -o %s" % self.resultdir + status = runCmd(cmd).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct.bmap"))) + + def _get_image_env_path(self, image): + """Generate and obtain the path to <image>.env""" + if image not in self.wicenv_cache: + self.assertEqual(0, bitbake('%s -c do_rootfs_wicenv' % image).status) + bb_vars = get_bb_vars(['STAGING_DIR', 'MACHINE'], image) + stdir = bb_vars['STAGING_DIR'] + machine = bb_vars['MACHINE'] + self.wicenv_cache[image] = os.path.join(stdir, machine, 'imgdata') + return self.wicenv_cache[image] @testcase(1347) def test_image_env(self): """Test generation of <image>.env files.""" image = 'core-image-minimal' - self.assertEqual(0, bitbake('%s -c do_rootfs_wicenv' % image).status) - stdir = get_bb_var('STAGING_DIR_TARGET', image) - imgdatadir = os.path.join(stdir, 'imgdata') + imgdatadir = self._get_image_env_path(image) - basename = get_bb_var('IMAGE_BASENAME', image) + bb_vars = get_bb_vars(['IMAGE_BASENAME', 'WICVARS'], image) + basename = bb_vars['IMAGE_BASENAME'] self.assertEqual(basename, image) path = os.path.join(imgdatadir, basename) + '.env' self.assertTrue(os.path.isfile(path)) - wicvars = set(get_bb_var('WICVARS', image).split()) + wicvars = set(bb_vars['WICVARS'].split()) # filter out optional variables - wicvars = wicvars.difference(('HDDDIR', 'IMAGE_BOOT_FILES', - 'INITRD', 'ISODIR')) + wicvars = wicvars.difference(('DEPLOY_DIR_IMAGE', 'IMAGE_BOOT_FILES', + 'INITRD', 'INITRD_LIVE', 'ISODIR')) with open(path) as envfile: content = dict(line.split("=", 1) for line in envfile) # test if variables used by wic present in the .env file @@ -219,13 +557,41 @@ class Wic(oeSelfTest): self.assertTrue(var in content, "%s is not in .env file" % var) self.assertTrue(content[var]) + @testcase(1559) + def test_image_vars_dir_short(self): + """Test image vars directory selection -v option""" + image = 'core-image-minimal' + imgenvdir = self._get_image_env_path(image) + + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=%s -v %s -o %s" + % (image, imgenvdir, self.resultdir)).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + + def test_image_vars_dir_long(self): + """Test image vars directory selection --vars option""" + image = 'core-image-minimal' + imgenvdir = self._get_image_env_path(image) + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=%s " + "--vars %s " + "--outdir %s" + % (image, imgenvdir, self.resultdir)).status) + self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) + @testcase(1351) + @only_for_arch(['i586', 'i686', 'x86_64']) def test_wic_image_type(self): """Test building wic images by bitbake""" + config = 'IMAGE_FSTYPES += "wic"\nWKS_FILE = "wic-image-minimal"\n'\ + 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) self.assertEqual(0, bitbake('wic-image-minimal').status) + self.remove_config(config) - deploy_dir = get_bb_var('DEPLOY_DIR_IMAGE') - machine = get_bb_var('MACHINE') + bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'MACHINE']) + deploy_dir = bb_vars['DEPLOY_DIR_IMAGE'] + machine = bb_vars['MACHINE'] prefix = os.path.join(deploy_dir, 'wic-image-minimal-%s.' % machine) # check if we have result image and manifests symlinks # pointing to existing files @@ -234,53 +600,193 @@ class Wic(oeSelfTest): self.assertTrue(os.path.islink(path)) self.assertTrue(os.path.isfile(os.path.realpath(path))) - @testcase(1348) - def test_qemux86_directdisk(self): - """Test creation of qemux-86-directdisk image""" - image = "qemux86-directdisk" - self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ - % image).status) - self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - - @testcase(1349) - def test_mkgummidisk(self): - """Test creation of mkgummidisk image""" - image = "mkgummidisk" - self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ - % image).status) - self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - - @testcase(1350) - def test_mkefidisk(self): - """Test creation of mkefidisk image""" - image = "mkefidisk" - self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ - % image).status) - self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - - @testcase(1385) - def test_directdisk_bootloader_config(self): - """Test creation of directdisk-bootloader-config image""" - image = "directdisk-bootloader-config" - self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ - % image).status) - self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - @testcase(1422) + @only_for_arch(['i586', 'i686', 'x86_64']) def test_qemu(self): """Test wic-image-minimal under qemu""" + config = 'IMAGE_FSTYPES += "wic"\nWKS_FILE = "wic-image-minimal"\n'\ + 'MACHINE_FEATURES_append = " efi"\n' + self.append_config(config) self.assertEqual(0, bitbake('wic-image-minimal').status) + self.remove_config(config) with runqemu('wic-image-minimal', ssh=False) as qemu: - command = "mount |grep '^/dev/' | cut -f1,3 -d ' '" - status, output = qemu.run_serial(command) - self.assertEqual(1, status, 'Failed to run command "%s": %s' % (command, output)) - self.assertEqual(output, '/dev/root /\r\n/dev/vda3 /mnt') - - def test_bmap(self): - """Test generation of .bmap file""" - image = "directdisk" - status = runCmd("wic create %s -e core-image-minimal --bmap" % image).status - self.assertEqual(0, status) - self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - self.assertEqual(1, len(glob(self.resultdir + "%s-*direct.bmap" % image))) + cmd = "mount |grep '^/dev/' | cut -f1,3 -d ' '" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '/dev/root /\r\n/dev/sda3 /mnt') + + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_qemu_efi(self): + """Test core-image-minimal efi image under qemu""" + config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "mkefidisk.wks"\n' + self.append_config(config) + self.assertEqual(0, bitbake('core-image-minimal ovmf').status) + self.remove_config(config) + + with runqemu('core-image-minimal', ssh=False, + runqemuparams='ovmf', image_fstype='wic') as qemu: + cmd = "grep sda. /proc/partitions |wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '3') + + @staticmethod + def _make_fixed_size_wks(size): + """ + Create a wks of an image with a single partition. Size of the partition is set + using --fixed-size flag. Returns a tuple: (path to wks file, wks image name) + """ + with NamedTemporaryFile("w", suffix=".wks", delete=False) as tempf: + wkspath = tempf.name + tempf.write("part " \ + "--source rootfs --ondisk hda --align 4 --fixed-size %d " + "--fstype=ext4\n" % size) + wksname = os.path.splitext(os.path.basename(wkspath))[0] + + return wkspath, wksname + + def test_fixed_size(self): + """ + Test creation of a simple image with partition size controlled through + --fixed-size flag + """ + wkspath, wksname = Wic._make_fixed_size_wks(200) + + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wkspath, self.resultdir)).status) + os.remove(wkspath) + wicout = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(wicout)) + + wicimg = wicout[0] + + # verify partition size with wic + res = runCmd("parted -m %s unit mib p 2>/dev/null" % wicimg, + ignore_status=True, + native_sysroot=self.native_sysroot) + self.assertEqual(0, res.status) + + # parse parted output which looks like this: + # BYT;\n + # /var/tmp/wic/build/tmpfwvjjkf_-201611101222-hda.direct:200MiB:file:512:512:msdos::;\n + # 1:0.00MiB:200MiB:200MiB:ext4::;\n + partlns = res.output.splitlines()[2:] + + self.assertEqual(1, len(partlns)) + self.assertEqual("1:0.00MiB:200MiB:200MiB:ext4::;", partlns[0]) + + def test_fixed_size_error(self): + """ + Test creation of a simple image with partition size controlled through + --fixed-size flag. The size of partition is intentionally set to 1MiB + in order to trigger an error in wic. + """ + wkspath, wksname = Wic._make_fixed_size_wks(1) + + self.assertEqual(1, runCmd("wic create %s -e core-image-minimal -o %s" \ + % (wkspath, self.resultdir), ignore_status=True).status) + os.remove(wkspath) + wicout = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(0, len(wicout)) + + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_rawcopy_plugin_qemu(self): + """Test rawcopy plugin in qemu""" + # build ext4 and wic images + for fstype in ("ext4", "wic"): + config = 'IMAGE_FSTYPES = "%s"\nWKS_FILE = "test_rawcopy_plugin.wks.in"\n' % fstype + self.append_config(config) + self.assertEqual(0, bitbake('core-image-minimal').status) + self.remove_config(config) + + with runqemu('core-image-minimal', ssh=False, image_fstype='wic') as qemu: + cmd = "grep sda. /proc/partitions |wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '2') + + def test_rawcopy_plugin(self): + """Test rawcopy plugin""" + img = 'core-image-minimal' + machine = get_bb_var('MACHINE', img) + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part /boot --active --source bootimg-pcbios\n', + 'part / --source rawcopy --sourceparams="file=%s-%s.ext4" --use-uuid\n'\ + % (img, machine), + 'bootloader --timeout=0 --append="console=ttyS0,115200n8"\n']) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + self.assertEqual(0, runCmd(cmd).status) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(out)) + + def test_fs_types(self): + """Test filesystem types for empty and not empty partitions""" + img = 'core-image-minimal' + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part ext2 --fstype ext2 --source rootfs\n', + 'part btrfs --fstype btrfs --source rootfs --size 40M\n', + 'part squash --fstype squashfs --source rootfs\n', + 'part swap --fstype swap --size 1M\n', + 'part emptyvfat --fstype vfat --size 1M\n', + 'part emptymsdos --fstype msdos --size 1M\n', + 'part emptyext2 --fstype ext2 --size 1M\n', + 'part emptybtrfs --fstype btrfs --size 100M\n']) + wks.flush() + cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir) + self.assertEqual(0, runCmd(cmd).status) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(out)) + + def test_kickstart_parser(self): + """Test wks parser options""" + with NamedTemporaryFile("w", suffix=".wks") as wks: + wks.writelines(['part / --fstype ext3 --source rootfs --system-id 0xFF '\ + '--overhead-factor 1.2 --size 100k\n']) + wks.flush() + cmd = "wic create %s -e core-image-minimal -o %s" % (wks.name, self.resultdir) + self.assertEqual(0, runCmd(cmd).status) + wksname = os.path.splitext(os.path.basename(wks.name))[0] + out = glob(self.resultdir + "%s-*direct" % wksname) + self.assertEqual(1, len(out)) + + def test_image_bootpart_globbed(self): + """Test globbed sources with image-bootpart plugin""" + img = "core-image-minimal" + cmd = "wic create sdimage-bootpart -e %s -o %s" % (img, self.resultdir) + config = 'IMAGE_BOOT_FILES = "%s*"' % get_bb_var('KERNEL_IMAGETYPE', img) + self.append_config(config) + self.assertEqual(0, runCmd(cmd).status) + self.remove_config(config) + self.assertEqual(1, len(glob(self.resultdir + "sdimage-bootpart-*direct"))) + + def test_sparse_copy(self): + """Test sparse_copy with FIEMAP and SEEK_HOLE filemap APIs""" + libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'wic') + sys.path.insert(0, libpath) + from filemap import FilemapFiemap, FilemapSeek, sparse_copy, ErrorNotSupp + with NamedTemporaryFile("w", suffix=".wic-sparse") as sparse: + src_name = sparse.name + src_size = 1024 * 10 + sparse.truncate(src_size) + # write one byte to the file + with open(src_name, 'r+b') as sfile: + sfile.seek(1024 * 4) + sfile.write(b'\x00') + dest = sparse.name + '.out' + # copy src file to dest using different filemap APIs + for api in (FilemapFiemap, FilemapSeek, None): + if os.path.exists(dest): + os.unlink(dest) + try: + sparse_copy(sparse.name, dest, api=api) + except ErrorNotSupp: + continue # skip unsupported API + dest_stat = os.stat(dest) + self.assertEqual(dest_stat.st_size, src_size) + # 8 blocks is 4K (physical sector size) + self.assertEqual(dest_stat.st_blocks, 8) + os.unlink(dest) diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py index 1c57efaaef..3255e3a5c6 100644 --- a/meta/lib/oeqa/targetcontrol.py +++ b/meta/lib/oeqa/targetcontrol.py @@ -18,8 +18,10 @@ from oeqa.utils.dump import TargetDumper from oeqa.controllers.testtargetloader import TestTargetLoader from abc import ABCMeta, abstractmethod +logger = logging.getLogger('BitBake.QemuRunner') + def get_target_controller(d): - testtarget = d.getVar("TEST_TARGET", True) + testtarget = d.getVar("TEST_TARGET") # old, simple names if testtarget == "qemu": return QemuTarget(d) @@ -33,7 +35,7 @@ def get_target_controller(d): except AttributeError: # nope, perhaps a layer defined one try: - bbpath = d.getVar("BBPATH", True).split(':') + bbpath = d.getVar("BBPATH").split(':') testtargetloader = TestTargetLoader() controller = testtargetloader.get_controller_module(testtarget, bbpath) except ImportError as e: @@ -51,9 +53,9 @@ class BaseTarget(object, metaclass=ABCMeta): self.connection = None self.ip = None self.server_ip = None - self.datetime = d.getVar('DATETIME', True) - self.testdir = d.getVar("TEST_LOG_DIR", True) - self.pn = d.getVar("PN", True) + self.datetime = d.getVar('DATETIME') + self.testdir = d.getVar("TEST_LOG_DIR") + self.pn = d.getVar("PN") @abstractmethod def deploy(self): @@ -63,10 +65,10 @@ class BaseTarget(object, metaclass=ABCMeta): if os.path.islink(sshloglink): os.unlink(sshloglink) os.symlink(self.sshlog, sshloglink) - bb.note("SSH log file: %s" % self.sshlog) + logger.info("SSH log file: %s" % self.sshlog) @abstractmethod - def start(self, params=None, ssh=True): + def start(self, params=None, ssh=True, extra_bootparams=None): pass @abstractmethod @@ -80,7 +82,7 @@ class BaseTarget(object, metaclass=ABCMeta): @classmethod def match_image_fstype(self, d, image_fstypes=None): if not image_fstypes: - image_fstypes = d.getVar('IMAGE_FSTYPES', True).split(' ') + image_fstypes = d.getVar('IMAGE_FSTYPES').split(' ') possible_image_fstypes = [fstype for fstype in self.supported_image_fstypes if fstype in image_fstypes] if possible_image_fstypes: return possible_image_fstypes[0] @@ -113,69 +115,81 @@ class QemuTarget(BaseTarget): supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] - def __init__(self, d): + def __init__(self, d, image_fstype=None): super(QemuTarget, self).__init__(d) - self.image_fstype = self.get_image_fstype(d) + self.rootfs = '' + self.kernel = '' + self.image_fstype = '' + + if d.getVar('FIND_ROOTFS') == '1': + self.image_fstype = image_fstype or self.get_image_fstype(d) + self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("IMAGE_LINK_NAME") + '.' + self.image_fstype) + self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) - self.origrootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) - self.rootfs = os.path.join(self.testdir, d.getVar("IMAGE_LINK_NAME", True) + '-testimage.' + self.image_fstype) - self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') - dump_target_cmds = d.getVar("testimage_dump_target", True) - dump_host_cmds = d.getVar("testimage_dump_host", True) - dump_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) + dump_target_cmds = d.getVar("testimage_dump_target") + dump_host_cmds = d.getVar("testimage_dump_host") + dump_dir = d.getVar("TESTIMAGE_DUMP_DIR") + qemu_use_kvm = d.getVar("QEMU_USE_KVM") + if qemu_use_kvm and \ + (qemu_use_kvm == "True" and "x86" in d.getVar("MACHINE") or \ + d.getVar("MACHINE") in qemu_use_kvm.split()): + use_kvm = True + else: + use_kvm = False # Log QemuRunner log output to a file import oe.path bb.utils.mkdirhier(self.testdir) self.qemurunnerlog = os.path.join(self.testdir, 'qemurunner_log.%s' % self.datetime) - logger = logging.getLogger('BitBake.QemuRunner') loggerhandler = logging.FileHandler(self.qemurunnerlog) loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) logger.addHandler(loggerhandler) oe.path.symlink(os.path.basename(self.qemurunnerlog), os.path.join(self.testdir, 'qemurunner_log'), force=True) - if d.getVar("DISTRO", True) == "poky-tiny": - self.runner = QemuTinyRunner(machine=d.getVar("MACHINE", True), + if d.getVar("DISTRO") == "poky-tiny": + self.runner = QemuTinyRunner(machine=d.getVar("MACHINE"), rootfs=self.rootfs, - tmpdir = d.getVar("TMPDIR", True), - deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True), - display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True), + tmpdir = d.getVar("TMPDIR"), + deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE"), + display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY"), logfile = self.qemulog, kernel = self.kernel, - boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True))) + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT"))) else: - self.runner = QemuRunner(machine=d.getVar("MACHINE", True), + self.runner = QemuRunner(machine=d.getVar("MACHINE"), rootfs=self.rootfs, - tmpdir = d.getVar("TMPDIR", True), - deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True), - display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True), + tmpdir = d.getVar("TMPDIR"), + deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE"), + display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY"), logfile = self.qemulog, - boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)), + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")), + use_kvm = use_kvm, dump_dir = dump_dir, - dump_host_cmds = d.getVar("testimage_dump_host", True)) + dump_host_cmds = d.getVar("testimage_dump_host")) self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) def deploy(self): - try: - bb.utils.mkdirhier(self.testdir) - shutil.copyfile(self.origrootfs, self.rootfs) - except Exception as e: - bb.fatal("Error copying rootfs: %s" % e) + bb.utils.mkdirhier(self.testdir) qemuloglink = os.path.join(self.testdir, "qemu_boot_log") if os.path.islink(qemuloglink): os.unlink(qemuloglink) os.symlink(self.qemulog, qemuloglink) - bb.note("rootfs file: %s" % self.rootfs) - bb.note("Qemu log file: %s" % self.qemulog) + logger.info("rootfs file: %s" % self.rootfs) + logger.info("Qemu log file: %s" % self.qemulog) super(QemuTarget, self).deploy() - def start(self, params=None, ssh=True): - if self.runner.start(params, get_ip=ssh): + def start(self, params=None, ssh=True, extra_bootparams='', runqemuparams='', launch_cmd='', discard_writes=True): + if launch_cmd: + start = self.runner.launch(get_ip=ssh, launch_cmd=launch_cmd) + else: + start = self.runner.start(params, get_ip=ssh, extra_bootparams=extra_bootparams, runqemuparams=runqemuparams, discard_writes=discard_writes) + + if start: if ssh: self.ip = self.runner.ip self.server_ip = self.runner.server_ip @@ -204,33 +218,33 @@ class QemuTarget(BaseTarget): else: raise bb.build.FuncFailed("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn) - def run_serial(self, command): - return self.runner.run_serial(command) + def run_serial(self, command, timeout=5): + return self.runner.run_serial(command, timeout=timeout) class SimpleRemoteTarget(BaseTarget): def __init__(self, d): super(SimpleRemoteTarget, self).__init__(d) - addr = d.getVar("TEST_TARGET_IP", True) or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') + addr = d.getVar("TEST_TARGET_IP") or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.') self.ip = addr.split(":")[0] try: self.port = addr.split(":")[1] except IndexError: self.port = None - bb.note("Target IP: %s" % self.ip) - self.server_ip = d.getVar("TEST_SERVER_IP", True) + logger.info("Target IP: %s" % self.ip) + self.server_ip = d.getVar("TEST_SERVER_IP") if not self.server_ip: try: self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split("\n")[0].split()[-1] except Exception as e: bb.fatal("Failed to determine the host IP address (alternatively you can set TEST_SERVER_IP with the IP address of this machine): %s" % e) - bb.note("Server IP: %s" % self.server_ip) + logger.info("Server IP: %s" % self.server_ip) def deploy(self): super(SimpleRemoteTarget, self).deploy() - def start(self, params=None, ssh=True): + def start(self, params=None, ssh=True, extra_bootparams=None): if ssh: self.connection = SSHControl(self.ip, logfile=self.sshlog, port=self.port) diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py index 8f706f3637..485de031a9 100644 --- a/meta/lib/oeqa/utils/__init__.py +++ b/meta/lib/oeqa/utils/__init__.py @@ -36,3 +36,33 @@ def avoid_paths_in_environ(paths): new_path = new_path[:-1] return new_path + +def make_logger_bitbake_compatible(logger): + import logging + + """ + Bitbake logger redifines debug() in order to + set a level within debug, this breaks compatibility + with vainilla logging, so we neeed to redifine debug() + method again also add info() method with INFO + 1 level. + """ + def _bitbake_log_debug(*args, **kwargs): + lvl = logging.DEBUG + + if isinstance(args[0], int): + lvl = args[0] + msg = args[1] + args = args[2:] + else: + msg = args[0] + args = args[1:] + + logger.log(lvl, msg, *args, **kwargs) + + def _bitbake_log_info(msg, *args, **kwargs): + logger.log(logging.INFO + 1, msg, *args, **kwargs) + + logger.debug = _bitbake_log_debug + logger.info = _bitbake_log_info + + return logger diff --git a/meta/lib/oeqa/utils/buildproject.py b/meta/lib/oeqa/utils/buildproject.py new file mode 100644 index 0000000000..487f08be49 --- /dev/null +++ b/meta/lib/oeqa/utils/buildproject.py @@ -0,0 +1,55 @@ +# Copyright (C) 2013-2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Provides a class for automating build tests for projects + +import os +import re +import subprocess +import shutil +import tempfile + +from abc import ABCMeta, abstractmethod + +class BuildProject(metaclass=ABCMeta): + def __init__(self, uri, foldername=None, tmpdir=None, dl_dir=None): + self.uri = uri + self.archive = os.path.basename(uri) + if not tmpdir: + tmpdir = tempfile.mkdtemp(prefix='buildproject') + self.localarchive = os.path.join(tmpdir, self.archive) + self.dl_dir = dl_dir + if foldername: + self.fname = foldername + else: + self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive) + + # Download self.archive to self.localarchive + def _download_archive(self): + if self.dl_dir and os.path.exists(os.path.join(self.dl_dir, self.archive)): + shutil.copyfile(os.path.join(self.dl_dir, self.archive), self.localarchive) + return + + cmd = "wget -O %s %s" % (self.localarchive, self.uri) + subprocess.check_output(cmd, shell=True) + + # This method should provide a way to run a command in the desired environment. + @abstractmethod + def _run(self, cmd): + pass + + # The timeout parameter of target.run is set to 0 to make the ssh command + # run with no timeout. + def run_configure(self, configure_args='', extra_cmds=''): + return self._run('cd %s; gnu-configize; %s ./configure %s' % (self.targetdir, extra_cmds, configure_args)) + + def run_make(self, make_args=''): + return self._run('cd %s; make %s' % (self.targetdir, make_args)) + + def run_install(self, install_args=''): + return self._run('cd %s; make install %s' % (self.targetdir, install_args)) + + def clean(self): + self._run('rm -rf %s' % self.targetdir) + subprocess.call('rm -f %s' % self.localarchive, shell=True) diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 18fe39ecfe..57286fcb10 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -78,7 +78,10 @@ class Command(object): self.process.kill() self.thread.join() - self.output = self.output.decode("utf-8").rstrip() + if not self.output: + self.output = "" + else: + self.output = self.output.decode("utf-8", errors='replace').rstrip() self.status = self.process.poll() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) @@ -94,22 +97,37 @@ class Result(object): pass -def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **options): +def runCmd(command, ignore_status=False, timeout=None, assert_error=True, + native_sysroot=None, limit_exc_output=0, **options): result = Result() + if native_sysroot: + extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ + (native_sysroot, native_sysroot, native_sysroot) + nenv = dict(options.get('env', os.environ)) + nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') + options['env'] = nenv + cmd = Command(command, timeout=timeout, **options) cmd.run() result.command = command result.status = cmd.status result.output = cmd.output + result.error = cmd.error result.pid = cmd.process.pid if result.status and not ignore_status: + exc_output = result.output + if limit_exc_output > 0: + split = result.output.splitlines() + if len(split) > limit_exc_output: + exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \ + '\n'.join(split[-limit_exc_output:]) if assert_error: - raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output)) + raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output)) else: - raise CommandError(result.status, command, result.output) + raise CommandError(result.status, command, exc_output) return result @@ -141,22 +159,47 @@ def get_bb_env(target=None, postconfig=None): else: return bitbake("-e", postconfig=postconfig).output -def get_bb_var(var, target=None, postconfig=None): - val = None +def get_bb_vars(variables=None, target=None, postconfig=None): + """Get values of multiple bitbake variables""" bbenv = get_bb_env(target, postconfig=postconfig) + + if variables is not None: + variables = variables.copy() + var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$') + unset_re = re.compile(r'^unset (?P<var>\w+)$') lastline = None + values = {} for line in bbenv.splitlines(): - if re.search("^(export )?%s=" % var, line): - val = line.split('=', 1)[1] - val = val.strip('\"') - break - elif re.match("unset %s$" % var, line): - # Handle [unexport] variables - if lastline.startswith('# "'): - val = lastline.split('\"')[1] - break + match = var_re.match(line) + val = None + if match: + val = match.group('value') + else: + match = unset_re.match(line) + if match: + # Handle [unexport] variables + if lastline.startswith('# "'): + val = lastline.split('"')[1] + if val: + var = match.group('var') + if variables is None: + values[var] = val + else: + if var in variables: + values[var] = val + variables.remove(var) + # Stop after all required variables have been found + if not variables: + break lastline = line - return val + if variables: + # Fill in missing values + for var in variables: + values[var] = None + return values + +def get_bb_var(var, target=None, postconfig=None): + return get_bb_vars([var], target, postconfig)[var] def get_test_layer(): layers = get_bb_var("BBLAYERS").split() @@ -182,21 +225,30 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec= @contextlib.contextmanager -def runqemu(pn, ssh=True): +def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True): + """ + launch_cmd means directly run the command, don't need set rootfs or env vars. + """ import bb.tinfoil import bb.build tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(False) + tinfoil.prepare(config_only=False, quiet=True) try: tinfoil.logger.setLevel(logging.WARNING) import oeqa.targetcontrol tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") - import oe.recipeutils - recipefile = oe.recipeutils.pn_to_recipe(tinfoil.cooker, pn) - recipedata = oe.recipeutils.parse_recipe(recipefile, [], tinfoil.config_data) + # Tell QemuTarget() whether need find rootfs/kernel or not + if launch_cmd: + tinfoil.config_data.setVar("FIND_ROOTFS", '0') + else: + tinfoil.config_data.setVar("FIND_ROOTFS", '1') + + recipedata = tinfoil.parse_recipe(pn) + for key, value in overrides.items(): + recipedata.setVar(key, value) # The QemuRunner log is saved out, but we need to ensure it is at the right # log level (and then ensure that since it's a child of the BitBake logger, @@ -204,9 +256,9 @@ def runqemu(pn, ssh=True): logger = logging.getLogger('BitBake.QemuRunner') logger.setLevel(logging.DEBUG) logger.propagate = False - logdir = recipedata.getVar("TEST_LOG_DIR", True) + logdir = recipedata.getVar("TEST_LOG_DIR") - qemu = oeqa.targetcontrol.QemuTarget(recipedata) + qemu = oeqa.targetcontrol.QemuTarget(recipedata, image_fstype) finally: # We need to shut down tinfoil early here in case we actually want # to run tinfoil-using utilities with the running QEMU instance. @@ -226,7 +278,7 @@ def runqemu(pn, ssh=True): try: qemu.deploy() try: - qemu.start(ssh=ssh) + qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) except bb.build.FuncFailed: raise Exception('Failed to start QEMU - see the logs in %s' % logdir) @@ -237,3 +289,15 @@ def runqemu(pn, ssh=True): qemu.stop() except: pass + +def updateEnv(env_file): + """ + Source a file and update environment. + """ + + cmd = ". %s; env -0" % env_file + result = runCmd(cmd) + + for line in result.output.split("\0"): + (key, _, value) = line.partition("=") + os.environ[key] = value diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index 0b23565485..d876896921 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -172,27 +172,42 @@ def LogResults(original_class): #check status of tests and record it + tcid = self.id() for (name, msg) in result.errors: - if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]): + if tcid == name.id(): local_log.results("Testcase "+str(test_case)+": ERROR") local_log.results("Testcase "+str(test_case)+":\n"+msg) passed = False for (name, msg) in result.failures: - if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]): + if tcid == name.id(): local_log.results("Testcase "+str(test_case)+": FAILED") local_log.results("Testcase "+str(test_case)+":\n"+msg) passed = False for (name, msg) in result.skipped: - if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]): + if tcid == name.id(): local_log.results("Testcase "+str(test_case)+": SKIPPED") passed = False if passed: local_log.results("Testcase "+str(test_case)+": PASSED") + # XXX: In order to avoid race condition when test if exists the linkfile + # use bb.utils.lock, the best solution is to create a unique name for the + # link file. + try: + import bb + has_bb = True + lockfilename = linkfile + '.lock' + except ImportError: + has_bb = False + + if has_bb: + lf = bb.utils.lockfile(lockfilename, block=True) # Create symlink to the current log - if os.path.exists(linkfile): + if os.path.lexists(linkfile): os.remove(linkfile) os.symlink(logfile, linkfile) + if has_bb: + bb.utils.unlockfile(lf) original_class.run = run diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index 71422a9aea..5a7edc1a86 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py @@ -5,12 +5,6 @@ import datetime import itertools from .commands import runCmd -def get_host_dumper(d): - cmds = d.getVar("testimage_dump_host", True) - parent_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) - return HostDumper(cmds, parent_dir) - - class BaseDumper(object): """ Base class to dump commands from host/target """ @@ -77,13 +71,12 @@ class HostDumper(BaseDumper): result = runCmd(cmd, ignore_status=True) self._write_dump(cmd.split()[0], result.output) - class TargetDumper(BaseDumper): """ Class to get dumps from target, it only works with QemuRunner """ - def __init__(self, cmds, parent_dir, qemurunner): + def __init__(self, cmds, parent_dir, runner): super(TargetDumper, self).__init__(cmds, parent_dir) - self.runner = qemurunner + self.runner = runner def dump_target(self, dump_dir=""): if dump_dir: diff --git a/meta/lib/oeqa/utils/git.py b/meta/lib/oeqa/utils/git.py new file mode 100644 index 0000000000..e0cb3f0db2 --- /dev/null +++ b/meta/lib/oeqa/utils/git.py @@ -0,0 +1,80 @@ +# +# Copyright (C) 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +"""Git repository interactions""" +import os + +from oeqa.utils.commands import runCmd + + +class GitError(Exception): + """Git error handling""" + pass + +class GitRepo(object): + """Class representing a Git repository clone""" + def __init__(self, path, is_topdir=False): + git_dir = self._run_git_cmd_at(['rev-parse', '--git-dir'], path) + git_dir = git_dir if os.path.isabs(git_dir) else os.path.join(path, git_dir) + self.git_dir = os.path.realpath(git_dir) + + if self._run_git_cmd_at(['rev-parse', '--is-bare-repository'], path) == 'true': + self.bare = True + self.top_dir = self.git_dir + else: + self.bare = False + self.top_dir = self._run_git_cmd_at(['rev-parse', '--show-toplevel'], + path) + realpath = os.path.realpath(path) + if is_topdir and realpath != self.top_dir: + raise GitError("{} is not a Git top directory".format(realpath)) + + @staticmethod + def _run_git_cmd_at(git_args, cwd, **kwargs): + """Run git command at a specified directory""" + git_cmd = 'git ' if isinstance(git_args, str) else ['git'] + git_cmd += git_args + ret = runCmd(git_cmd, ignore_status=True, cwd=cwd, **kwargs) + if ret.status: + cmd_str = git_cmd if isinstance(git_cmd, str) \ + else ' '.join(git_cmd) + raise GitError("'{}' failed with exit code {}: {}".format( + cmd_str, ret.status, ret.output)) + return ret.output.strip() + + @staticmethod + def init(path, bare=False): + """Initialize a new Git repository""" + cmd = ['init'] + if bare: + cmd.append('--bare') + GitRepo._run_git_cmd_at(cmd, cwd=path) + return GitRepo(path, is_topdir=True) + + def run_cmd(self, git_args, env_update=None): + """Run Git command""" + env = None + if env_update: + env = os.environ.copy() + env.update(env_update) + return self._run_git_cmd_at(git_args, self.top_dir, env=env) + + def rev_parse(self, revision): + """Do git rev-parse""" + try: + return self.run_cmd(['rev-parse', revision]) + except GitError: + # Revision does not exist + return None + + def get_current_branch(self): + """Get current branch""" + try: + # Strip 11 chars, i.e. 'refs/heads' from the beginning + return self.run_cmd(['symbolic-ref', 'HEAD'])[11:] + except GitError: + return None + + diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index bd76f36468..7d12331453 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -1,8 +1,9 @@ import http.server import multiprocessing import os +from socketserver import ThreadingMixIn -class HTTPServer(http.server.HTTPServer): +class HTTPServer(ThreadingMixIn, http.server.HTTPServer): def server_start(self, root_dir): import signal diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py new file mode 100644 index 0000000000..cb81155e54 --- /dev/null +++ b/meta/lib/oeqa/utils/metadata.py @@ -0,0 +1,118 @@ +# Copyright (C) 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +# Functions to get metadata from the testing host used +# for analytics of test results. + +from collections import OrderedDict +from collections.abc import MutableMapping +from xml.dom.minidom import parseString +from xml.etree.ElementTree import Element, tostring + +from oeqa.utils.commands import runCmd, get_bb_vars + +def get_os_release(): + """Get info from /etc/os-release as a dict""" + data = OrderedDict() + os_release_file = '/etc/os-release' + if not os.path.exists(os_release_file): + return None + with open(os_release_file) as fobj: + for line in fobj: + key, value = line.split('=', 1) + data[key.strip().lower()] = value.strip().strip('"') + return data + +def metadata_from_bb(): + """ Returns test's metadata as OrderedDict. + + Data will be gathered using bitbake -e thanks to get_bb_vars. + """ + metadata_config_vars = ('MACHINE', 'BB_NUMBER_THREADS', 'PARALLEL_MAKE') + + info_dict = OrderedDict() + hostname = runCmd('hostname') + info_dict['hostname'] = hostname.output + data_dict = get_bb_vars() + + # Distro information + info_dict['distro'] = {'id': data_dict['DISTRO'], + 'version_id': data_dict['DISTRO_VERSION'], + 'pretty_name': '%s %s' % (data_dict['DISTRO'], data_dict['DISTRO_VERSION'])} + + # Host distro information + os_release = get_os_release() + if os_release: + info_dict['host_distro'] = OrderedDict() + for key in ('id', 'version_id', 'pretty_name'): + if key in os_release: + info_dict['host_distro'][key] = os_release[key] + + info_dict['layers'] = get_layers(data_dict['BBLAYERS']) + info_dict['bitbake'] = git_rev_info(os.path.dirname(bb.__file__)) + + info_dict['config'] = OrderedDict() + for var in sorted(metadata_config_vars): + info_dict['config'][var] = data_dict[var] + return info_dict + +def metadata_from_data_store(d): + """ Returns test's metadata as OrderedDict. + + Data will be collected from the provided data store. + """ + # TODO: Getting metadata from the data store would + # be useful when running within bitbake. + pass + +def git_rev_info(path): + """Get git revision information as a dict""" + from git import Repo, InvalidGitRepositoryError, NoSuchPathError + + info = OrderedDict() + try: + repo = Repo(path, search_parent_directories=True) + except (InvalidGitRepositoryError, NoSuchPathError): + return info + info['commit'] = repo.head.commit.hexsha + info['commit_count'] = repo.head.commit.count() + try: + info['branch'] = repo.active_branch.name + except TypeError: + info['branch'] = '(nobranch)' + return info + +def get_layers(layers): + """Returns layer information in dict format""" + layer_dict = OrderedDict() + for layer in layers.split(): + layer_name = os.path.basename(layer) + layer_dict[layer_name] = git_rev_info(layer) + return layer_dict + +def write_metadata_file(file_path, metadata): + """ Writes metadata to a XML file in directory. """ + + xml = dict_to_XML('metadata', metadata) + xml_doc = parseString(tostring(xml).decode('UTF-8')) + with open(file_path, 'w') as f: + f.write(xml_doc.toprettyxml()) + +def dict_to_XML(tag, dictionary, **kwargs): + """ Return XML element converting dicts recursively. """ + + elem = Element(tag, **kwargs) + for key, val in dictionary.items(): + if tag == 'layers': + child = (dict_to_XML('layer', val, name=key)) + elif isinstance(val, MutableMapping): + child = (dict_to_XML(key, val)) + else: + if tag == 'config': + child = Element('variable', name=key) + else: + child = Element(key) + child.text = str(val) + elem.append(child) + return elem diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py index 099ecc9728..724afb2b5e 100644 --- a/meta/lib/oeqa/utils/package_manager.py +++ b/meta/lib/oeqa/utils/package_manager.py @@ -1,29 +1,210 @@ +import os +import json +import shutil + +from oeqa.core.utils.test import getCaseFile, getCaseMethod + def get_package_manager(d, root_path): """ Returns an OE package manager that can install packages in root_path. """ from oe.package_manager import RpmPM, OpkgPM, DpkgPM - pkg_class = d.getVar("IMAGE_PKGTYPE", True) + pkg_class = d.getVar("IMAGE_PKGTYPE") if pkg_class == "rpm": pm = RpmPM(d, root_path, - d.getVar('TARGET_VENDOR', True)) + d.getVar('TARGET_VENDOR')) pm.create_configs() elif pkg_class == "ipk": pm = OpkgPM(d, root_path, - d.getVar("IPKGCONF_TARGET", True), - d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)) + d.getVar("IPKGCONF_TARGET"), + d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) elif pkg_class == "deb": pm = DpkgPM(d, root_path, - d.getVar('PACKAGE_ARCHS', True), - d.getVar('DPKG_ARCH', True)) + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH')) pm.write_index() pm.update() return pm + +def find_packages_to_extract(test_suite): + """ + Returns packages to extract required by runtime tests. + """ + from oeqa.core.utils.test import getSuiteCasesFiles + + needed_packages = {} + files = getSuiteCasesFiles(test_suite) + + for f in set(files): + json_file = _get_json_file(f) + if json_file: + needed_packages.update(_get_needed_packages(json_file)) + + return needed_packages + +def _get_json_file(module_path): + """ + Returns the path of the JSON file for a module, empty if doesn't exitst. + """ + + json_file = '%s.json' % module_path.rsplit('.', 1)[0] + if os.path.isfile(module_path) and os.path.isfile(json_file): + return json_file + else: + return '' + +def _get_needed_packages(json_file, test=None): + """ + Returns a dict with needed packages based on a JSON file. + + If a test is specified it will return the dict just for that test. + """ + needed_packages = {} + + with open(json_file) as f: + test_packages = json.load(f) + for key,value in test_packages.items(): + needed_packages[key] = value + + if test: + if test in needed_packages: + needed_packages = needed_packages[test] + else: + needed_packages = {} + + return needed_packages + +def extract_packages(d, needed_packages): + """ + Extract packages that will be needed during runtime. + """ + + import bb + import oe.path + + extracted_path = d.getVar('TEST_EXTRACTED_DIR') + + for key,value in needed_packages.items(): + packages = () + if isinstance(value, dict): + packages = (value, ) + elif isinstance(value, list): + packages = value + else: + bb.fatal('Failed to process needed packages for %s; ' + 'Value must be a dict or list' % key) + + for package in packages: + pkg = package['pkg'] + rm = package.get('rm', False) + extract = package.get('extract', True) + + if extract: + #logger.debug(1, 'Extracting %s' % pkg) + dst_dir = os.path.join(extracted_path, pkg) + # Same package used for more than one test, + # don't need to extract again. + if os.path.exists(dst_dir): + continue + + # Extract package and copy it to TEST_EXTRACTED_DIR + pkg_dir = _extract_in_tmpdir(d, pkg) + oe.path.copytree(pkg_dir, dst_dir) + shutil.rmtree(pkg_dir) + + else: + #logger.debug(1, 'Copying %s' % pkg) + _copy_package(d, pkg) + +def _extract_in_tmpdir(d, pkg): + """" + Returns path to a temp directory where the package was + extracted without dependencies. + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(d.getVar('TEST_INSTALL_TMP_DIR'), pkg) + pm = get_package_manager(d, pkg_path) + extract_dir = pm.extract(pkg) + shutil.rmtree(pkg_path) + + return extract_dir + +def _copy_package(d, pkg): + """ + Copy the RPM, DEB or IPK package to dst_dir + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(d.getVar('TEST_INSTALL_TMP_DIR'), pkg) + dst_dir = d.getVar('TEST_PACKAGED_DIR') + pm = get_package_manager(d, pkg_path) + pkg_info = pm.package_info(pkg) + file_path = pkg_info[pkg]['filepath'] + shutil.copy2(file_path, dst_dir) + shutil.rmtree(pkg_path) + +def install_package(test_case): + """ + Installs package in DUT if required. + """ + needed_packages = test_needs_package(test_case) + if needed_packages: + _install_uninstall_packages(needed_packages, test_case, True) + +def uninstall_package(test_case): + """ + Uninstalls package in DUT if required. + """ + needed_packages = test_needs_package(test_case) + if needed_packages: + _install_uninstall_packages(needed_packages, test_case, False) + +def test_needs_package(test_case): + """ + Checks if a test case requires to install/uninstall packages. + """ + test_file = getCaseFile(test_case) + json_file = _get_json_file(test_file) + + if json_file: + test_method = getCaseMethod(test_case) + needed_packages = _get_needed_packages(json_file, test_method) + if needed_packages: + return needed_packages + + return None + +def _install_uninstall_packages(needed_packages, test_case, install=True): + """ + Install/Uninstall packages in the DUT without using a package manager + """ + + if isinstance(needed_packages, dict): + packages = [needed_packages] + elif isinstance(needed_packages, list): + packages = needed_packages + + for package in packages: + pkg = package['pkg'] + rm = package.get('rm', False) + extract = package.get('extract', True) + src_dir = os.path.join(test_case.tc.extract_dir, pkg) + + # Install package + if install and extract: + test_case.tc.target.copyDirTo(src_dir, '/') + + # Uninstall package + elif not install and rm: + test_case.tc.target.deleteDirStructure(src_dir, '/') diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index b8ac3f0bb6..ba44b96f53 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -7,6 +7,7 @@ import subprocess import os +import sys import time import signal import re @@ -20,6 +21,7 @@ from oeqa.utils.dump import HostDumper import logging logger = logging.getLogger("BitBake.QemuRunner") +logger.addHandler(logging.StreamHandler()) # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -29,16 +31,18 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm): # Popen object for runqemu self.runqemu = None # pid of the qemu process that runqemu will start self.qemupid = None - # target ip - from the command line + # target ip - from the command line or runqemu output self.ip = None # host ip - where qemu is running self.server_ip = None + # target ip netmask + self.netmask = None self.machine = machine self.rootfs = rootfs @@ -49,6 +53,7 @@ class QemuRunner: self.boottime = boottime self.logged = False self.thread = None + self.use_kvm = use_kvm self.runqemutime = 60 self.host_dumper = HostDumper(dump_host_cmds, dump_dir) @@ -71,7 +76,7 @@ class QemuRunner: if self.logfile: # It is needed to sanitize the data received from qemu # because is possible to have control characters - msg = msg.decode("utf-8") + msg = msg.decode("utf-8", errors='ignore') msg = re_control_char.sub('', msg) with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) @@ -92,7 +97,7 @@ class QemuRunner: self._dump_host() raise SystemExit - def start(self, qemuparams = None, get_ip = True): + def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): if self.display: os.environ["DISPLAY"] = self.display # Set this flag so that Qemu doesn't do any grabs as SDL grabs @@ -112,6 +117,20 @@ class QemuRunner: else: os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + if not launch_cmd: + launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams) + if self.use_kvm: + logger.info('Using kvm for runqemu') + launch_cmd += ' kvm' + else: + logger.info('Not using kvm for runqemu') + if not self.display: + launch_cmd += ' nographic' + launch_cmd += ' %s %s' % (self.machine, self.rootfs) + + return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams) + + def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None): try: threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() @@ -119,17 +138,21 @@ class QemuRunner: logger.error("Failed to create listening socket: %s" % msg[1]) return False + bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' + if extra_bootparams: + bootparams = bootparams + ' ' + extra_bootparams - self.qemuparams = 'bootparams="console=tty1 console=ttyS0,115200n8 printk.time=1" qemuparams="-serial tcp:127.0.0.1:{}"'.format(threadport) - if not self.display: - self.qemuparams = 'nographic ' + self.qemuparams + self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1}"'.format(bootparams, threadport) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' + launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) + self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) - launch_cmd = 'runqemu tcpserial=%s %s %s %s' % (self.serverport, self.machine, self.rootfs, self.qemuparams) + logger.info('launchcmd=%s'%(launch_cmd)) + # FIXME: We pass in stdin=subprocess.PIPE here to work around stty # blocking at the end of the runqemu script when using this within # oe-selftest (this makes stty error out immediately). There ought @@ -138,12 +161,12 @@ class QemuRunner: output = self.runqemu.stdout # - # We need the preexec_fn above so that all runqemu processes can easily be killed + # We need the preexec_fn above so that all runqemu processes can easily be killed # (by killing their process group). This presents a problem if this controlling - # process itself is killed however since those processes don't notice the death + # process itself is killed however since those processes don't notice the death # of the parent and merrily continue on. # - # Rather than hack runqemu to deal with this, we add something here instead. + # Rather than hack runqemu to deal with this, we add something here instead. # Basically we fork off another process which holds an open pipe to the parent # and also is setpgrp. If/when the pipe sees EOF from the parent dieing, it kills # the process group. This is like pctrl's PDEATHSIG but for a process group @@ -177,6 +200,8 @@ class QemuRunner: return False time.sleep(1) + out = self.getOutput(output) + netconf = False # network configuration is not required by default if self.is_alive(): logger.info("qemu started - qemu procces pid is %s" % self.qemupid) if get_ip: @@ -188,17 +213,27 @@ class QemuRunner: cmdline = re_control_char.sub('', cmdline) try: ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - if not ips or len(ips) != 3: - raise ValueError - else: - self.ip = ips[0] - self.server_ip = ips[1] + self.ip = ips[0] + self.server_ip = ips[1] + logger.info("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): - logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) - self._dump_host() - self.stop() - return False - logger.info("qemu cmdline used:\n{}".format(cmdline)) + # Try to get network configuration from runqemu output + match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + out, re.MULTILINE|re.DOTALL) + if match: + self.ip, self.server_ip, self.netmask = match.groups() + # network configuration is required as we couldn't get it + # from the runqemu command line, so qemu doesn't run kernel + # and guest networking is not configured + netconf = True + else: + logger.error("Couldn't get ip from qemu command line and runqemu output! " + "Here is the qemu command line used:\n%s\n" + "and output from runqemu:\n%s" % (cmdline, out)) + self._dump_host() + self.stop() + return False + logger.info("Target IP: %s" % self.ip) logger.info("Server IP: %s" % self.server_ip) @@ -207,12 +242,11 @@ class QemuRunner: if not self.thread.connection_established.wait(self.boottime): logger.error("Didn't receive a console connection from qemu. " "Here is the qemu command line used:\n%s\nand " - "output from runqemu:\n%s" % (cmdline, - self.getOutput(output))) + "output from runqemu:\n%s" % (cmdline, out)) self.stop_thread() return False - logger.info("Output from runqemu:\n%s", self.getOutput(output)) + logger.info("Output from runqemu:\n%s", out) logger.info("Waiting at most %d seconds for login banner" % self.boottime) endtime = time.time() + self.boottime socklist = [self.server_socket] @@ -222,7 +256,10 @@ class QemuRunner: bootlog = '' data = b'' while time.time() < endtime and not stopread: - sread, swrite, serror = select.select(socklist, [], [], 5) + try: + sread, swrite, serror = select.select(socklist, [], [], 5) + except InterruptedError: + continue for sock in sread: if sock is self.server_socket: qemusock, addr = self.server_socket.accept() @@ -264,6 +301,14 @@ class QemuRunner: if re.search("root@[a-zA-Z0-9\-]+:~#", output): self.logged = True logger.info("Logged as root in serial console") + if netconf: + # configure guest networking + cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) + output = self.run_serial(cmd, raw=True)[1] + if re.search("root@[a-zA-Z0-9\-]+:~#", output): + logger.info("configured ip address %s", self.ip) + else: + logger.info("Couldn't configure guest networking") else: logger.info("Couldn't login into serial console" " as root using blank password") @@ -281,6 +326,7 @@ class QemuRunner: def stop(self): self.stop_thread() + self.stop_qemu_system() if hasattr(self, "origchldhandler"): signal.signal(signal.SIGCHLD, self.origchldhandler) if self.runqemu: @@ -305,6 +351,14 @@ class QemuRunner: self.qemupid = None self.ip = None + def stop_qemu_system(self): + if self.qemupid: + try: + # qemu-system behaves well and a SIGTERM is enough + os.kill(self.qemupid, signal.SIGTERM) + except ProcessLookupError as e: + logger.warn('qemu-system ended unexpectedly') + def stop_thread(self): if self.thread and self.thread.is_alive(): self.thread.stop() @@ -364,14 +418,14 @@ class QemuRunner: newparents = next #print("Children matching %s:" % str(parents)) for p in parents: - # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx" + # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" basecmd = commands[p].split()[0] basecmd = os.path.basename(basecmd) if "qemu-system" in basecmd and "-serial tcp" in commands[p]: return [int(p),commands[p]] - def run_serial(self, command, raw=False): + def run_serial(self, command, raw=False, timeout=5): # We assume target system have echo to get command status if not raw: command = "%s; echo $?\n" % command @@ -379,20 +433,26 @@ class QemuRunner: data = '' status = 0 self.server_socket.sendall(command.encode('utf-8')) - keepreading = True - while keepreading: - sread, _, _ = select.select([self.server_socket],[],[],5) + start = time.time() + end = start + timeout + while True: + now = time.time() + if now >= end: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout + break + try: + sread, _, _ = select.select([self.server_socket],[],[], end - now) + except InterruptedError: + continue if sread: answer = self.server_socket.recv(1024) if answer: data += answer.decode('utf-8') # Search the prompt to stop if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): - keepreading = False + break else: raise Exception("No data on serial console socket") - else: - keepreading = False if data: if raw: diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py index c823157ad6..1bf59007ff 100644 --- a/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/meta/lib/oeqa/utils/qemutinyrunner.py @@ -60,7 +60,7 @@ class QemuTinyRunner(QemuRunner): with open(self.logfile, "a") as f: f.write("%s" % msg) - def start(self, qemuparams = None): + def start(self, qemuparams = None, ssh=True, extra_bootparams=None, runqemuparams='', discard_writes=True): if self.display: os.environ["DISPLAY"] = self.display @@ -107,14 +107,17 @@ class QemuTinyRunner(QemuRunner): return self.is_alive() - def run_serial(self, command): + def run_serial(self, command, timeout=5): self.server_socket.sendall(command+'\n') data = '' status = 0 stopread = False - endtime = time.time()+5 + endtime = time.time()+timeout while time.time()<endtime and not stopread: - sread, _, _ = select.select([self.server_socket],[],[],5) + try: + sread, _, _ = select.select([self.server_socket],[],[],1) + except InterruptedError: + continue for sock in sread: answer = sock.recv(1024) if answer: @@ -124,6 +127,8 @@ class QemuTinyRunner(QemuRunner): stopread = True if not data: status = 1 + if not stopread: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout return (status, str(data)) def find_child(self,parent_pid): @@ -162,9 +167,9 @@ class QemuTinyRunner(QemuRunner): newparents = next #print("Children matching %s:" % str(parents)) for p in parents: - # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx" + # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" basecmd = commands[p].split()[0] basecmd = os.path.basename(basecmd) if "qemu-system" in basecmd and "-serial unix" in commands[p]: - return [int(p),commands[p]]
\ No newline at end of file + return [int(p),commands[p]] diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index f5d46e03cc..05d6502550 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -52,17 +52,19 @@ class SSHProcess(object): endtime = self.starttime + timeout eof = False while time.time() < endtime and not eof: - if select.select([self.process.stdout], [], [], 5)[0] != []: - data = os.read(self.process.stdout.fileno(), 1024) - if not data: - self.process.stdout.close() - eof = True - else: - data = data.decode("utf-8") - output += data - self.log(data) - endtime = time.time() + timeout - + try: + if select.select([self.process.stdout], [], [], 5)[0] != []: + data = os.read(self.process.stdout.fileno(), 1024) + if not data: + self.process.stdout.close() + eof = True + else: + data = data.decode("utf-8") + output += data + self.log(data) + endtime = time.time() + timeout + except InterruptedError: + continue # process hasn't returned yet if not eof: @@ -147,8 +149,13 @@ class SSHControl(object): return self._internal_run(command, timeout, self.ignore_status) def copy_to(self, localpath, remotepath): - command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] - return self._internal_run(command, ignore_status=False) + if os.path.islink(localpath): + link = os.readlink(localpath) + dst_dir, dst_base = os.path.split(remotepath) + return self.run("cd %s; ln -s %s %s" % (dst_dir, link, dst_base)) + else: + command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] + return self._internal_run(command, ignore_status=False) def copy_from(self, remotepath, localpath): command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath] diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py new file mode 100644 index 0000000000..1f7d11b55c --- /dev/null +++ b/meta/lib/oeqa/utils/subprocesstweak.py @@ -0,0 +1,19 @@ +import subprocess + +class OETestCalledProcessError(subprocess.CalledProcessError): + def __str__(self): + def strify(o): + if isinstance(o, bytes): + return o.decode("utf-8", errors="replace") + else: + return o + + s = "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + if hasattr(self, "output") and self.output: + s = s + "\nStandard Output: " + strify(self.output) + if hasattr(self, "stderr") and self.stderr: + s = s + "\nStandard Error: " + strify(self.stderr) + return s + +def errors_have_output(): + subprocess.CalledProcessError = OETestCalledProcessError diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py index d538f6b65f..9249fa2635 100644 --- a/meta/lib/oeqa/utils/targetbuild.py +++ b/meta/lib/oeqa/utils/targetbuild.py @@ -8,23 +8,28 @@ import os import re import bb.utils import subprocess +import tempfile from abc import ABCMeta, abstractmethod class BuildProject(metaclass=ABCMeta): - def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"): + def __init__(self, d, uri, foldername=None, tmpdir=None): self.d = d self.uri = uri self.archive = os.path.basename(uri) + if not tmpdir: + tmpdir = self.d.getVar('WORKDIR') + if not tmpdir: + tmpdir = tempfile.mkdtemp(prefix='buildproject') self.localarchive = os.path.join(tmpdir,self.archive) - self.fname = re.sub(r'.tar.bz2|tar.gz$', '', self.archive) if foldername: self.fname = foldername + else: + self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive) # Download self.archive to self.localarchive def _download_archive(self): - - dl_dir = self.d.getVar("DL_DIR", True) + dl_dir = self.d.getVar("DL_DIR") if dl_dir and os.path.exists(os.path.join(dl_dir, self.archive)): bb.utils.copyfile(os.path.join(dl_dir, self.archive), self.localarchive) return @@ -39,12 +44,12 @@ class BuildProject(metaclass=ABCMeta): cmd = '' for var in exportvars: - val = self.d.getVar(var, True) + val = self.d.getVar(var) if val: cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri) - subprocess.check_call(cmd, shell=True) + subprocess.check_output(cmd, shell=True) # This method should provide a way to run a command in the desired environment. @abstractmethod @@ -72,7 +77,7 @@ class TargetBuildProject(BuildProject): def __init__(self, target, d, uri, foldername=None): self.target = target self.targetdir = "~/" - BuildProject.__init__(self, d, uri, foldername, tmpdir="/tmp") + BuildProject.__init__(self, d, uri, foldername) def download_archive(self): @@ -102,8 +107,8 @@ class SDKBuildProject(BuildProject): self.testdir = testpath self.targetdir = testpath bb.utils.mkdirhier(testpath) - self.datetime = d.getVar('DATETIME', True) - self.testlogdir = d.getVar("TEST_LOG_DIR", True) + self.datetime = d.getVar('DATETIME') + self.testlogdir = d.getVar("TEST_LOG_DIR") bb.utils.mkdirhier(self.testlogdir) self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime) BuildProject.__init__(self, d, uri, foldername, tmpdir=testpath) @@ -113,13 +118,13 @@ class SDKBuildProject(BuildProject): self._download_archive() cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir) - subprocess.check_call(cmd, shell=True) + subprocess.check_output(cmd, shell=True) #Change targetdir to project folder - self.targetdir = self.targetdir + self.fname + self.targetdir = os.path.join(self.targetdir, self.fname) - def run_configure(self, configure_args=''): - return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=' gnu-configize; ') + def run_configure(self, configure_args='', extra_cmds=' gnu-configize; '): + return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=extra_cmds) def run_install(self, install_args=''): return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir)) @@ -132,4 +137,3 @@ class SDKBuildProject(BuildProject): def _run(self, cmd): self.log("Running . %s; " % self.sdkenv + cmd) return subprocess.call(". %s; " % self.sdkenv + cmd, shell=True) - diff --git a/meta/lib/oeqa/utils/testexport.py b/meta/lib/oeqa/utils/testexport.py index 57be2ca449..be2a2110fc 100644 --- a/meta/lib/oeqa/utils/testexport.py +++ b/meta/lib/oeqa/utils/testexport.py @@ -72,9 +72,9 @@ def process_binaries(d, params): return extract_bin_command if determine_if_poky_env(): # machine with poky environment - exportpath = d.getVar("TEST_EXPORT_DIR", True) if export_env else d.getVar("DEPLOY_DIR", True) - rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM", True) - arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(rpm_deploy_dir)) + exportpath = d.getVar("TEST_EXPORT_DIR") if export_env else d.getVar("DEPLOY_DIR") + rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM") + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(rpm_deploy_dir)) arch_rpm_dir = os.path.join(rpm_deploy_dir, arch) extracted_bin_dir = os.path.join(exportpath,"binaries", arch, "extracted_binaries") packaged_bin_dir = os.path.join(exportpath,"binaries", arch, "packaged_binaries") @@ -92,7 +92,7 @@ def process_binaries(d, params): return "" for item in native_rpm_file_list:# will copy all versions of package. Used version will be selected on remote machine bb.plain("Copying native package file: %s" % item) - sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR", True), "binaries", "native")) + sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries", "native")) else: # nothing to do here; running tests under bitbake, so we asume native binaries are in sysroots dir. if param_list[1] or param_list[4]: bb.warn("Native binary %s %s%s. Running tests under bitbake environment. Version can't be checked except when the test itself does it" @@ -148,7 +148,7 @@ def process_binaries(d, params): else: # this is for target device if param_list[2] == "rpm": return "No need to extract, this is an .rpm file" - arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(binaries_path)) + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(binaries_path)) extracted_bin_path = os.path.join(binaries_path, arch, "extracted_binaries") extracted_bin_list = [item for item in os.listdir(extracted_bin_path)] packaged_bin_path = os.path.join(binaries_path, arch, "packaged_binaries") @@ -206,9 +206,9 @@ def send_bin_to_DUT(d,params): from oeqa.oetest import oeRuntimeTest param_list = params cleanup_list = list() - bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "binaries") if determine_if_poky_env() \ + bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries") if determine_if_poky_env() \ else os.getenv("bin_dir") - arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(bins_dir)) + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(bins_dir)) arch_rpms_dir = os.path.join(bins_dir, arch, "packaged_binaries") extracted_bin_dir = os.path.join(bins_dir, arch, "extracted_binaries", param_list[0]) diff --git a/meta/lib/rootfspostcommands.py b/meta/lib/rootfspostcommands.py new file mode 100644 index 0000000000..4742e0613c --- /dev/null +++ b/meta/lib/rootfspostcommands.py @@ -0,0 +1,56 @@ +import os + +def sort_file(filename, mapping): + """ + Sorts a passwd or group file based on the numeric ID in the third column. + If a mapping is given, the name from the first column is mapped via that + dictionary instead (necessary for /etc/shadow and /etc/gshadow). If not, + a new mapping is created on the fly and returned. + """ + new_mapping = {} + with open(filename, 'rb+') as f: + lines = f.readlines() + # No explicit error checking for the sake of simplicity. /etc + # files are assumed to be well-formed, causing exceptions if + # not. + for line in lines: + entries = line.split(b':') + name = entries[0] + if mapping is None: + id = int(entries[2]) + else: + id = mapping[name] + new_mapping[name] = id + # Sort by numeric id first, with entire line as secondary key + # (just in case that there is more than one entry for the same id). + lines.sort(key=lambda line: (new_mapping[line.split(b':')[0]], line)) + # We overwrite the entire file, i.e. no truncate() necessary. + f.seek(0) + f.write(b''.join(lines)) + return new_mapping + +def remove_backup(filename): + """ + Removes the backup file for files like /etc/passwd. + """ + backup_filename = filename + '-' + if os.path.exists(backup_filename): + os.unlink(backup_filename) + +def sort_passwd(sysconfdir): + """ + Sorts passwd and group files in a rootfs /etc directory by ID. + Backup files are sometimes are inconsistent and then cannot be + sorted (YOCTO #11043), and more importantly, are not needed in + the initial rootfs, so they get deleted. + """ + for main, shadow in (('passwd', 'shadow'), + ('group', 'gshadow')): + filename = os.path.join(sysconfdir, main) + remove_backup(filename) + if os.path.exists(filename): + mapping = sort_file(filename, None) + filename = os.path.join(sysconfdir, shadow) + remove_backup(filename) + if os.path.exists(filename): + sort_file(filename, mapping) |