diff options
Diffstat (limited to 'meta/lib/oe')
30 files changed, 8192 insertions, 675 deletions
diff --git a/meta/lib/oe/__init__.py b/meta/lib/oe/__init__.py index e69de29bb2..3ad9513f40 100644 --- a/meta/lib/oe/__init__.py +++ b/meta/lib/oe/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py new file mode 100644 index 0000000000..3a5b7b6b44 --- /dev/null +++ b/meta/lib/oe/buildhistory_analysis.py @@ -0,0 +1,607 @@ +# Report significant differences in the buildhistory repository since a specific revision +# +# Copyright (C) 2012-2013, 2016-2017 Intel Corporation +# Author: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Note: requires GitPython 0.3.1+ +# +# You can use this from the command line by running scripts/buildhistory-diff +# + +import sys +import os.path +import difflib +import git +import re +import hashlib +import collections +import bb.utils +import bb.tinfoil + + +# How to display fields +list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] +list_order_fields = ['PACKAGES'] +defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'} +numeric_fields = ['PKGSIZE', 'IMAGESIZE'] +# Fields to monitor +monitor_fields = ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RREPLACES', 'RCONFLICTS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE', 'PKG'] +ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR'] +# Percentage change to alert for numeric fields +monitor_numeric_threshold = 10 +# Image files to monitor (note that image-info.txt is handled separately) +img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt'] +# Related context fields for reporting (note: PE, PV & PR are always reported for monitored package fields) +related_fields = {} +related_fields['RDEPENDS'] = ['DEPENDS'] +related_fields['RRECOMMENDS'] = ['DEPENDS'] +related_fields['FILELIST'] = ['FILES'] +related_fields['PKGSIZE'] = ['FILELIST'] +related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND'] +related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'NO_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] + + +class ChangeRecord: + def __init__(self, path, fieldname, oldvalue, newvalue, monitored): + self.path = path + self.fieldname = fieldname + self.oldvalue = oldvalue + self.newvalue = newvalue + self.monitored = monitored + self.related = [] + self.filechanges = None + + def __str__(self): + return self._str_internal(True) + + def _str_internal(self, outer): + if outer: + if '/image-files/' in self.path: + prefix = '%s: ' % self.path.split('/image-files/')[0] + else: + prefix = '%s: ' % self.path + else: + prefix = '' + + def pkglist_combine(depver): + pkglist = [] + for k,v in depver.items(): + if v: + pkglist.append("%s (%s)" % (k,v)) + else: + pkglist.append(k) + return pkglist + + def detect_renamed_dirs(aitems, bitems): + adirs = set(map(os.path.dirname, aitems)) + bdirs = set(map(os.path.dirname, bitems)) + files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \ + for name in adirs - bdirs] + files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \ + for name in bdirs - adirs] + renamed_dirs = [(dir1, dir2) for dir1, files1 in files_ab for dir2, files2 in files_ba if files1 == files2] + # remove files that belong to renamed dirs from aitems and bitems + for dir1, dir2 in renamed_dirs: + aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)] + bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)] + return renamed_dirs, aitems, bitems + + if self.fieldname in list_fields or self.fieldname in list_order_fields: + renamed_dirs = [] + if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: + (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue) + aitems = pkglist_combine(depvera) + bitems = pkglist_combine(depverb) + else: + aitems = self.oldvalue.split() + bitems = self.newvalue.split() + if self.fieldname == 'FILELIST': + renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + + removed = list(set(aitems) - set(bitems)) + added = list(set(bitems) - set(aitems)) + + lines = [] + if renamed_dirs: + for dfrom, dto in renamed_dirs: + lines.append('directory renamed %s -> %s' % (dfrom, dto)) + if removed or added: + if removed and not bitems: + lines.append('removed all items "%s"' % ' '.join(removed)) + else: + if removed: + lines.append('removed "%s"' % ' '.join(removed)) + if added: + lines.append('added "%s"' % ' '.join(added)) + else: + lines.append('changed order') + + out = '%s: %s' % (self.fieldname, ', '.join(lines)) + + elif self.fieldname in numeric_fields: + aval = int(self.oldvalue or 0) + bval = int(self.newvalue or 0) + if aval != 0: + percentchg = ((bval - aval) / float(aval)) * 100 + else: + percentchg = 100 + out = '%s changed from %s to %s (%s%d%%)' % (self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg) + elif self.fieldname in defaultval_map: + out = '%s changed from %s to %s' % (self.fieldname, self.oldvalue, self.newvalue) + if self.fieldname == 'PKG' and '[default]' in self.newvalue: + out += ' - may indicate debian renaming failure' + elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']: + if self.oldvalue and self.newvalue: + out = '%s changed:\n ' % self.fieldname + elif self.newvalue: + out = '%s added:\n ' % self.fieldname + elif self.oldvalue: + out = '%s cleared:\n ' % self.fieldname + alines = self.oldvalue.splitlines() + blines = self.newvalue.splitlines() + diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='') + out += '\n '.join(list(diff)[2:]) + out += '\n --' + elif self.fieldname in img_monitor_files or '/image-files/' in self.path: + fieldname = self.fieldname + if '/image-files/' in self.path: + fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname) + out = 'Changes to %s:\n ' % fieldname + else: + if outer: + prefix = 'Changes to %s ' % self.path + out = '(%s):\n ' % self.fieldname + if self.filechanges: + out += '\n '.join(['%s' % i for i in self.filechanges]) + else: + alines = self.oldvalue.splitlines() + blines = self.newvalue.splitlines() + diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='') + out += '\n '.join(list(diff)) + out += '\n --' + else: + out = '%s changed from "%s" to "%s"' % (self.fieldname, self.oldvalue, self.newvalue) + + if self.related: + for chg in self.related: + if not outer and chg.fieldname in ['PE', 'PV', 'PR']: + continue + for line in chg._str_internal(False).splitlines(): + out += '\n * %s' % line + + return '%s%s' % (prefix, out) + +class FileChange: + changetype_add = 'A' + changetype_remove = 'R' + changetype_type = 'T' + changetype_perms = 'P' + changetype_ownergroup = 'O' + changetype_link = 'L' + + def __init__(self, path, changetype, oldvalue = None, newvalue = None): + self.path = path + self.changetype = changetype + self.oldvalue = oldvalue + self.newvalue = newvalue + + def _ftype_str(self, ftype): + if ftype == '-': + return 'file' + elif ftype == 'd': + return 'directory' + elif ftype == 'l': + return 'symlink' + elif ftype == 'c': + return 'char device' + elif ftype == 'b': + return 'block device' + elif ftype == 'p': + return 'fifo' + elif ftype == 's': + return 'socket' + else: + return 'unknown (%s)' % ftype + + def __str__(self): + if self.changetype == self.changetype_add: + return '%s was added' % self.path + elif self.changetype == self.changetype_remove: + return '%s was removed' % self.path + elif self.changetype == self.changetype_type: + return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue)) + elif self.changetype == self.changetype_perms: + return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue) + elif self.changetype == self.changetype_ownergroup: + return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue) + elif self.changetype == self.changetype_link: + return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue) + else: + return '%s changed (unknown)' % self.path + + +def blob_to_dict(blob): + alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] + adict = {} + for line in alines: + splitv = [i.strip() for i in line.split('=',1)] + if len(splitv) > 1: + adict[splitv[0]] = splitv[1] + return adict + + +def file_list_to_dict(lines): + adict = {} + for line in lines: + # Leave the last few fields intact so we handle file names containing spaces + splitv = line.split(None,4) + # Grab the path and remove the leading . + path = splitv[4][1:].strip() + # Handle symlinks + if(' -> ' in path): + target = path.split(' -> ')[1] + path = path.split(' -> ')[0] + adict[path] = splitv[0:3] + [target] + else: + adict[path] = splitv[0:3] + return adict + + +def compare_file_lists(alines, blines): + adict = file_list_to_dict(alines) + bdict = file_list_to_dict(blines) + filechanges = [] + for path, splitv in adict.items(): + newsplitv = bdict.pop(path, None) + if newsplitv: + # Check type + oldvalue = splitv[0][0] + newvalue = newsplitv[0][0] + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue)) + # Check permissions + oldvalue = splitv[0][1:] + newvalue = newsplitv[0][1:] + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue)) + # Check owner/group + oldvalue = '%s/%s' % (splitv[1], splitv[2]) + newvalue = '%s/%s' % (newsplitv[1], newsplitv[2]) + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue)) + # Check symlink target + if newsplitv[0][0] == 'l': + if len(splitv) > 3: + oldvalue = splitv[3] + else: + oldvalue = None + newvalue = newsplitv[3] + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue)) + else: + filechanges.append(FileChange(path, FileChange.changetype_remove)) + + # Whatever is left over has been added + for path in bdict: + filechanges.append(FileChange(path, FileChange.changetype_add)) + + return filechanges + + +def compare_lists(alines, blines): + removed = list(set(alines) - set(blines)) + added = list(set(blines) - set(alines)) + + filechanges = [] + for pkg in removed: + filechanges.append(FileChange(pkg, FileChange.changetype_remove)) + for pkg in added: + filechanges.append(FileChange(pkg, FileChange.changetype_add)) + + return filechanges + + +def compare_pkg_lists(astr, bstr): + depvera = bb.utils.explode_dep_versions2(astr) + depverb = bb.utils.explode_dep_versions2(bstr) + + # Strip out changes where the version has increased + remove = [] + for k in depvera: + if k in depverb: + dva = depvera[k] + dvb = depverb[k] + if dva and dvb and len(dva) == len(dvb): + # Since length is the same, sort so that prefixes (e.g. >=) will line up + dva.sort() + dvb.sort() + removeit = True + for dvai, dvbi in zip(dva, dvb): + if dvai != dvbi: + aiprefix = dvai.split(' ')[0] + biprefix = dvbi.split(' ')[0] + if aiprefix == biprefix and aiprefix in ['>=', '=']: + if bb.utils.vercmp(bb.utils.split_version(dvai), bb.utils.split_version(dvbi)) > 0: + removeit = False + break + else: + removeit = False + break + if removeit: + remove.append(k) + + for k in remove: + depvera.pop(k) + depverb.pop(k) + + return (depvera, depverb) + + +def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): + adict = blob_to_dict(ablob) + bdict = blob_to_dict(bblob) + + pkgname = os.path.basename(path) + + defaultvals = {} + defaultvals['PKG'] = pkgname + defaultvals['PKGE'] = '0' + + changes = [] + keys = list(set(adict.keys()) | set(bdict.keys()) | set(defaultval_map.keys())) + for key in keys: + astr = adict.get(key, '') + bstr = bdict.get(key, '') + if key in ver_monitor_fields: + monitored = report_ver or astr or bstr + else: + monitored = key in monitor_fields + mapped_key = defaultval_map.get(key, '') + if mapped_key: + if not astr: + astr = '%s [default]' % adict.get(mapped_key, defaultvals.get(key, '')) + if not bstr: + bstr = '%s [default]' % bdict.get(mapped_key, defaultvals.get(key, '')) + + if astr != bstr: + if (not report_all) and key in numeric_fields: + aval = int(astr or 0) + bval = int(bstr or 0) + if aval != 0: + percentchg = ((bval - aval) / float(aval)) * 100 + else: + percentchg = 100 + if abs(percentchg) < monitor_numeric_threshold: + continue + elif (not report_all) and key in list_fields: + if key == "FILELIST" and path.endswith("-dbg") and bstr.strip() != '': + continue + if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: + (depvera, depverb) = compare_pkg_lists(astr, bstr) + if depvera == depverb: + continue + alist = astr.split() + alist.sort() + blist = bstr.split() + blist.sort() + # We don't care about the removal of self-dependencies + if pkgname in alist and not pkgname in blist: + alist.remove(pkgname) + if ' '.join(alist) == ' '.join(blist): + continue + + if key == 'PKGR' and not report_all: + vers = [] + # strip leading 'r' and dots + for ver in (astr.split()[0], bstr.split()[0]): + if ver.startswith('r'): + ver = ver[1:] + vers.append(ver.replace('.', '')) + maxlen = max(len(vers[0]), len(vers[1])) + try: + # pad with '0' and convert to int + vers = [int(ver.ljust(maxlen, '0')) for ver in vers] + except ValueError: + pass + else: + # skip decrements and increments + if abs(vers[0] - vers[1]) == 1: + continue + + chg = ChangeRecord(path, key, astr, bstr, monitored) + changes.append(chg) + return changes + + +def compare_siglists(a_blob, b_blob, taskdiff=False): + # FIXME collapse down a recipe's tasks? + alines = a_blob.data_stream.read().decode('utf-8').splitlines() + blines = b_blob.data_stream.read().decode('utf-8').splitlines() + keys = [] + pnmap = {} + def readsigs(lines): + sigs = {} + for line in lines: + linesplit = line.split() + if len(linesplit) > 2: + sigs[linesplit[0]] = linesplit[2] + if not linesplit[0] in keys: + keys.append(linesplit[0]) + pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0] + return sigs + adict = readsigs(alines) + bdict = readsigs(blines) + out = [] + + changecount = 0 + addcount = 0 + removecount = 0 + if taskdiff: + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + + changes = collections.OrderedDict() + + def compare_hashfiles(pn, taskname, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data) + + if not taskname: + (pn, taskname) = pn.rsplit('.', 1) + pn = pnmap.get(pn, pn) + desc = '%s.%s' % (pn, taskname) + + if len(hashfiles) == 0: + out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2)) + elif not hash1 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1)) + elif not hash2 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2)) + else: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True) + for line in out2: + m = hashlib.sha256() + m.update(line.encode('utf-8')) + entry = changes.get(m.hexdigest(), (line, [])) + if desc not in entry[1]: + changes[m.hexdigest()] = (line, entry[1] + [desc]) + + # Define recursion callback + def recursecb(key, hash1, hash2): + compare_hashfiles(key, None, hash1, hash2) + return [] + + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + changecount += 1 + (pn, taskname) = key.rsplit('.', 1) + compare_hashfiles(pn, taskname, siga, sigb) + elif siga is None: + addcount += 1 + elif sigb is None: + removecount += 1 + for key, item in changes.items(): + line, tasks = item + if len(tasks) == 1: + desc = tasks[0] + elif len(tasks) == 2: + desc = '%s and %s' % (tasks[0], tasks[1]) + else: + desc = '%s and %d others' % (tasks[-1], len(tasks)-1) + out.append('%s: %s' % (desc, line)) + else: + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + out.append('%s changed from %s to %s' % (key, siga, sigb)) + changecount += 1 + elif siga is None: + out.append('%s was added' % key) + addcount += 1 + elif sigb is None: + out.append('%s was removed' % key) + removecount += 1 + out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100))) + return '\n'.join(out) + + +def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False): + repo = git.Repo(repopath) + assert repo.bare == False + commit = repo.commit(revision1) + diff = commit.diff(revision2) + + changes = [] + + if sigs or sigsdiff: + for d in diff.iter_change_type('M'): + if d.a_blob.path == 'siglist.txt': + changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff)) + return changes + + for d in diff.iter_change_type('M'): + path = os.path.dirname(d.a_blob.path) + if path.startswith('packages/'): + filename = os.path.basename(d.a_blob.path) + if filename == 'latest': + changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) + elif filename.startswith('latest.'): + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) + changes.append(chg) + elif path.startswith('images/'): + filename = os.path.basename(d.a_blob.path) + if filename in img_monitor_files: + if filename == 'files-in-image.txt': + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() + filechanges = compare_file_lists(alines,blines) + if filechanges: + chg = ChangeRecord(path, filename, None, None, True) + chg.filechanges = filechanges + changes.append(chg) + elif filename == 'installed-package-names.txt': + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() + filechanges = compare_lists(alines,blines) + if filechanges: + chg = ChangeRecord(path, filename, None, None, True) + chg.filechanges = filechanges + changes.append(chg) + else: + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) + changes.append(chg) + elif filename == 'image-info.txt': + changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) + elif '/image-files/' in path: + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) + changes.append(chg) + + # Look for added preinst/postinst/prerm/postrm + # (without reporting newly added recipes) + addedpkgs = [] + addedchanges = [] + for d in diff.iter_change_type('A'): + path = os.path.dirname(d.b_blob.path) + if path.startswith('packages/'): + filename = os.path.basename(d.b_blob.path) + if filename == 'latest': + addedpkgs.append(path) + elif filename.startswith('latest.'): + chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True) + addedchanges.append(chg) + for chg in addedchanges: + found = False + for pkg in addedpkgs: + if chg.path.startswith(pkg): + found = True + break + if not found: + changes.append(chg) + + # Look for cleared preinst/postinst/prerm/postrm + for d in diff.iter_change_type('D'): + path = os.path.dirname(d.a_blob.path) + if path.startswith('packages/'): + filename = os.path.basename(d.a_blob.path) + if filename != 'latest' and filename.startswith('latest.'): + chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) + changes.append(chg) + + # Link related changes + for chg in changes: + if chg.monitored: + for chg2 in changes: + # (Check dirname in the case of fields from recipe info files) + if chg.path == chg2.path or os.path.dirname(chg.path) == chg2.path: + if chg2.fieldname in related_fields.get(chg.fieldname, []): + chg.related.append(chg2) + elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']: + chg.related.append(chg2) + + if report_all: + return changes + else: + return [chg for chg in changes if chg.monitored] diff --git a/meta/lib/oe/cachedpath.py b/meta/lib/oe/cachedpath.py new file mode 100644 index 0000000000..0840cc4c3f --- /dev/null +++ b/meta/lib/oe/cachedpath.py @@ -0,0 +1,233 @@ +# +# Based on standard python library functions but avoid +# repeated stat calls. Its assumed the files will not change from under us +# so we can cache stat calls. +# + +import os +import errno +import stat as statmod + +class CachedPath(object): + def __init__(self): + self.statcache = {} + self.lstatcache = {} + self.normpathcache = {} + return + + def updatecache(self, x): + x = self.normpath(x) + if x in self.statcache: + del self.statcache[x] + if x in self.lstatcache: + del self.lstatcache[x] + + def normpath(self, path): + if path in self.normpathcache: + return self.normpathcache[path] + newpath = os.path.normpath(path) + self.normpathcache[path] = newpath + return newpath + + def _callstat(self, path): + if path in self.statcache: + return self.statcache[path] + try: + st = os.stat(path) + self.statcache[path] = st + return st + except os.error: + self.statcache[path] = False + return False + + # We might as well call lstat and then only + # call stat as well in the symbolic link case + # since this turns out to be much more optimal + # in real world usage of this cache + def callstat(self, path): + path = self.normpath(path) + self.calllstat(path) + return self.statcache[path] + + def calllstat(self, path): + path = self.normpath(path) + if path in self.lstatcache: + return self.lstatcache[path] + #bb.error("LStatpath:" + path) + try: + lst = os.lstat(path) + self.lstatcache[path] = lst + if not statmod.S_ISLNK(lst.st_mode): + self.statcache[path] = lst + else: + self._callstat(path) + return lst + except (os.error, AttributeError): + self.lstatcache[path] = False + self.statcache[path] = False + return False + + # This follows symbolic links, so both islink() and isdir() can be true + # for the same path ono systems that support symlinks + def isfile(self, path): + """Test whether a path is a regular file""" + st = self.callstat(path) + if not st: + return False + return statmod.S_ISREG(st.st_mode) + + # Is a path a directory? + # This follows symbolic links, so both islink() and isdir() + # can be true for the same path on systems that support symlinks + def isdir(self, s): + """Return true if the pathname refers to an existing directory.""" + st = self.callstat(s) + if not st: + return False + return statmod.S_ISDIR(st.st_mode) + + def islink(self, path): + """Test whether a path is a symbolic link""" + st = self.calllstat(path) + if not st: + return False + return statmod.S_ISLNK(st.st_mode) + + # Does a path exist? + # This is false for dangling symbolic links on systems that support them. + def exists(self, path): + """Test whether a path exists. Returns False for broken symbolic links""" + if self.callstat(path): + return True + return False + + def lexists(self, path): + """Test whether a path exists. Returns True for broken symbolic links""" + if self.calllstat(path): + return True + return False + + def stat(self, path): + return self.callstat(path) + + def lstat(self, path): + return self.calllstat(path) + + def walk(self, top, topdown=True, onerror=None, followlinks=False): + # Matches os.walk, not os.path.walk() + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. os.path.walk + # always suppressed the exception then, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. That logic is copied here. + try: + names = os.listdir(top) + except os.error as err: + if onerror is not None: + onerror(err) + return + + dirs, nondirs = [], [] + for name in names: + if self.isdir(os.path.join(top, name)): + dirs.append(name) + else: + nondirs.append(name) + + if topdown: + yield top, dirs, nondirs + for name in dirs: + new_path = os.path.join(top, name) + if followlinks or not self.islink(new_path): + for x in self.walk(new_path, topdown, onerror, followlinks): + yield x + if not topdown: + yield top, dirs, nondirs + + ## realpath() related functions + def __is_path_below(self, file, root): + return (file + os.path.sep).startswith(root) + + def __realpath_rel(self, start, rel_path, root, loop_cnt, assume_dir): + """Calculates real path of symlink 'start' + 'rel_path' below + 'root'; no part of 'start' below 'root' must contain symlinks. """ + have_dir = True + + for d in rel_path.split(os.path.sep): + if not have_dir and not assume_dir: + raise OSError(errno.ENOENT, "no such directory %s" % start) + + if d == os.path.pardir: # '..' + if len(start) >= len(root): + # do not follow '..' before root + start = os.path.dirname(start) + else: + # emit warning? + pass + else: + (start, have_dir) = self.__realpath(os.path.join(start, d), + root, loop_cnt, assume_dir) + + assert(self.__is_path_below(start, root)) + + return start + + def __realpath(self, file, root, loop_cnt, assume_dir): + while self.islink(file) and len(file) >= len(root): + if loop_cnt == 0: + raise OSError(errno.ELOOP, file) + + loop_cnt -= 1 + target = os.path.normpath(os.readlink(file)) + + if not os.path.isabs(target): + tdir = os.path.dirname(file) + assert(self.__is_path_below(tdir, root)) + else: + tdir = root + + file = self.__realpath_rel(tdir, target, root, loop_cnt, assume_dir) + + try: + is_dir = self.isdir(file) + except: + is_dir = False + + return (file, is_dir) + + def realpath(self, file, root, use_physdir = True, loop_cnt = 100, assume_dir = False): + """ Returns the canonical path of 'file' with assuming a + toplevel 'root' directory. When 'use_physdir' is set, all + preceding path components of 'file' will be resolved first; + this flag should be set unless it is guaranteed that there is + no symlink in the path. When 'assume_dir' is not set, missing + path components will raise an ENOENT error""" + + root = os.path.normpath(root) + file = os.path.normpath(file) + + if not root.endswith(os.path.sep): + # letting root end with '/' makes some things easier + root = root + os.path.sep + + if not self.__is_path_below(file, root): + raise OSError(errno.EINVAL, "file '%s' is not below root" % file) + + try: + if use_physdir: + file = self.__realpath_rel(root, file[(len(root) - 1):], root, loop_cnt, assume_dir) + else: + file = self.__realpath(file, root, loop_cnt, assume_dir)[0] + except OSError as e: + if e.errno == errno.ELOOP: + # make ELOOP more readable; without catching it, there will + # be printed a backtrace with 100s of OSError exceptions + # else + raise OSError(errno.ELOOP, + "too much recursions while resolving '%s'; loop in '%s'" % + (file, e.strerror)) + + raise + + return file diff --git a/meta/lib/oe/classextend.py b/meta/lib/oe/classextend.py new file mode 100644 index 0000000000..d2eeaf0e5c --- /dev/null +++ b/meta/lib/oe/classextend.py @@ -0,0 +1,122 @@ +import collections + +class ClassExtender(object): + def __init__(self, extname, d): + self.extname = extname + self.d = d + self.pkgs_mapping = [] + + def extend_name(self, name): + if name.startswith("kernel-") or name == "virtual/kernel": + return name + if name.startswith("rtld"): + return name + if name.endswith("-crosssdk"): + return name + if name.endswith("-" + self.extname): + name = name.replace("-" + self.extname, "") + if name.startswith("virtual/"): + subs = name.split("/", 1)[1] + if not subs.startswith(self.extname): + return "virtual/" + self.extname + "-" + subs + return name + if not name.startswith(self.extname): + return self.extname + "-" + name + return name + + def map_variable(self, varname, setvar = True): + var = self.d.getVar(varname) + if not var: + return "" + var = var.split() + newvar = [] + for v in var: + newvar.append(self.extend_name(v)) + newdata = " ".join(newvar) + if setvar: + self.d.setVar(varname, newdata) + return newdata + + def map_regexp_variable(self, varname, setvar = True): + var = self.d.getVar(varname) + if not var: + return "" + var = var.split() + newvar = [] + for v in var: + if v.startswith("^" + self.extname): + newvar.append(v) + elif v.startswith("^"): + newvar.append("^" + self.extname + "-" + v[1:]) + else: + newvar.append(self.extend_name(v)) + newdata = " ".join(newvar) + if setvar: + self.d.setVar(varname, newdata) + return newdata + + def map_depends(self, dep): + if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('cross-canadian' in dep) or ('-crosssdk-' in dep): + return dep + else: + # Do not extend for that already have multilib prefix + var = self.d.getVar("MULTILIB_VARIANTS") + if var: + var = var.split() + for v in var: + if dep.startswith(v): + return dep + return self.extend_name(dep) + + def map_depends_variable(self, varname, suffix = ""): + # We need to preserve EXTENDPKGV so it can be expanded correctly later + if suffix: + varname = varname + "_" + suffix + orig = self.d.getVar("EXTENDPKGV", False) + self.d.setVar("EXTENDPKGV", "EXTENDPKGV") + deps = self.d.getVar(varname) + if not deps: + self.d.setVar("EXTENDPKGV", orig) + return + deps = bb.utils.explode_dep_versions2(deps) + newdeps = collections.OrderedDict() + for dep in deps: + newdeps[self.map_depends(dep)] = deps[dep] + + self.d.setVar(varname, bb.utils.join_deps(newdeps, False).replace("EXTENDPKGV", "${EXTENDPKGV}")) + self.d.setVar("EXTENDPKGV", orig) + + def map_packagevars(self): + for pkg in (self.d.getVar("PACKAGES").split() + [""]): + self.map_depends_variable("RDEPENDS", pkg) + self.map_depends_variable("RRECOMMENDS", pkg) + self.map_depends_variable("RSUGGESTS", pkg) + self.map_depends_variable("RPROVIDES", pkg) + self.map_depends_variable("RREPLACES", pkg) + self.map_depends_variable("RCONFLICTS", pkg) + self.map_depends_variable("PKG", pkg) + + def rename_packages(self): + for pkg in (self.d.getVar("PACKAGES") or "").split(): + if pkg.startswith(self.extname): + self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg]) + continue + self.pkgs_mapping.append([pkg, self.extend_name(pkg)]) + + self.d.setVar("PACKAGES", " ".join([row[1] for row in self.pkgs_mapping])) + + def rename_package_variables(self, variables): + for pkg_mapping in self.pkgs_mapping: + for subs in variables: + self.d.renameVar("%s_%s" % (subs, pkg_mapping[0]), "%s_%s" % (subs, pkg_mapping[1])) + +class NativesdkClassExtender(ClassExtender): + def map_depends(self, dep): + if dep.startswith(self.extname): + return dep + if dep.endswith(("-gcc-initial", "-gcc", "-g++")): + return dep + "-crosssdk" + elif dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep): + return dep + else: + return self.extend_name(dep) diff --git a/meta/lib/oe/classutils.py b/meta/lib/oe/classutils.py index 58188fdd6e..45cd5249be 100644 --- a/meta/lib/oe/classutils.py +++ b/meta/lib/oe/classutils.py @@ -1,4 +1,11 @@ -class ClassRegistry(type): + +class ClassRegistryMeta(type): + """Give each ClassRegistry their own registry""" + def __init__(cls, name, bases, attrs): + cls.registry = {} + type.__init__(cls, name, bases, attrs) + +class ClassRegistry(type, metaclass=ClassRegistryMeta): """Maintain a registry of classes, indexed by name. Note that this implementation requires that the names be unique, as it uses @@ -12,12 +19,6 @@ Subclasses of ClassRegistry may define an 'implemented' property to exert control over whether the class will be added to the registry (e.g. to keep abstract base classes out of the registry).""" priority = 0 - class __metaclass__(type): - """Give each ClassRegistry their own registry""" - def __init__(cls, name, bases, attrs): - cls.registry = {} - type.__init__(cls, name, bases, attrs) - def __init__(cls, name, bases, attrs): super(ClassRegistry, cls).__init__(name, bases, attrs) try: @@ -34,8 +35,8 @@ abstract base classes out of the registry).""" @classmethod def prioritized(tcls): - return sorted(tcls.registry.values(), - key=lambda v: v.priority, reverse=True) + return sorted(list(tcls.registry.values()), + key=lambda v: (v.priority, v.name), reverse=True) def unregister(cls): for key in cls.registry.keys(): diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py new file mode 100644 index 0000000000..a372904183 --- /dev/null +++ b/meta/lib/oe/copy_buildsystem.py @@ -0,0 +1,244 @@ +# This class should provide easy access to the different aspects of the +# buildsystem such as layers, bitbake location, etc. +import stat +import shutil + +def _smart_copy(src, dest): + import subprocess + # smart_copy will choose the correct function depending on whether the + # source is a file or a directory. + mode = os.stat(src).st_mode + if stat.S_ISDIR(mode): + bb.utils.mkdirhier(dest) + cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + else: + shutil.copyfile(src, dest) + shutil.copymode(src, dest) + +class BuildSystem(object): + def __init__(self, context, d): + self.d = d + self.context = context + self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] + self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() + + def copy_bitbake_and_layers(self, destdir, workspace_name=None): + # Copy in all metadata layers + bitbake (as repositories) + layers_copied = [] + bb.utils.mkdirhier(destdir) + layers = list(self.layerdirs) + + corebase = os.path.abspath(self.d.getVar('COREBASE')) + layers.append(corebase) + + # Exclude layers + for layer_exclude in self.layers_exclude: + if layer_exclude in layers: + layers.remove(layer_exclude) + + workspace_newname = workspace_name + if workspace_newname: + layernames = [os.path.basename(layer) for layer in layers] + extranum = 0 + while workspace_newname in layernames: + extranum += 1 + workspace_newname = '%s-%d' % (workspace_name, extranum) + + corebase_files = self.d.getVar('COREBASE_FILES').split() + corebase_files = [corebase + '/' +x for x in corebase_files] + # Make sure bitbake goes in + bitbake_dir = bb.__file__.rsplit('/', 3)[0] + corebase_files.append(bitbake_dir) + + for layer in layers: + layerconf = os.path.join(layer, 'conf', 'layer.conf') + layernewname = os.path.basename(layer) + workspace = False + if os.path.exists(layerconf): + with open(layerconf, 'r') as f: + if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): + if workspace_newname: + layernewname = workspace_newname + workspace = True + else: + bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) + continue + + # If the layer was already under corebase, leave it there + # since layers such as meta have issues when moved. + layerdestpath = destdir + if corebase == os.path.dirname(layer): + layerdestpath += '/' + os.path.basename(corebase) + layerdestpath += '/' + layernewname + + layer_relative = os.path.relpath(layerdestpath, + destdir) + layers_copied.append(layer_relative) + + # Treat corebase as special since it typically will contain + # build directories or other custom items. + if corebase == layer: + bb.utils.mkdirhier(layerdestpath) + for f in corebase_files: + f_basename = os.path.basename(f) + destname = os.path.join(layerdestpath, f_basename) + _smart_copy(f, destname) + else: + if os.path.exists(layerdestpath): + bb.note("Skipping layer %s, already handled" % layer) + else: + _smart_copy(layer, layerdestpath) + + if workspace: + # Make some adjustments original workspace layer + # Drop sources (recipe tasks will be locked, so we don't need them) + srcdir = os.path.join(layerdestpath, 'sources') + if os.path.isdir(srcdir): + shutil.rmtree(srcdir) + # Drop all bbappends except the one for the image the SDK is being built for + # (because of externalsrc, the workspace bbappends will interfere with the + # locked signatures if present, and we don't need them anyway) + image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend' + appenddir = os.path.join(layerdestpath, 'appends') + if os.path.isdir(appenddir): + for fn in os.listdir(appenddir): + if fn == image_bbappend: + continue + else: + os.remove(os.path.join(appenddir, fn)) + # Drop README + readme = os.path.join(layerdestpath, 'README') + if os.path.exists(readme): + os.remove(readme) + # Filter out comments in layer.conf and change layer name + layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') + with open(layerconf, 'r') as f: + origlines = f.readlines() + with open(layerconf, 'w') as f: + for line in origlines: + if line.startswith('#'): + continue + line = line.replace('workspacelayer', workspace_newname) + f.write(line) + + return layers_copied + +def generate_locked_sigs(sigfile, d): + bb.utils.mkdirhier(os.path.dirname(sigfile)) + depd = d.getVar('BB_TASKDEPDATA', False) + tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()] + bb.parse.siggen.dump_lockedsigs(sigfile, tasks) + +def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output): + with open(lockedsigs, 'r') as infile: + bb.utils.mkdirhier(os.path.dirname(pruned_output)) + with open(pruned_output, 'w') as f: + invalue = False + for line in infile: + if invalue: + if line.endswith('\\\n'): + splitval = line.strip().split(':') + if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets: + f.write(line) + else: + f.write(line) + invalue = False + elif line.startswith('SIGGEN_LOCKEDSIGS'): + invalue = True + f.write(line) + +def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): + merged = {} + arch_order = [] + with open(lockedsigs_main, 'r') as f: + invalue = None + for line in f: + if invalue: + if line.endswith('\\\n'): + merged[invalue].append(line) + else: + invalue = None + elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): + invalue = line[18:].split('=', 1)[0].rstrip() + merged[invalue] = [] + arch_order.append(invalue) + + with open(lockedsigs_extra, 'r') as f: + invalue = None + tocopy = {} + for line in f: + if invalue: + if line.endswith('\\\n'): + if not line in merged[invalue]: + target, task = line.strip().split(':')[:2] + if not copy_tasks or task in copy_tasks: + tocopy[invalue].append(line) + merged[invalue].append(line) + else: + invalue = None + elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): + invalue = line[18:].split('=', 1)[0].rstrip() + if not invalue in merged: + merged[invalue] = [] + arch_order.append(invalue) + tocopy[invalue] = [] + + def write_sigs_file(fn, types, sigs): + fulltypes = [] + bb.utils.mkdirhier(os.path.dirname(fn)) + with open(fn, 'w') as f: + for typename in types: + lines = sigs[typename] + if lines: + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename) + for line in lines: + f.write(line) + f.write(' "\n') + fulltypes.append(typename) + f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) + + if copy_output: + write_sigs_file(copy_output, list(tocopy.keys()), tocopy) + if merged_output: + write_sigs_file(merged_output, arch_order, merged) + +def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): + import shutil + bb.note('Generating sstate-cache...') + + nativelsbstring = d.getVar('NATIVELSBSTRING') + bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) + if fixedlsbstring and nativelsbstring != fixedlsbstring: + nativedir = output_sstate_cache + '/' + nativelsbstring + if os.path.isdir(nativedir): + destdir = os.path.join(output_sstate_cache, fixedlsbstring) + for root, _, files in os.walk(nativedir): + for fn in files: + src = os.path.join(root, fn) + dest = os.path.join(destdir, os.path.relpath(src, nativedir)) + if os.path.exists(dest): + # Already exists, and it'll be the same file, so just delete it + os.unlink(src) + else: + bb.utils.mkdirhier(os.path.dirname(dest)) + shutil.move(src, dest) + +def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): + import subprocess + + bb.note('Generating sstate task list...') + + if not cwd: + cwd = os.getcwd() + if logfile: + logparam = '-l %s' % logfile + else: + logparam = '' + cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + env = dict(d.getVar('BB_ORIGENV', False)) + env.pop('BUILDDIR', '') + pathitems = env['PATH'].split(':') + env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) + bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py index af900be6e4..b8901e63f5 100644 --- a/meta/lib/oe/data.py +++ b/meta/lib/oe/data.py @@ -1,5 +1,5 @@ +import json import oe.maketype -import bb.msg def typed_value(key, d): """Construct a value for the specified metadata variable, using its flags @@ -7,12 +7,41 @@ def typed_value(key, d): var_type = d.getVarFlag(key, 'type') flags = d.getVarFlags(key) if flags is not None: - flags = dict((flag, bb.data.expand(value, d)) - for flag, value in flags.iteritems()) + flags = dict((flag, d.expand(value)) + for flag, value in list(flags.items())) else: flags = {} try: - return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags) - except (TypeError, ValueError), exc: + return oe.maketype.create(d.getVar(key) or '', var_type, **flags) + except (TypeError, ValueError) as exc: bb.msg.fatal("Data", "%s: %s" % (key, str(exc))) + +def export2json(d, json_file, expand=True, searchString="",replaceString=""): + data2export = {} + keys2export = [] + + for key in d.keys(): + if key.startswith("_"): + continue + elif key.startswith("BB"): + continue + elif key.startswith("B_pn"): + continue + elif key.startswith("do_"): + continue + elif d.getVarFlag(key, "func"): + continue + + keys2export.append(key) + + for key in keys2export: + try: + data2export[key] = d.getVar(key, expand).replace(searchString,replaceString) + except bb.data_smart.ExpansionError: + data2export[key] = '' + except AttributeError: + pass + + with open(json_file, "w") as f: + json.dump(data2export, f, skipkeys=True, indent=4, sort_keys=True) diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py index ad391e3b01..37f04ed359 100644 --- a/meta/lib/oe/distro_check.py +++ b/meta/lib/oe/distro_check.py @@ -1,43 +1,28 @@ -def get_links_from_url(url): +def create_socket(url, d): + import urllib + from bb.utils import export_proxies + + export_proxies(d) + return urllib.request.urlopen(url) + +def get_links_from_url(url, d): "Return all the href links found on the web location" - import urllib, sgmllib - - class LinksParser(sgmllib.SGMLParser): - def parse(self, s): - "Parse the given string 's'." - self.feed(s) - self.close() - - def __init__(self, verbose=0): - "Initialise an object passing 'verbose' to the superclass." - sgmllib.SGMLParser.__init__(self, verbose) - self.hyperlinks = [] - - def start_a(self, attributes): - "Process a hyperlink and its 'attributes'." - for name, value in attributes: - if name == "href": - self.hyperlinks.append(value.strip('/')) - - def get_hyperlinks(self): - "Return the list of hyperlinks." - return self.hyperlinks - - sock = urllib.urlopen(url) - webpage = sock.read() - sock.close() - - linksparser = LinksParser() - linksparser.parse(webpage) - return linksparser.get_hyperlinks() - -def find_latest_numeric_release(url): + from bs4 import BeautifulSoup, SoupStrainer + + soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) + hyperlinks = [] + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks + +def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" max=0 maxstr="" - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): try: + # TODO use LooseVersion release = float(link) except: release = 0 @@ -48,195 +33,158 @@ def find_latest_numeric_release(url): def is_src_rpm(name): "Check if the link is pointing to a src.rpm file" - if name[-8:] == ".src.rpm": - return True - else: - return False + return name.endswith(".src.rpm") def package_name_from_srpm(srpm): "Strip out the package name from the src.rpm filename" - strings = srpm.split('-') - package_name = strings[0] - for i in range(1, len (strings) - 1): - str = strings[i] - if not str[0].isdigit(): - package_name += '-' + str - return package_name - -def clean_package_list(package_list): - "Removes multiple entries of packages and sorts the list" - set = {} - map(set.__setitem__, package_list, []) - return set.keys() - - -def get_latest_released_meego_source_package_list(): - "Returns list of all the name os packages in the latest meego distro" - - if not os.path.isfile("/tmp/Meego-1.1"): - os.mknod("/tmp/Meego-1.1") - f = open("/tmp/Meego-1.1", "r") - package_names = [] - for line in f: - package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end - - package_list=clean_package_list(package_names) - return "1.0", package_list - -def get_source_package_list_from_url(url, section): + + # ca-certificates-2016.2.7-1.0.fc24.src.rpm + # ^name ^ver ^release^removed + (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) + return name + +def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) - links = get_links_from_url(url) + links = get_links_from_url(url, d) srpms = filter(is_src_rpm, links) names_list = map(package_name_from_srpm, srpms) - new_pkgs = [] + new_pkgs = set() for pkgs in names_list: - new_pkgs.append(pkgs + ":" + section) - + new_pkgs.add(pkgs + ":" + section) return new_pkgs -def get_latest_released_fedora_source_package_list(): - "Returns list of all the name os packages in the latest fedora distro" - latest = find_latest_numeric_release("http://download.fedora.redhat.com/pub/fedora/linux/releases/") - - package_names = get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main") - -# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") - package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates") +def get_source_package_list_from_url_by_letter(url, section, d): + import string + from urllib.error import HTTPError + packages = set() + for letter in (string.ascii_lowercase + string.digits): + # Not all subfolders may exist, so silently handle 404 + try: + packages |= get_source_package_list_from_url(url + "/" + letter, section, d) + except HTTPError as e: + if e.code != 404: raise + return packages - package_list=clean_package_list(package_names) - - return latest, package_list +def get_latest_released_fedora_source_package_list(d): + "Returns list of all the name os packages in the latest fedora distro" + latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) + package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) + package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) + return latest, package_names -def get_latest_released_opensuse_source_package_list(): +def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main") - package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates") + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d) + return latest, package_names - package_list=clean_package_list(package_names) - return latest, package_list - -def get_latest_released_mandriva_source_package_list(): +def get_latest_released_mandriva_source_package_list(d): "Returns list of all the name os packages in the latest mandriva distro" - latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") - package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main") -# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") - package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates") + latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) + package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) + return latest, package_names - package_list=clean_package_list(package_names) - return latest, package_list +def get_latest_released_clear_source_package_list(d): + latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) + package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) + return latest, package_names -def find_latest_debian_release(url): +def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" - releases = [] - for link in get_links_from_url(url): - if link[:6] == "Debian": - if ';' not in link: - releases.append(link) + releases = [link.replace("Debian", "") + for link in get_links_from_url(url, d) + if link.startswith("Debian")] releases.sort() try: - return releases.pop()[6:] + return releases[-1] except: return "_NotFound_" -def get_debian_style_source_package_list(url, section): +def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - import urllib - sock = urllib.urlopen(url) - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - sock.close() - tmpfile.close() import gzip - bb.note("Reading %s: %s" % (url, section)) - - f = gzip.open(tmpfilename) - package_names = [] - for line in f: - if line[:9] == "Package: ": - package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end - os.unlink(tmpfilename) + package_names = set() + for line in gzip.open(create_socket(url, d), mode="rt"): + if line.startswith("Package:"): + pkg = line.split(":", 1)[1].strip() + package_names.add(pkg + ":" + section) return package_names -def get_latest_released_debian_source_package_list(): - "Returns list of all the name os packages in the latest debian distro" - latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/") - url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" - package_names = get_debian_style_source_package_list(url, "main") -# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" -# package_names += get_debian_style_source_package_list(url, "contrib") - url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" - package_names += get_debian_style_source_package_list(url, "updates") - package_list=clean_package_list(package_names) - return latest, package_list - -def find_latest_ubuntu_release(url): - "Find the latest listed ubuntu release on the given url" +def get_latest_released_debian_source_package_list(d): + "Returns list of all the name of packages in the latest debian distro" + latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) + url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" + package_names = get_debian_style_source_package_list(url, "main", d) + url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def find_latest_ubuntu_release(url, d): + """ + Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. + + To avoid matching development releases look for distributions that have + updates, so the resulting distro could be any supported release. + """ url += "?C=M;O=D" # Descending Sort by Last Modified - for link in get_links_from_url(url): - if link[-8:] == "-updates": - return link[:-8] + for link in get_links_from_url(url, d): + if "-updates" in link: + distro = link.replace("-updates", "") + return distro return "_NotFound_" -def get_latest_released_ubuntu_source_package_list(): +def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" - latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") + latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest - package_names = get_debian_style_source_package_list(url, "main") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "multiverse") -# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest -# package_names += get_debian_style_source_package_list(url, "universe") + package_names = get_debian_style_source_package_list(url, "main", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest - package_names += get_debian_style_source_package_list(url, "updates") - package_list=clean_package_list(package_names) - return latest, package_list + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def create_distro_packages_list(distro_check_dir, d): + import shutil -def create_distro_packages_list(distro_check_dir): pkglst_dir = os.path.join(distro_check_dir, "package_lists") - if not os.path.isdir (pkglst_dir): - os.makedirs(pkglst_dir) - # first clear old stuff - for file in os.listdir(pkglst_dir): - os.unlink(os.path.join(pkglst_dir, file)) - - per_distro_functions = [ - ["Debian", get_latest_released_debian_source_package_list], - ["Ubuntu", get_latest_released_ubuntu_source_package_list], - ["Fedora", get_latest_released_fedora_source_package_list], - ["OpenSuSE", get_latest_released_opensuse_source_package_list], - ["Mandriva", get_latest_released_mandriva_source_package_list], - ["Meego", get_latest_released_meego_source_package_list] - ] - - from datetime import datetime - begin = datetime.now() - for distro in per_distro_functions: - name = distro[0] - release, package_list = distro[1]() + bb.utils.remove(pkglst_dir, True) + bb.utils.mkdirhier(pkglst_dir) + + per_distro_functions = ( + ("Debian", get_latest_released_debian_source_package_list), + ("Ubuntu", get_latest_released_ubuntu_source_package_list), + ("Fedora", get_latest_released_fedora_source_package_list), + ("OpenSuSE", get_latest_released_opensuse_source_package_list), + ("Mandriva", get_latest_released_mandriva_source_package_list), + ("Clear", get_latest_released_clear_source_package_list), + ) + + for name, fetcher_func in per_distro_functions: + try: + release, package_list = fetcher_func(d) + except Exception as e: + bb.warn("Cannot fetch packages for %s: %s" % (name, e)) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) + if len(package_list) == 0: + bb.error("Didn't fetch any packages for %s %s" % (name, release)) + package_list_file = os.path.join(pkglst_dir, name + "-" + release) - f = open(package_list_file, "w+b") - for pkg in package_list: - f.write(pkg + "\n") - f.close() - end = datetime.now() - delta = end - begin - bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) + with open(package_list_file, 'w') as f: + for pkg in sorted(package_list): + f.write(pkg + "\n") -def update_distro_data(distro_check_dir, datetime): +def update_distro_data(distro_check_dir, datetime, d): """ - If distro packages list data is old then rebuild it. - The operations has to be protected by a lock so that - only one thread performes it at a time. + If distro packages list data is old then rebuild it. + The operations has to be protected by a lock so that + only one thread performes it at a time. """ if not os.path.isdir (distro_check_dir): try: @@ -251,77 +199,71 @@ def update_distro_data(distro_check_dir, datetime): import fcntl try: if not os.path.exists(datetime_file): - open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail - f = open(datetime_file, "r+b") + f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") - create_distro_packages_list(distro_check_dir) + create_distro_packages_list(distro_check_dir, d) f.seek(0) f.write(datetime) - except OSError: - raise Exception('Unable to read/write this file: %s' % (datetime_file)) + except OSError as e: + raise Exception('Unable to open timestamp: %s' % e) finally: fcntl.lockf(f, fcntl.LOCK_UN) f.close() - + def compare_in_distro_packages_list(distro_check_dir, d): if not os.path.isdir(distro_check_dir): raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") - + localdata = bb.data.createCopy(d) pkglst_dir = os.path.join(distro_check_dir, "package_lists") matching_distros = [] - pn = d.getVar('PN', True) - recipe_name = d.getVar('PN', True) + pn = recipe_name = d.getVar('PN') bb.note("Checking: %s" % pn) - trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"}) - if pn.find("-native") != -1: pnstripped = pn.split("-native") - bb.data.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True), localdata) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] + if pn.startswith("nativesdk-"): + pnstripped = pn.split("nativesdk-") + localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) + recipe_name = pnstripped[1] + if pn.find("-cross") != -1: pnstripped = pn.split("-cross") - bb.data.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True), localdata) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] if pn.find("-initial") != -1: pnstripped = pn.split("-initial") - bb.data.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True), localdata) - bb.data.update_data(localdata) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) recipe_name = pnstripped[0] bb.note("Recipe: %s" % recipe_name) - tmp = localdata.getVar('DISTRO_PN_ALIAS', True) distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) - - if tmp: - list = tmp.split(' ') - for str in list: - if str and str.find("=") == -1 and distro_exceptions[str]: - matching_distros.append(str) + tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" + for str in tmp.split(): + if str and str.find("=") == -1 and distro_exceptions[str]: + matching_distros.append(str) distro_pn_aliases = {} - if tmp: - list = tmp.split(' ') - for str in list: - if str.find("=") != -1: - (dist, pn_alias) = str.split('=') - distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() - + for str in tmp.split(): + if "=" in str: + (dist, pn_alias) = str.split('=') + distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() + for file in os.listdir(pkglst_dir): (distro, distro_release) = file.split("-") - f = open(os.path.join(pkglst_dir, file), "rb") + f = open(os.path.join(pkglst_dir, file), "r") for line in f: (pkg, section) = line.split(":") if distro.lower() in distro_pn_aliases: @@ -334,37 +276,34 @@ def compare_in_distro_packages_list(distro_check_dir, d): break f.close() - - if tmp != None: - list = tmp.split(' ') - for item in list: - matching_distros.append(item) + for item in tmp.split(): + matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros def create_log_file(d, logname): - logpath = d.getVar('LOG_DIR', True) + logpath = d.getVar('LOG_DIR') bb.utils.mkdirhier(logpath) logfn, logsuffix = os.path.splitext(logname) - logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix)) + logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) if not os.path.exists(logfile): slogfile = os.path.join(logpath, logname) if os.path.exists(slogfile): os.remove(slogfile) - os.system("touch %s" % logfile) + open(logfile, 'w+').close() os.symlink(logfile, slogfile) d.setVar('LOG_FILE', logfile) return logfile def save_distro_check_result(result, datetime, result_file, d): - pn = d.getVar('PN', True) - logdir = d.getVar('LOG_DIR', True) + pn = d.getVar('PN') + logdir = d.getVar('LOG_DIR') if not logdir: bb.error("LOG_DIR variable is not defined, can't write the distro_check results") return - if not os.path.isdir(logdir): - os.makedirs(logdir) + bb.utils.mkdirhier(logdir) + line = pn for i in result: line = line + "," + i diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py new file mode 100644 index 0000000000..7ce767ee0a --- /dev/null +++ b/meta/lib/oe/gpg_sign.py @@ -0,0 +1,119 @@ +"""Helper module for GPG signing""" +import os + +import bb +import oe.utils + +class LocalSigner(object): + """Class for handling local (on the build host) signing""" + def __init__(self, d): + self.gpg_bin = d.getVar('GPG_BIN') or \ + bb.utils.which(os.getenv('PATH'), 'gpg') + self.gpg_path = d.getVar('GPG_PATH') + self.gpg_version = self.get_gpg_version() + self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign") + + def export_pubkey(self, output_file, keyid, armor=True): + """Export GPG public key to a file""" + cmd = '%s --batch --yes --export -o %s ' % \ + (self.gpg_bin, output_file) + if self.gpg_path: + cmd += "--homedir %s " % self.gpg_path + if armor: + cmd += "--armor " + cmd += keyid + status, output = oe.utils.getstatusoutput(cmd) + if status: + raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % + (keyid, output)) + + def sign_rpms(self, files, keyid, passphrase): + """Sign RPM files""" + + cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid + gpg_args = '--batch --passphrase=%s' % passphrase + if self.gpg_version > (2,1,): + gpg_args += ' --pinentry-mode=loopback' + cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args + if self.gpg_bin: + cmd += "--define '%%__gpg %s' " % self.gpg_bin + if self.gpg_path: + cmd += "--define '_gpg_path %s' " % self.gpg_path + + # Sign in chunks of 100 packages + for i in range(0, len(files), 100): + status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100])) + if status: + raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) + + def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): + """Create a detached signature of a file""" + import subprocess + + if passphrase_file and passphrase: + raise Exception("You should use either passphrase_file of passphrase, not both") + + cmd = [self.gpg_bin, '--detach-sign', '--batch', '--no-tty', '--yes', + '--passphrase-fd', '0', '-u', keyid] + + if self.gpg_path: + cmd += ['--homedir', self.gpg_path] + if armor: + cmd += ['--armor'] + + #gpg > 2.1 supports password pipes only through the loopback interface + #gpg < 2.1 errors out if given unknown parameters + if self.gpg_version > (2,1,): + cmd += ['--pinentry-mode', 'loopback'] + + cmd += [input_file] + + try: + if passphrase_file: + with open(passphrase_file) as fobj: + passphrase = fobj.readline(); + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) + + if job.returncode: + raise bb.build.FuncFailed("GPG exited with code %d: %s" % + (job.returncode, stderr.decode("utf-8"))) + + except IOError as e: + bb.error("IO error (%s): %s" % (e.errno, e.strerror)) + raise Exception("Failed to sign '%s'" % input_file) + + except OSError as e: + bb.error("OS error (%s): %s" % (e.errno, e.strerror)) + raise Exception("Failed to sign '%s" % input_file) + + + def get_gpg_version(self): + """Return the gpg version as a tuple of ints""" + import subprocess + try: + ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + return tuple([int(i) for i in ver_str.split('.')]) + except subprocess.CalledProcessError as e: + raise bb.build.FuncFailed("Could not get gpg version: %s" % e) + + + def verify(self, sig_file): + """Verify signature""" + cmd = self.gpg_bin + " --verify " + if self.gpg_path: + cmd += "--homedir %s " % self.gpg_path + cmd += sig_file + status, _ = oe.utils.getstatusoutput(cmd) + ret = False if status else True + return ret + + +def get_signer(d, backend): + """Get signer object for the specified backend""" + # Use local signing by default + if backend == 'local': + return LocalSigner(d) + else: + bb.fatal("Unsupported signing backend '%s'" % backend) diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py new file mode 100644 index 0000000000..8d2fd1709c --- /dev/null +++ b/meta/lib/oe/license.py @@ -0,0 +1,235 @@ +# vi:sts=4:sw=4:et +"""Code for parsing OpenEmbedded license strings""" + +import ast +import re +from fnmatch import fnmatchcase as fnmatch + +def license_ok(license, dont_want_licenses): + """ Return False if License exist in dont_want_licenses else True """ + for dwl in dont_want_licenses: + # If you want to exclude license named generically 'X', we + # surely want to exclude 'X+' as well. In consequence, we + # will exclude a trailing '+' character from LICENSE in + # case INCOMPATIBLE_LICENSE is not a 'X+' license. + lic = license + if not re.search('\+$', dwl): + lic = re.sub('\+', '', license) + if fnmatch(lic, dwl): + return False + return True + +class LicenseError(Exception): + pass + +class LicenseSyntaxError(LicenseError): + def __init__(self, licensestr, exc): + self.licensestr = licensestr + self.exc = exc + LicenseError.__init__(self) + + def __str__(self): + return "error in '%s': %s" % (self.licensestr, self.exc) + +class InvalidLicense(LicenseError): + def __init__(self, license): + self.license = license + LicenseError.__init__(self) + + def __str__(self): + return "invalid characters in license '%s'" % self.license + +license_operator_chars = '&|() ' +license_operator = re.compile('([' + license_operator_chars + '])') +license_pattern = re.compile('[a-zA-Z0-9.+_\-]+$') + +class LicenseVisitor(ast.NodeVisitor): + """Get elements based on OpenEmbedded license strings""" + def get_elements(self, licensestr): + new_elements = [] + elements = list([x for x in license_operator.split(licensestr) if x.strip()]) + for pos, element in enumerate(elements): + if license_pattern.match(element): + if pos > 0 and license_pattern.match(elements[pos-1]): + new_elements.append('&') + element = '"' + element + '"' + elif not license_operator.match(element): + raise InvalidLicense(element) + new_elements.append(element) + + return new_elements + + """Syntax tree visitor which can accept elements previously generated with + OpenEmbedded license string""" + def visit_elements(self, elements): + self.visit(ast.parse(' '.join(elements))) + + """Syntax tree visitor which can accept OpenEmbedded license strings""" + def visit_string(self, licensestr): + self.visit_elements(self.get_elements(licensestr)) + +class FlattenVisitor(LicenseVisitor): + """Flatten a license tree (parsed from a string) by selecting one of each + set of OR options, in the way the user specifies""" + def __init__(self, choose_licenses): + self.choose_licenses = choose_licenses + self.licenses = [] + LicenseVisitor.__init__(self) + + def visit_Str(self, node): + self.licenses.append(node.s) + + def visit_BinOp(self, node): + if isinstance(node.op, ast.BitOr): + left = FlattenVisitor(self.choose_licenses) + left.visit(node.left) + + right = FlattenVisitor(self.choose_licenses) + right.visit(node.right) + + selected = self.choose_licenses(left.licenses, right.licenses) + self.licenses.extend(selected) + else: + self.generic_visit(node) + +def flattened_licenses(licensestr, choose_licenses): + """Given a license string and choose_licenses function, return a flat list of licenses""" + flatten = FlattenVisitor(choose_licenses) + try: + flatten.visit_string(licensestr) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + return flatten.licenses + +def is_included(licensestr, whitelist=None, blacklist=None): + """Given a license string and whitelist and blacklist, determine if the + license string matches the whitelist and does not match the blacklist. + + Returns a tuple holding the boolean state and a list of the applicable + licenses which were excluded (or None, if the state is True) + """ + + def include_license(license): + return any(fnmatch(license, pattern) for pattern in whitelist) + + def exclude_license(license): + return any(fnmatch(license, pattern) for pattern in blacklist) + + def choose_licenses(alpha, beta): + """Select the option in an OR which is the 'best' (has the most + included licenses).""" + alpha_weight = len(list(filter(include_license, alpha))) + beta_weight = len(list(filter(include_license, beta))) + if alpha_weight > beta_weight: + return alpha + else: + return beta + + if not whitelist: + whitelist = ['*'] + + if not blacklist: + blacklist = [] + + licenses = flattened_licenses(licensestr, choose_licenses) + excluded = [lic for lic in licenses if exclude_license(lic)] + included = [lic for lic in licenses if include_license(lic)] + if excluded: + return False, excluded + else: + return True, included + +class ManifestVisitor(LicenseVisitor): + """Walk license tree (parsed from a string) removing the incompatible + licenses specified""" + def __init__(self, dont_want_licenses, canonical_license, d): + self._dont_want_licenses = dont_want_licenses + self._canonical_license = canonical_license + self._d = d + self._operators = [] + + self.licenses = [] + self.licensestr = '' + + LicenseVisitor.__init__(self) + + def visit(self, node): + if isinstance(node, ast.Str): + lic = node.s + + if license_ok(self._canonical_license(self._d, lic), + self._dont_want_licenses) == True: + if self._operators: + ops = [] + for op in self._operators: + if op == '[': + ops.append(op) + elif op == ']': + ops.append(op) + else: + if not ops: + ops.append(op) + elif ops[-1] in ['[', ']']: + ops.append(op) + else: + ops[-1] = op + + for op in ops: + if op == '[' or op == ']': + self.licensestr += op + elif self.licenses: + self.licensestr += ' ' + op + ' ' + + self._operators = [] + + self.licensestr += lic + self.licenses.append(lic) + elif isinstance(node, ast.BitAnd): + self._operators.append("&") + elif isinstance(node, ast.BitOr): + self._operators.append("|") + elif isinstance(node, ast.List): + self._operators.append("[") + elif isinstance(node, ast.Load): + self.licensestr += "]" + + self.generic_visit(node) + +def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): + """Given a license string and dont_want_licenses list, + return license string filtered and a list of licenses""" + manifest = ManifestVisitor(dont_want_licenses, canonical_license, d) + + try: + elements = manifest.get_elements(licensestr) + + # Replace '()' to '[]' for handle in ast as List and Load types. + elements = ['[' if e == '(' else e for e in elements] + elements = [']' if e == ')' else e for e in elements] + + manifest.visit_elements(elements) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + + # Replace '[]' to '()' for output correct license. + manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') + + return (manifest.licensestr, manifest.licenses) + +class ListVisitor(LicenseVisitor): + """Record all different licenses found in the license string""" + def __init__(self): + self.licenses = set() + + def visit_Str(self, node): + self.licenses.add(node.s) + +def list_licenses(licensestr): + """Simply get a list of all licenses mentioned in a license string. + Binary operators are not applied or taken into account in any way""" + visitor = ListVisitor() + try: + visitor.visit_string(licensestr) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + return visitor.licenses diff --git a/meta/lib/oe/lsb.py b/meta/lib/oe/lsb.py new file mode 100644 index 0000000000..3a945e0fce --- /dev/null +++ b/meta/lib/oe/lsb.py @@ -0,0 +1,110 @@ +def release_dict_osr(): + """ Populate a dict with pertinent values from /etc/os-release """ + if not os.path.exists('/etc/os-release'): + return None + + data = {} + with open('/etc/os-release') as f: + for line in f: + try: + key, val = line.rstrip().split('=', 1) + except ValueError: + continue + if key == 'ID': + data['DISTRIB_ID'] = val.strip('"') + if key == 'VERSION_ID': + data['DISTRIB_RELEASE'] = val.strip('"') + + return data + +def release_dict_lsb(): + """ Return the output of lsb_release -ir as a dictionary """ + from subprocess import PIPE + + try: + output, err = bb.process.run(['lsb_release', '-ir'], stderr=PIPE) + except bb.process.CmdError as exc: + return {} + + lsb_map = { 'Distributor ID': 'DISTRIB_ID', + 'Release': 'DISTRIB_RELEASE'} + lsb_keys = lsb_map.keys() + + data = {} + for line in output.splitlines(): + if line.startswith("-e"): + line = line[3:] + try: + key, value = line.split(":\t", 1) + except ValueError: + continue + if key in lsb_keys: + data[lsb_map[key]] = value + + if len(data.keys()) != 2: + return None + + return data + +def release_dict_file(): + """ Try to gather release information manually when other methods fail """ + data = {} + try: + if os.path.exists('/etc/lsb-release'): + data = {} + with open('/etc/lsb-release') as f: + for line in f: + key, value = line.split("=", 1) + data[key] = value.strip() + elif os.path.exists('/etc/redhat-release'): + data = {} + with open('/etc/redhat-release') as f: + distro = f.readline().strip() + import re + match = re.match(r'(.*) release (.*) \((.*)\)', distro) + if match: + data['DISTRIB_ID'] = match.group(1) + data['DISTRIB_RELEASE'] = match.group(2) + elif os.path.exists('/etc/SuSE-release'): + data = {} + data['DISTRIB_ID'] = 'SUSE LINUX' + with open('/etc/SuSE-release') as f: + for line in f: + if line.startswith('VERSION = '): + data['DISTRIB_RELEASE'] = line[10:].rstrip() + break + + except IOError: + return {} + return data + +def distro_identifier(adjust_hook=None): + """Return a distro identifier string based upon lsb_release -ri, + with optional adjustment via a hook""" + + import re + + # Try /etc/os-release first, then the output of `lsb_release -ir` and + # finally fall back on parsing various release files in order to determine + # host distro name and version. + distro_data = release_dict_osr() + if not distro_data: + distro_data = release_dict_lsb() + if not distro_data: + distro_data = release_dict_file() + + distro_id = distro_data.get('DISTRIB_ID', '') + release = distro_data.get('DISTRIB_RELEASE', '') + + if adjust_hook: + distro_id, release = adjust_hook(distro_id, release) + if not distro_id: + return "Unknown" + # Filter out any non-alphanumerics + distro_id = re.sub(r'\W', '', distro_id) + + if release: + id_str = '{0}-{1}'.format(distro_id.lower(), release) + else: + id_str = distro_id + return id_str.replace(' ','-').replace('/','-') diff --git a/meta/lib/oe/maketype.py b/meta/lib/oe/maketype.py index 0e9dbc67fb..f88981dd90 100644 --- a/meta/lib/oe/maketype.py +++ b/meta/lib/oe/maketype.py @@ -5,9 +5,9 @@ variable. Other flags may be utilized in the construction of the types. See the arguments of the type's factory for details. """ -import bb import inspect -import types +import oe.types as types +import collections available_types = {} @@ -54,7 +54,9 @@ def get_callable_args(obj): if type(obj) is type: obj = obj.__init__ - args, varargs, keywords, defaults = inspect.getargspec(obj) + sig = inspect.signature(obj) + args = list(sig.parameters.keys()) + defaults = list(s for s in sig.parameters.keys() if sig.parameters[s].default != inspect.Parameter.empty) flaglist = [] if args: if len(args) > 1 and args[0] == 'self': @@ -94,7 +96,7 @@ for name in dir(types): continue obj = getattr(types, name) - if not callable(obj): + if not isinstance(obj, collections.Callable): continue register(name, obj) diff --git a/meta/lib/oe/manifest.py b/meta/lib/oe/manifest.py new file mode 100644 index 0000000000..60c49be0e9 --- /dev/null +++ b/meta/lib/oe/manifest.py @@ -0,0 +1,344 @@ +from abc import ABCMeta, abstractmethod +import os +import re +import bb + + +class Manifest(object, metaclass=ABCMeta): + """ + This is an abstract class. Do not instantiate this directly. + """ + + PKG_TYPE_MUST_INSTALL = "mip" + PKG_TYPE_MULTILIB = "mlp" + PKG_TYPE_LANGUAGE = "lgp" + PKG_TYPE_ATTEMPT_ONLY = "aop" + + MANIFEST_TYPE_IMAGE = "image" + MANIFEST_TYPE_SDK_HOST = "sdk_host" + MANIFEST_TYPE_SDK_TARGET = "sdk_target" + + var_maps = { + MANIFEST_TYPE_IMAGE: { + "PACKAGE_INSTALL": PKG_TYPE_MUST_INSTALL, + "PACKAGE_INSTALL_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY, + "LINGUAS_INSTALL": PKG_TYPE_LANGUAGE + }, + MANIFEST_TYPE_SDK_HOST: { + "TOOLCHAIN_HOST_TASK": PKG_TYPE_MUST_INSTALL, + "TOOLCHAIN_HOST_TASK_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY + }, + MANIFEST_TYPE_SDK_TARGET: { + "TOOLCHAIN_TARGET_TASK": PKG_TYPE_MUST_INSTALL, + "TOOLCHAIN_TARGET_TASK_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY + } + } + + INSTALL_ORDER = [ + PKG_TYPE_LANGUAGE, + PKG_TYPE_MUST_INSTALL, + PKG_TYPE_ATTEMPT_ONLY, + PKG_TYPE_MULTILIB + ] + + initial_manifest_file_header = \ + "# This file was generated automatically and contains the packages\n" \ + "# passed on to the package manager in order to create the rootfs.\n\n" \ + "# Format:\n" \ + "# <package_type>,<package_name>\n" \ + "# where:\n" \ + "# <package_type> can be:\n" \ + "# 'mip' = must install package\n" \ + "# 'aop' = attempt only package\n" \ + "# 'mlp' = multilib package\n" \ + "# 'lgp' = language package\n\n" + + def __init__(self, d, manifest_dir=None, manifest_type=MANIFEST_TYPE_IMAGE): + self.d = d + self.manifest_type = manifest_type + + if manifest_dir is None: + if manifest_type != self.MANIFEST_TYPE_IMAGE: + self.manifest_dir = self.d.getVar('SDK_DIR') + else: + self.manifest_dir = self.d.getVar('WORKDIR') + else: + self.manifest_dir = manifest_dir + + bb.utils.mkdirhier(self.manifest_dir) + + self.initial_manifest = os.path.join(self.manifest_dir, "%s_initial_manifest" % manifest_type) + self.final_manifest = os.path.join(self.manifest_dir, "%s_final_manifest" % manifest_type) + self.full_manifest = os.path.join(self.manifest_dir, "%s_full_manifest" % manifest_type) + + # packages in the following vars will be split in 'must install' and + # 'multilib' + self.vars_to_split = ["PACKAGE_INSTALL", + "TOOLCHAIN_HOST_TASK", + "TOOLCHAIN_TARGET_TASK"] + + """ + This creates a standard initial manifest for core-image-(minimal|sato|sato-sdk). + This will be used for testing until the class is implemented properly! + """ + def _create_dummy_initial(self): + image_rootfs = self.d.getVar('IMAGE_ROOTFS') + pkg_list = dict() + if image_rootfs.find("core-image-sato-sdk") > 0: + pkg_list[self.PKG_TYPE_MUST_INSTALL] = \ + "packagegroup-core-x11-sato-games packagegroup-base-extended " \ + "packagegroup-core-x11-sato packagegroup-core-x11-base " \ + "packagegroup-core-sdk packagegroup-core-tools-debug " \ + "packagegroup-core-boot packagegroup-core-tools-testapps " \ + "packagegroup-core-eclipse-debug packagegroup-core-qt-demoapps " \ + "apt packagegroup-core-tools-profile psplash " \ + "packagegroup-core-standalone-sdk-target " \ + "packagegroup-core-ssh-openssh dpkg kernel-dev" + pkg_list[self.PKG_TYPE_LANGUAGE] = \ + "locale-base-en-us locale-base-en-gb" + elif image_rootfs.find("core-image-sato") > 0: + pkg_list[self.PKG_TYPE_MUST_INSTALL] = \ + "packagegroup-core-ssh-dropbear packagegroup-core-x11-sato-games " \ + "packagegroup-core-x11-base psplash apt dpkg packagegroup-base-extended " \ + "packagegroup-core-x11-sato packagegroup-core-boot" + pkg_list['lgp'] = \ + "locale-base-en-us locale-base-en-gb" + elif image_rootfs.find("core-image-minimal") > 0: + pkg_list[self.PKG_TYPE_MUST_INSTALL] = "packagegroup-core-boot" + + with open(self.initial_manifest, "w+") as manifest: + manifest.write(self.initial_manifest_file_header) + + for pkg_type in pkg_list: + for pkg in pkg_list[pkg_type].split(): + manifest.write("%s,%s\n" % (pkg_type, pkg)) + + """ + This will create the initial manifest which will be used by Rootfs class to + generate the rootfs + """ + @abstractmethod + def create_initial(self): + pass + + """ + This creates the manifest after everything has been installed. + """ + @abstractmethod + def create_final(self): + pass + + """ + This creates the manifest after the package in initial manifest has been + dummy installed. It lists all *to be installed* packages. There is no real + installation, just a test. + """ + @abstractmethod + def create_full(self, pm): + pass + + """ + The following function parses an initial manifest and returns a dictionary + object with the must install, attempt only, multilib and language packages. + """ + def parse_initial_manifest(self): + pkgs = dict() + + with open(self.initial_manifest) as manifest: + for line in manifest.read().split('\n'): + comment = re.match("^#.*", line) + pattern = "^(%s|%s|%s|%s),(.*)$" % \ + (self.PKG_TYPE_MUST_INSTALL, + self.PKG_TYPE_ATTEMPT_ONLY, + self.PKG_TYPE_MULTILIB, + self.PKG_TYPE_LANGUAGE) + pkg = re.match(pattern, line) + + if comment is not None: + continue + + if pkg is not None: + pkg_type = pkg.group(1) + pkg_name = pkg.group(2) + + if not pkg_type in pkgs: + pkgs[pkg_type] = [pkg_name] + else: + pkgs[pkg_type].append(pkg_name) + + return pkgs + + ''' + This following function parses a full manifest and return a list + object with packages. + ''' + def parse_full_manifest(self): + installed_pkgs = list() + if not os.path.exists(self.full_manifest): + bb.note('full manifest not exist') + return installed_pkgs + + with open(self.full_manifest, 'r') as manifest: + for pkg in manifest.read().split('\n'): + installed_pkgs.append(pkg.strip()) + + return installed_pkgs + + +class RpmManifest(Manifest): + """ + Returns a dictionary object with mip and mlp packages. + """ + def _split_multilib(self, pkg_list): + pkgs = dict() + + for pkg in pkg_list.split(): + pkg_type = self.PKG_TYPE_MUST_INSTALL + + ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() + + for ml_variant in ml_variants: + if pkg.startswith(ml_variant + '-'): + pkg_type = self.PKG_TYPE_MULTILIB + + if not pkg_type in pkgs: + pkgs[pkg_type] = pkg + else: + pkgs[pkg_type] += " " + pkg + + return pkgs + + def create_initial(self): + pkgs = dict() + + with open(self.initial_manifest, "w+") as manifest: + manifest.write(self.initial_manifest_file_header) + + for var in self.var_maps[self.manifest_type]: + if var in self.vars_to_split: + split_pkgs = self._split_multilib(self.d.getVar(var)) + if split_pkgs is not None: + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) + else: + pkg_list = self.d.getVar(var) + if pkg_list is not None: + pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) + + for pkg_type in pkgs: + for pkg in pkgs[pkg_type].split(): + manifest.write("%s,%s\n" % (pkg_type, pkg)) + + def create_final(self): + pass + + def create_full(self, pm): + pass + + +class OpkgManifest(Manifest): + """ + Returns a dictionary object with mip and mlp packages. + """ + def _split_multilib(self, pkg_list): + pkgs = dict() + + for pkg in pkg_list.split(): + pkg_type = self.PKG_TYPE_MUST_INSTALL + + ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() + + for ml_variant in ml_variants: + if pkg.startswith(ml_variant + '-'): + pkg_type = self.PKG_TYPE_MULTILIB + + if not pkg_type in pkgs: + pkgs[pkg_type] = pkg + else: + pkgs[pkg_type] += " " + pkg + + return pkgs + + def create_initial(self): + pkgs = dict() + + with open(self.initial_manifest, "w+") as manifest: + manifest.write(self.initial_manifest_file_header) + + for var in self.var_maps[self.manifest_type]: + if var in self.vars_to_split: + split_pkgs = self._split_multilib(self.d.getVar(var)) + if split_pkgs is not None: + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) + else: + pkg_list = self.d.getVar(var) + if pkg_list is not None: + pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) + + for pkg_type in pkgs: + for pkg in pkgs[pkg_type].split(): + manifest.write("%s,%s\n" % (pkg_type, pkg)) + + def create_final(self): + pass + + def create_full(self, pm): + if not os.path.exists(self.initial_manifest): + self.create_initial() + + initial_manifest = self.parse_initial_manifest() + pkgs_to_install = list() + for pkg_type in initial_manifest: + pkgs_to_install += initial_manifest[pkg_type] + if len(pkgs_to_install) == 0: + return + + output = pm.dummy_install(pkgs_to_install) + + with open(self.full_manifest, 'w+') as manifest: + pkg_re = re.compile('^Installing ([^ ]+) [^ ].*') + for line in set(output.split('\n')): + m = pkg_re.match(line) + if m: + manifest.write(m.group(1) + '\n') + + return + + +class DpkgManifest(Manifest): + def create_initial(self): + with open(self.initial_manifest, "w+") as manifest: + manifest.write(self.initial_manifest_file_header) + + for var in self.var_maps[self.manifest_type]: + pkg_list = self.d.getVar(var) + + if pkg_list is None: + continue + + for pkg in pkg_list.split(): + manifest.write("%s,%s\n" % + (self.var_maps[self.manifest_type][var], pkg)) + + def create_final(self): + pass + + def create_full(self, pm): + pass + + +def create_manifest(d, final_manifest=False, manifest_dir=None, + manifest_type=Manifest.MANIFEST_TYPE_IMAGE): + manifest_map = {'rpm': RpmManifest, + 'ipk': OpkgManifest, + 'deb': DpkgManifest} + + manifest = manifest_map[d.getVar('IMAGE_PKGTYPE')](d, manifest_dir, manifest_type) + + if final_manifest: + manifest.create_final() + else: + manifest.create_initial() + + +if __name__ == "__main__": + pass diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py new file mode 100644 index 0000000000..52c5f16cf8 --- /dev/null +++ b/meta/lib/oe/package.py @@ -0,0 +1,167 @@ +def runstrip(arg): + # Function to strip a single file, called from split_and_strip_files below + # A working 'file' (one which works on the target architecture) + # + # The elftype is a bit pattern (explained in split_and_strip_files) to tell + # us what type of file we're processing... + # 4 - executable + # 8 - shared library + # 16 - kernel module + + import stat, subprocess + + (file, elftype, strip) = arg + + newmode = None + if not os.access(file, os.W_OK) or os.access(file, os.R_OK): + origmode = os.stat(file)[stat.ST_MODE] + newmode = origmode | stat.S_IWRITE | stat.S_IREAD + os.chmod(file, newmode) + + stripcmd = [strip] + + # kernel module + if elftype & 16: + stripcmd.extend(["--strip-debug", "--remove-section=.comment", + "--remove-section=.note", "--preserve-dates"]) + # .so and shared library + elif ".so" in file and elftype & 8: + stripcmd.extend(["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"]) + # shared or executable: + elif elftype & 8 or elftype & 4: + stripcmd.extend(["--remove-section=.comment", "--remove-section=.note"]) + + stripcmd.append(file) + bb.debug(1, "runstrip: %s" % stripcmd) + + try: + output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.error("runstrip: '%s' strip command failed with %s (%s)" % (stripcmd, e.returncode, e.output)) + + if newmode: + os.chmod(file, origmode) + + return + + +def file_translate(file): + ft = file.replace("@", "@at@") + ft = ft.replace(" ", "@space@") + ft = ft.replace("\t", "@tab@") + ft = ft.replace("[", "@openbrace@") + ft = ft.replace("]", "@closebrace@") + ft = ft.replace("_", "@underscore@") + return ft + +def filedeprunner(arg): + import re, subprocess, shlex + + (pkg, pkgfiles, rpmdeps, pkgdest, magic) = arg + provides = {} + requires = {} + + r = re.compile(r'[<>=]+ +[^ ]*') + + def process_deps(pipe, pkg, pkgdest, provides, requires): + for line in pipe: + f = line.decode("utf-8").split(" ", 1)[0].strip() + line = line.decode("utf-8").split(" ", 1)[1].strip() + + if line.startswith("Requires:"): + i = requires + elif line.startswith("Provides:"): + i = provides + else: + continue + + file = f.replace(pkgdest + "/" + pkg, "") + file = file_translate(file) + value = line.split(":", 1)[1].strip() + value = r.sub(r'(\g<0>)', value) + + if value.startswith("rpmlib("): + continue + if value == "python": + continue + if file not in i: + i[file] = [] + i[file].append(value) + + return provides, requires + + env = os.environ.copy() + env["MAGIC"] = magic + + try: + dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE, env=env) + provides, requires = process_deps(dep_popen.stdout, pkg, pkgdest, provides, requires) + except OSError as e: + bb.error("rpmdeps: '%s' command failed, '%s'" % (shlex.split(rpmdeps) + pkgfiles, e)) + raise e + + return (pkg, provides, requires) + + +def read_shlib_providers(d): + import re + + shlib_provider = {} + shlibs_dirs = d.getVar('SHLIBSDIRS').split() + list_re = re.compile('^(.*)\.list$') + # Go from least to most specific since the last one found wins + for dir in reversed(shlibs_dirs): + bb.debug(2, "Reading shlib providers in %s" % (dir)) + if not os.path.exists(dir): + continue + for file in os.listdir(dir): + m = list_re.match(file) + if m: + dep_pkg = m.group(1) + try: + fd = open(os.path.join(dir, file)) + except IOError: + # During a build unrelated shlib files may be deleted, so + # handle files disappearing between the listdirs and open. + continue + lines = fd.readlines() + fd.close() + for l in lines: + s = l.strip().split(":") + if s[0] not in shlib_provider: + shlib_provider[s[0]] = {} + shlib_provider[s[0]][s[1]] = (dep_pkg, s[2]) + return shlib_provider + + +def npm_split_package_dirs(pkgdir): + """ + Work out the packages fetched and unpacked by BitBake's npm fetcher + Returns a dict of packagename -> (relpath, package.json) ordered + such that it is suitable for use in PACKAGES and FILES + """ + from collections import OrderedDict + import json + packages = {} + for root, dirs, files in os.walk(pkgdir): + if os.path.basename(root) == 'node_modules': + for dn in dirs: + relpth = os.path.relpath(os.path.join(root, dn), pkgdir) + pkgitems = ['${PN}'] + for pathitem in relpth.split('/'): + if pathitem == 'node_modules': + continue + pkgitems.append(pathitem) + pkgname = '-'.join(pkgitems).replace('_', '-') + pkgname = pkgname.replace('@', '') + pkgfile = os.path.join(root, dn, 'package.json') + data = None + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + data = json.loads(f.read()) + packages[pkgname] = (relpth, data) + # We want the main package for a module sorted *after* its subpackages + # (so that it doesn't otherwise steal the files for the subpackage), so + # this is a cheap way to do that whilst still having an otherwise + # alphabetical sort + return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~')) diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py new file mode 100644 index 0000000000..f1b65bdbbc --- /dev/null +++ b/meta/lib/oe/package_manager.py @@ -0,0 +1,1581 @@ +from abc import ABCMeta, abstractmethod +import os +import glob +import subprocess +import shutil +import multiprocessing +import re +import collections +import bb +import tempfile +import oe.utils +import oe.path +import string +from oe.gpg_sign import get_signer + +# this can be used by all PM backends to create the index files in parallel +def create_index(arg): + index_cmd = arg + + try: + bb.note("Executing '%s' ..." % index_cmd) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + return("Index creation command '%s' failed with return code %d:\n%s" % + (e.cmd, e.returncode, e.output.decode("utf-8"))) + + if result: + bb.note(result) + + return None + +""" +This method parse the output from the package managerand return +a dictionary with the information of the packages. This is used +when the packages are in deb or ipk format. +""" +def opkg_query(cmd_output): + verregex = re.compile(' \([=<>]* [^ )]*\)') + output = dict() + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + for line in cmd_output.splitlines(): + line = line.rstrip() + if ':' in line: + if line.startswith("Package: "): + pkg = line.split(": ")[1] + elif line.startswith("Architecture: "): + arch = line.split(": ")[1] + elif line.startswith("Version: "): + ver = line.split(": ")[1] + elif line.startswith("File: ") or line.startswith("Filename:"): + filename = line.split(": ")[1] + if "/" in filename: + filename = os.path.basename(filename) + elif line.startswith("Depends: "): + depends = verregex.sub('', line.split(": ")[1]) + for depend in depends.split(", "): + dep.append(depend) + elif line.startswith("Recommends: "): + recommends = verregex.sub('', line.split(": ")[1]) + for recommend in recommends.split(", "): + dep.append("%s [REC]" % recommend) + elif line.startswith("PackageArch: "): + pkgarch = line.split(": ")[1] + + # When there is a blank line save the package information + elif not line: + # IPK doesn't include the filename + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + if pkg: + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep, "pkgarch":pkgarch } + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + + if pkg: + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + + return output + + +class Indexer(object, metaclass=ABCMeta): + def __init__(self, d, deploy_dir): + self.d = d + self.deploy_dir = deploy_dir + + @abstractmethod + def write_index(self): + pass + + +class RpmIndexer(Indexer): + def write_index(self): + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + raise NotImplementedError('Package feed signing not yet implementd for rpm') + + createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c") + result = create_index("%s --update -q %s" % (createrepo_c, self.deploy_dir)) + if result: + bb.fatal(result) + +class OpkgIndexer(Indexer): + def write_index(self): + arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", + "SDK_PACKAGE_ARCHS", + "MULTILIB_ARCHS"] + + opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) + else: + signer = None + + if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): + open(os.path.join(self.deploy_dir, "Packages"), "w").close() + + index_cmds = set() + index_sign_files = set() + for arch_var in arch_vars: + archs = self.d.getVar(arch_var) + if archs is None: + continue + + for arch in archs.split(): + pkgs_dir = os.path.join(self.deploy_dir, arch) + pkgs_file = os.path.join(pkgs_dir, "Packages") + + if not os.path.isdir(pkgs_dir): + continue + + if not os.path.exists(pkgs_file): + open(pkgs_file, "w").close() + + index_cmds.add('%s -r %s -p %s -m %s' % + (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) + + index_sign_files.add(pkgs_file) + + if len(index_cmds) == 0: + bb.note("There are no packages in %s!" % self.deploy_dir) + return + + result = oe.utils.multiprocess_exec(index_cmds, create_index) + if result: + bb.fatal('%s' % ('\n'.join(result))) + + if signer: + feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') + is_ascii_sig = (feed_sig_type.upper() != "BIN") + for f in index_sign_files: + signer.detach_sign(f, + self.d.getVar('PACKAGE_FEED_GPG_NAME'), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), + armor=is_ascii_sig) + + +class DpkgIndexer(Indexer): + def _create_configs(self): + bb.utils.mkdirhier(self.apt_conf_dir) + bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "lists", "partial")) + bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "apt.conf.d")) + bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "preferences.d")) + + with open(os.path.join(self.apt_conf_dir, "preferences"), + "w") as prefs_file: + pass + with open(os.path.join(self.apt_conf_dir, "sources.list"), + "w+") as sources_file: + pass + + with open(self.apt_conf_file, "w") as apt_conf: + with open(os.path.join(self.d.expand("${STAGING_ETCDIR_NATIVE}"), + "apt", "apt.conf.sample")) as apt_conf_sample: + for line in apt_conf_sample.read().split("\n"): + line = re.sub("#ROOTFS#", "/dev/null", line) + line = re.sub("#APTCONF#", self.apt_conf_dir, line) + apt_conf.write(line + "\n") + + def write_index(self): + self.apt_conf_dir = os.path.join(self.d.expand("${APTCONF_TARGET}"), + "apt-ftparchive") + self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") + self._create_configs() + + os.environ['APT_CONFIG'] = self.apt_conf_file + + pkg_archs = self.d.getVar('PACKAGE_ARCHS') + if pkg_archs is not None: + arch_list = pkg_archs.split() + sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS') + if sdk_pkg_archs is not None: + for a in sdk_pkg_archs.split(): + if a not in pkg_archs: + arch_list.append(a) + + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() + arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list) + + apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive") + gzip = bb.utils.which(os.getenv('PATH'), "gzip") + + index_cmds = [] + deb_dirs_found = False + for arch in arch_list: + arch_dir = os.path.join(self.deploy_dir, arch) + if not os.path.isdir(arch_dir): + continue + + cmd = "cd %s; PSEUDO_UNLOAD=1 %s packages . > Packages;" % (arch_dir, apt_ftparchive) + + cmd += "%s -fc Packages > Packages.gz;" % gzip + + with open(os.path.join(arch_dir, "Release"), "w+") as release: + release.write("Label: %s\n" % arch) + + cmd += "PSEUDO_UNLOAD=1 %s release . >> Release" % apt_ftparchive + + index_cmds.append(cmd) + + deb_dirs_found = True + + if not deb_dirs_found: + bb.note("There are no packages in %s" % self.deploy_dir) + return + + result = oe.utils.multiprocess_exec(index_cmds, create_index) + if result: + bb.fatal('%s' % ('\n'.join(result))) + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + raise NotImplementedError('Package feed signing not implementd for dpkg') + + + +class PkgsList(object, metaclass=ABCMeta): + def __init__(self, d, rootfs_dir): + self.d = d + self.rootfs_dir = rootfs_dir + + @abstractmethod + def list_pkgs(self): + pass + +class RpmPkgsList(PkgsList): + def list_pkgs(self): + return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR')).list_installed() + +class OpkgPkgsList(PkgsList): + def __init__(self, d, rootfs_dir, config_file): + super(OpkgPkgsList, self).__init__(d, rootfs_dir) + + self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") + self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) + self.opkg_args += self.d.getVar("OPKG_ARGS") + + def list_pkgs(self, format=None): + cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) + + # opkg returns success even when it printed some + # "Collected errors:" report to stderr. Mixing stderr into + # stdout then leads to random failures later on when + # parsing the output. To avoid this we need to collect both + # output streams separately and check for empty stderr. + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + cmd_output, cmd_stderr = p.communicate() + cmd_output = cmd_output.decode("utf-8") + cmd_stderr = cmd_stderr.decode("utf-8") + if p.returncode or cmd_stderr: + bb.fatal("Cannot get the installed packages list. Command '%s' " + "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) + + return opkg_query(cmd_output) + + +class DpkgPkgsList(PkgsList): + + def list_pkgs(self): + cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"), + "--admindir=%s/var/lib/dpkg" % self.rootfs_dir, + "-W"] + + cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") + + try: + cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Cannot get the installed packages list. Command '%s' " + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + + return opkg_query(cmd_output) + + +class PackageManager(object, metaclass=ABCMeta): + """ + This is an abstract class. Do not instantiate this directly. + """ + + def __init__(self, d): + self.d = d + self.deploy_dir = None + self.deploy_lock = None + + """ + Update the package manager package database. + """ + @abstractmethod + def update(self): + pass + + """ + Install a list of packages. 'pkgs' is a list object. If 'attempt_only' is + True, installation failures are ignored. + """ + @abstractmethod + def install(self, pkgs, attempt_only=False): + pass + + """ + Remove a list of packages. 'pkgs' is a list object. If 'with_dependencies' + is False, the any dependencies are left in place. + """ + @abstractmethod + def remove(self, pkgs, with_dependencies=True): + pass + + """ + This function creates the index files + """ + @abstractmethod + def write_index(self): + pass + + @abstractmethod + def remove_packaging_data(self): + pass + + @abstractmethod + def list_installed(self): + pass + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + """ + @abstractmethod + def extract(self, pkg): + pass + + """ + Add remote package feeds into repository manager configuration. The parameters + for the feeds are set by feed_uris, feed_base_paths and feed_archs. + See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS + for their description. + """ + @abstractmethod + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + pass + + """ + Install complementary packages based upon the list of currently installed + packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install + these packages, if they don't exist then no error will occur. Note: every + backend needs to call this function explicitly after the normal package + installation + """ + def install_complementary(self, globs=None): + if globs is None: + globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY') + split_linguas = set() + + for translation in self.d.getVar('IMAGE_LINGUAS').split(): + split_linguas.add(translation) + split_linguas.add(translation.split('-')[0]) + + split_linguas = sorted(split_linguas) + + for lang in split_linguas: + globs += " *-locale-%s" % lang + + if globs is None: + return + + # we need to write the list of installed packages to a file because the + # oe-pkgdata-util reads it from a file + with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs: + pkgs = self.list_installed() + output = oe.utils.format_pkg_list(pkgs, "arch") + installed_pkgs.write(output) + + cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), + "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name, + globs] + exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY') + if exclude: + cmd.extend(['--exclude=' + '|'.join(exclude.split())]) + try: + bb.note("Installing complementary packages ...") + bb.note('Running %s' % cmd) + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not compute complementary packages list. Command " + "'%s' returned %d:\n%s" % + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + self.install(complementary_pkgs.split(), attempt_only=True) + + def deploy_dir_lock(self): + if self.deploy_dir is None: + raise RuntimeError("deploy_dir is not set!") + + lock_file_name = os.path.join(self.deploy_dir, "deploy.lock") + + self.deploy_lock = bb.utils.lockfile(lock_file_name) + + def deploy_dir_unlock(self): + if self.deploy_lock is None: + return + + bb.utils.unlockfile(self.deploy_lock) + + self.deploy_lock = None + + """ + Construct URIs based on the following pattern: uri/base_path where 'uri' + and 'base_path' correspond to each element of the corresponding array + argument leading to len(uris) x len(base_paths) elements on the returned + array + """ + def construct_uris(self, uris, base_paths): + def _append(arr1, arr2, sep='/'): + res = [] + narr1 = [a.rstrip(sep) for a in arr1] + narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2] + for a1 in narr1: + if arr2: + for a2 in narr2: + res.append("%s%s%s" % (a1, sep, a2)) + else: + res.append(a1) + return res + return _append(uris, base_paths) + +class RpmPM(PackageManager): + def __init__(self, + d, + target_rootfs, + target_vendor, + task_name='target', + providename=None, + arch_var=None, + os_var=None): + super(RpmPM, self).__init__(d) + self.target_rootfs = target_rootfs + self.target_vendor = target_vendor + self.task_name = task_name + if arch_var == None: + self.archs = self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS').replace("-","_") + else: + self.archs = self.d.getVar(arch_var).replace("-","_") + if task_name == "host": + self.primary_arch = self.d.getVar('SDK_ARCH') + else: + self.primary_arch = self.d.getVar('MACHINE_ARCH') + + self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), "oe-rootfs-repo") + bb.utils.mkdirhier(self.rpm_repo_dir) + oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True) + + self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name) + if not os.path.exists(self.d.expand('${T}/saved_packaging_data')): + bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data')) + self.packaging_data_dirs = ['var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf'] + self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % + self.task_name) + if not os.path.exists(self.d.expand('${T}/saved')): + bb.utils.mkdirhier(self.d.expand('${T}/saved')) + + def _configure_dnf(self): + # libsolv handles 'noarch' internally, we don't need to specify it explicitly + archs = [i for i in reversed(self.archs.split()) if i not in ["any", "all", "noarch"]] + # This prevents accidental matching against libsolv's built-in policies + if len(archs) <= 1: + archs = archs + ["bogusarch"] + archconfdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/") + bb.utils.mkdirhier(archconfdir) + open(archconfdir + "arch", 'w').write(":".join(archs)) + + open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w').write("") + + + def _configure_rpm(self): + # We need to configure rpm to use our primary package architecture as the installation architecture, + # and to make it compatible with other package architectures that we use. + # Otherwise it will refuse to proceed with packages installation. + platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/") + rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/") + bb.utils.mkdirhier(platformconfdir) + open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch) + open(rpmrcconfdir + "rpmrc", 'w').write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch)) + + open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n") + if self.d.getVar('RPM_PREFER_ELF_ARCH'): + open(platformconfdir + "macros", 'a').write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH'))) + else: + open(platformconfdir + "macros", 'a').write("%_prefer_color 7") + + if self.d.getVar('RPM_SIGN_PACKAGES') == '1': + signer = get_signer(self.d, self.d.getVar('RPM_GPG_BACKEND')) + pubkey_path = oe.path.join(self.d.getVar('B'), 'rpm-key') + signer.export_pubkey(pubkey_path, self.d.getVar('RPM_GPG_NAME')) + rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmkeys") + cmd = [rpm_bin, '--root=%s' % self.target_rootfs, '--import', pubkey_path] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Importing GPG key failed. Command '%s' " + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + + def create_configs(self): + self._configure_dnf() + self._configure_rpm() + + def write_index(self): + lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock" + lf = bb.utils.lockfile(lockfilename, False) + RpmIndexer(self.d, self.rpm_repo_dir).write_index() + bb.utils.unlockfile(lf) + + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + from urllib.parse import urlparse + + if feed_uris == "": + return + + bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d")) + remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + for uri in remote_uris: + repo_base = "oe-remote-repo" + "-".join(urlparse(uri).path.split("/")) + if feed_archs is not None: + for arch in feed_archs.split(): + repo_uri = uri + "/" + arch + repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/")) + repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/")) + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write( + "[%s]\nname=%s\nbaseurl=%s\n\n" % (repo_id, repo_name, repo_uri)) + else: + repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/")) + repo_uri = uri + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write( + "[%s]\nname=%s\nbaseurl=%s\n" % (repo_base, repo_name, repo_uri)) + + def _prepare_pkg_transaction(self): + os.environ['D'] = self.target_rootfs + os.environ['OFFLINE_ROOT'] = self.target_rootfs + os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['INTERCEPT_DIR'] = oe.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') + + + def install(self, pkgs, attempt_only = False): + if len(pkgs) == 0: + return + self._prepare_pkg_transaction() + + bad_recommendations = self.d.getVar('BAD_RECOMMENDATIONS') + package_exclude = self.d.getVar('PACKAGE_EXCLUDE') + exclude_pkgs = (bad_recommendations.split() if bad_recommendations else []) + (package_exclude.split() if package_exclude else []) + + output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) + + (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) + + (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) + + (["--nogpgcheck"] if self.d.getVar('RPM_SIGN_PACKAGES') != '1' else ["--setopt=gpgcheck=True"]) + + ["install"] + + pkgs) + + failed_scriptlets_pkgnames = collections.OrderedDict() + for line in output.splitlines(): + if line.startswith("Non-fatal POSTIN scriptlet failure in rpm package"): + failed_scriptlets_pkgnames[line.split()[-1]] = True + + for pkg in failed_scriptlets_pkgnames.keys(): + self.save_rpmpostinst(pkg) + + def remove(self, pkgs, with_dependencies = True): + if len(pkgs) == 0: + return + self._prepare_pkg_transaction() + + if with_dependencies: + self._invoke_dnf(["remove"] + pkgs) + else: + cmd = bb.utils.which(os.getenv('PATH'), "rpm") + args = ["-e", "--nodeps", "--root=%s" %self.target_rootfs] + + try: + output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not invoke rpm. Command " + "'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8"))) + + def upgrade(self): + self._prepare_pkg_transaction() + self._invoke_dnf(["upgrade"]) + + def autoremove(self): + self._prepare_pkg_transaction() + self._invoke_dnf(["autoremove"]) + + def remove_packaging_data(self): + self._invoke_dnf(["clean", "all"]) + for dir in self.packaging_data_dirs: + bb.utils.remove(oe.path.join(self.target_rootfs, dir), True) + + def backup_packaging_data(self): + # Save the packaging dirs for increment rpm image generation + if os.path.exists(self.saved_packaging_data): + bb.utils.remove(self.saved_packaging_data, True) + for i in self.packaging_data_dirs: + source_dir = oe.path.join(self.target_rootfs, i) + target_dir = oe.path.join(self.saved_packaging_data, i) + shutil.copytree(source_dir, target_dir, symlinks=True) + + def recovery_packaging_data(self): + # Move the rpmlib back + if os.path.exists(self.saved_packaging_data): + for i in self.packaging_data_dirs: + target_dir = oe.path.join(self.target_rootfs, i) + if os.path.exists(target_dir): + bb.utils.remove(target_dir, True) + source_dir = oe.path.join(self.saved_packaging_data, i) + shutil.copytree(source_dir, + target_dir, + symlinks=True) + + def list_installed(self): + output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{sourcerpm}\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"], + print_output = False) + packages = {} + current_package = None + current_deps = None + current_state = "initial" + for line in output.splitlines(): + if line.startswith("Package:"): + package_info = line.split(" ")[1:] + current_package = package_info[0] + package_arch = package_info[1] + package_version = package_info[2] + package_srpm = package_info[3] + packages[current_package] = {"arch":package_arch, "ver":package_version, "filename":package_srpm} + current_deps = [] + elif line.startswith("Dependencies:"): + current_state = "dependencies" + elif line.startswith("Recommendations"): + current_state = "recommendations" + elif line.startswith("DependenciesEndHere:"): + current_state = "initial" + packages[current_package]["deps"] = current_deps + elif len(line) > 0: + if current_state == "dependencies": + current_deps.append(line) + elif current_state == "recommendations": + current_deps.append("%s [REC]" % line) + + return packages + + def update(self): + self._invoke_dnf(["makecache"]) + + def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ): + os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs + + dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf") + standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y", + "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), + "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")), + "--repofrompath=oe-repo,%s" % (self.rpm_repo_dir), + "--installroot=%s" % (self.target_rootfs), + "--setopt=logdir=%s" % (self.d.getVar('T')) + ] + cmd = [dnf_cmd] + standard_dnf_args + dnf_args + try: + output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8") + if print_output: + bb.note(output) + return output + except subprocess.CalledProcessError as e: + if print_output: + (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command " + "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + else: + (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command " + "'%s' returned %d:" % (' '.join(cmd), e.returncode)) + return e.output.decode("utf-8") + + def dump_install_solution(self, pkgs): + open(self.solution_manifest, 'w').write(" ".join(pkgs)) + return pkgs + + def load_old_install_solution(self): + if not os.path.exists(self.solution_manifest): + return [] + + return open(self.solution_manifest, 'r').read().split() + + def _script_num_prefix(self, path): + files = os.listdir(path) + numbers = set() + numbers.add(99) + for f in files: + numbers.add(int(f.split("-")[0])) + return max(numbers) + 1 + + def save_rpmpostinst(self, pkg): + bb.note("Saving postinstall script of %s" % (pkg)) + cmd = bb.utils.which(os.getenv('PATH'), "rpm") + args = ["-q", "--root=%s" % self.target_rootfs, "--queryformat", "%{postin}", pkg] + + try: + output = subprocess.check_output([cmd] + args,stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Could not invoke rpm. Command " + "'%s' returned %d:\n%s" % (' '.join([cmd] + args), e.returncode, e.output.decode("utf-8"))) + + # may need to prepend #!/bin/sh to output + + target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/')) + bb.utils.mkdirhier(target_path) + num = self._script_num_prefix(target_path) + saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg)) + open(saved_script_name, 'w').write(output) + os.chmod(saved_script_name, 0o755) + + def extract(self, pkg): + output = self._invoke_dnf(["repoquery", "--queryformat", "%{location}", pkg]) + pkg_name = output.splitlines()[-1] + if not pkg_name.endswith(".rpm"): + bb.fatal("dnf could not find package %s in repository: %s" %(pkg, output)) + pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name) + + cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") + rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + os.chdir(current_dir) + + return tmp_dir + + +class OpkgDpkgPM(PackageManager): + """ + This is an abstract class. Do not instantiate this directly. + """ + def __init__(self, d): + super(OpkgDpkgPM, self).__init__(d) + + """ + Returns a dictionary with the package info. + + This method extracts the common parts for Opkg and Dpkg + """ + def package_info(self, pkg, cmd): + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + return opkg_query(output) + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + This method extracts the common parts for Opkg and Dpkg + """ + def extract(self, pkg, pkg_info): + + ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") + tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") + pkg_path = pkg_info[pkg]["filepath"] + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + if self.d.getVar('IMAGE_PKGTYPE') == 'deb': + data_tar = 'data.tar.xz' + else: + data_tar = 'data.tar.gz' + + try: + cmd = [ar_cmd, 'x', pkg_path] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + cmd = [tar_cmd, 'xf', data_tar] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) + bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) + os.chdir(current_dir) + + return tmp_dir + + +class OpkgPM(OpkgDpkgPM): + def __init__(self, d, target_rootfs, config_file, archs, task_name='target'): + super(OpkgPM, self).__init__(d) + + self.target_rootfs = target_rootfs + self.config_file = config_file + self.pkg_archs = archs + self.task_name = task_name + + self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK") + self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") + self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") + self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) + self.opkg_args += self.d.getVar("OPKG_ARGS") + + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') + if opkg_lib_dir[0] == "/": + opkg_lib_dir = opkg_lib_dir[1:] + + self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg") + + bb.utils.mkdirhier(self.opkg_dir) + + self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name) + if not os.path.exists(self.d.expand('${T}/saved')): + bb.utils.mkdirhier(self.d.expand('${T}/saved')) + + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" + if self.from_feeds: + self._create_custom_config() + else: + self._create_config() + + self.indexer = OpkgIndexer(self.d, self.deploy_dir) + + """ + This function will change a package's status in /var/lib/opkg/status file. + If 'packages' is None then the new_status will be applied to all + packages + """ + def mark_packages(self, status_tag, packages=None): + status_file = os.path.join(self.opkg_dir, "status") + + with open(status_file, "r") as sf: + with open(status_file + ".tmp", "w+") as tmp_sf: + if packages is None: + tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)", + r"Package: \1\n\2Status: \3%s" % status_tag, + sf.read())) + else: + if type(packages).__name__ != "list": + raise TypeError("'packages' should be a list object") + + status = sf.read() + for pkg in packages: + status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg, + r"Package: %s\n\1Status: \2%s" % (pkg, status_tag), + status) + + tmp_sf.write(status) + + os.rename(status_file + ".tmp", status_file) + + def _create_custom_config(self): + bb.note("Building from feeds activated!") + + with open(self.config_file, "w+") as config_file: + priority = 1 + for arch in self.pkg_archs.split(): + config_file.write("arch %s %d\n" % (arch, priority)) + priority += 5 + + for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): + feed_match = re.match("^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) + + if feed_match is not None: + feed_name = feed_match.group(1) + feed_uri = feed_match.group(2) + + bb.note("Add %s feed with URL %s" % (feed_name, feed_uri)) + + config_file.write("src/gz %s %s\n" % (feed_name, feed_uri)) + + """ + Allow to use package deploy directory contents as quick devel-testing + feed. This creates individual feed configs for each arch subdir of those + specified as compatible for the current machine. + NOTE: Development-helper feature, NOT a full-fledged feed. + """ + if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": + for arch in self.pkg_archs.split(): + cfg_file_name = os.path.join(self.target_rootfs, + self.d.getVar("sysconfdir"), + "opkg", + "local-%s-feed.conf" % arch) + + with open(cfg_file_name, "w+") as cfg_file: + cfg_file.write("src/gz local-%s %s/%s" % + (arch, + self.d.getVar('FEED_DEPLOYDIR_BASE_URI'), + arch)) + + if self.d.getVar('OPKGLIBDIR') != '/var/lib': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" + cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) + cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) + cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) + + + def _create_config(self): + with open(self.config_file, "w+") as config_file: + priority = 1 + for arch in self.pkg_archs.split(): + config_file.write("arch %s %d\n" % (arch, priority)) + priority += 5 + + config_file.write("src oe file:%s\n" % self.deploy_dir) + + for arch in self.pkg_archs.split(): + pkgs_dir = os.path.join(self.deploy_dir, arch) + if os.path.isdir(pkgs_dir): + config_file.write("src oe-%s file:%s\n" % + (arch, pkgs_dir)) + + if self.d.getVar('OPKGLIBDIR') != '/var/lib': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" + config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) + config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) + config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) + + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + if feed_uris == "": + return + + rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' + % self.target_rootfs) + + feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split() + + with open(rootfs_config, "w+") as config_file: + uri_iterator = 0 + for uri in feed_uris: + if archs: + for arch in archs: + if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))): + continue + bb.note('Adding opkg feed url-%s-%d (%s)' % + (arch, uri_iterator, uri)) + config_file.write("src/gz uri-%s-%d %s/%s\n" % + (arch, uri_iterator, uri, arch)) + else: + bb.note('Adding opkg feed url-%d (%s)' % + (uri_iterator, uri)) + config_file.write("src/gz uri-%d %s\n" % + (uri_iterator, uri)) + + uri_iterator += 1 + + def update(self): + self.deploy_dir_lock() + + cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args) + + try: + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + self.deploy_dir_unlock() + bb.fatal("Unable to update the package index files. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + self.deploy_dir_unlock() + + def install(self, pkgs, attempt_only=False): + if not pkgs: + return + + cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) + + os.environ['D'] = self.target_rootfs + os.environ['OFFLINE_ROOT'] = self.target_rootfs + os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') + + try: + bb.note("Installing the following packages: %s" % ' '.join(pkgs)) + bb.note(cmd) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") + bb.note(output) + except subprocess.CalledProcessError as e: + (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " + "Command '%s' returned %d:\n%s" % + (cmd, e.returncode, e.output.decode("utf-8"))) + + def remove(self, pkgs, with_dependencies=True): + if with_dependencies: + cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ + (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) + else: + cmd = "%s %s --force-depends remove %s" % \ + (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) + + try: + bb.note(cmd) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") + bb.note(output) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to remove packages. Command '%s' " + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) + + def write_index(self): + self.deploy_dir_lock() + + result = self.indexer.write_index() + + self.deploy_dir_unlock() + + if result is not None: + bb.fatal(result) + + def remove_packaging_data(self): + bb.utils.remove(self.opkg_dir, True) + # create the directory back, it's needed by PM lock + bb.utils.mkdirhier(self.opkg_dir) + + def remove_lists(self): + if not self.from_feeds: + bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True) + + def list_installed(self): + return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs() + + def handle_bad_recommendations(self): + bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS") or "" + if bad_recommendations.strip() == "": + return + + status_file = os.path.join(self.opkg_dir, "status") + + # If status file existed, it means the bad recommendations has already + # been handled + if os.path.exists(status_file): + return + + cmd = "%s %s info " % (self.opkg_cmd, self.opkg_args) + + with open(status_file, "w+") as status: + for pkg in bad_recommendations.split(): + pkg_info = cmd + pkg + + try: + output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip().decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Cannot get package info. Command '%s' " + "returned %d:\n%s" % (pkg_info, e.returncode, e.output.decode("utf-8"))) + + if output == "": + bb.note("Ignored bad recommendation: '%s' is " + "not a package" % pkg) + continue + + for line in output.split('\n'): + if line.startswith("Status:"): + status.write("Status: deinstall hold not-installed\n") + else: + status.write(line + "\n") + + # Append a blank line after each package entry to ensure that it + # is separated from the following entry + status.write("\n") + + ''' + The following function dummy installs pkgs and returns the log of output. + ''' + def dummy_install(self, pkgs): + if len(pkgs) == 0: + return + + # Create an temp dir as opkg root for dummy installation + temp_rootfs = self.d.expand('${T}/opkg') + opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + if opkg_lib_dir[0] == "/": + opkg_lib_dir = opkg_lib_dir[1:] + temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') + bb.utils.mkdirhier(temp_opkg_dir) + + opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) + opkg_args += self.d.getVar("OPKG_ARGS") + + cmd = "%s %s update" % (self.opkg_cmd, opkg_args) + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to update. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + # Dummy installation + cmd = "%s %s --noaction install %s " % (self.opkg_cmd, + opkg_args, + ' '.join(pkgs)) + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to dummy install packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + bb.utils.remove(temp_rootfs, True) + + return output + + def backup_packaging_data(self): + # Save the opkglib for increment ipk image generation + if os.path.exists(self.saved_opkg_dir): + bb.utils.remove(self.saved_opkg_dir, True) + shutil.copytree(self.opkg_dir, + self.saved_opkg_dir, + symlinks=True) + + def recover_packaging_data(self): + # Move the opkglib back + if os.path.exists(self.saved_opkg_dir): + if os.path.exists(self.opkg_dir): + bb.utils.remove(self.opkg_dir, True) + + bb.note('Recover packaging data') + shutil.copytree(self.saved_opkg_dir, + self.opkg_dir, + symlinks=True) + + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) + pkg_info = super(OpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["arch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz")) + + return tmp_dir + +class DpkgPM(OpkgDpkgPM): + def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): + super(DpkgPM, self).__init__(d) + self.target_rootfs = target_rootfs + self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB') + if apt_conf_dir is None: + self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt") + else: + self.apt_conf_dir = apt_conf_dir + self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") + self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") + self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") + + self.apt_args = d.getVar("APT_ARGS") + + self.all_arch_list = archs.split() + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() + self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list) + + self._create_configs(archs, base_archs) + + self.indexer = DpkgIndexer(self.d, self.deploy_dir) + + """ + This function will change a package's status in /var/lib/dpkg/status file. + If 'packages' is None then the new_status will be applied to all + packages + """ + def mark_packages(self, status_tag, packages=None): + status_file = self.target_rootfs + "/var/lib/dpkg/status" + + with open(status_file, "r") as sf: + with open(status_file + ".tmp", "w+") as tmp_sf: + if packages is None: + tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)", + r"Package: \1\n\2Status: \3%s" % status_tag, + sf.read())) + else: + if type(packages).__name__ != "list": + raise TypeError("'packages' should be a list object") + + status = sf.read() + for pkg in packages: + status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg, + r"Package: %s\n\1Status: \2%s" % (pkg, status_tag), + status) + + tmp_sf.write(status) + + os.rename(status_file + ".tmp", status_file) + + """ + Run the pre/post installs for package "package_name". If package_name is + None, then run all pre/post install scriptlets. + """ + def run_pre_post_installs(self, package_name=None): + info_dir = self.target_rootfs + "/var/lib/dpkg/info" + ControlScript = collections.namedtuple("ControlScript", ["suffix", "name", "argument"]) + control_scripts = [ + ControlScript(".preinst", "Preinstall", "install"), + ControlScript(".postinst", "Postinstall", "configure")] + status_file = self.target_rootfs + "/var/lib/dpkg/status" + installed_pkgs = [] + + with open(status_file, "r") as status: + for line in status.read().split('\n'): + m = re.match("^Package: (.*)", line) + if m is not None: + installed_pkgs.append(m.group(1)) + + if package_name is not None and not package_name in installed_pkgs: + return + + os.environ['D'] = self.target_rootfs + os.environ['OFFLINE_ROOT'] = self.target_rootfs + os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs + os.environ['INTERCEPT_DIR'] = os.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') + + failed_pkgs = [] + for pkg_name in installed_pkgs: + for control_script in control_scripts: + p_full = os.path.join(info_dir, pkg_name + control_script.suffix) + if os.path.exists(p_full): + try: + bb.note("Executing %s for package: %s ..." % + (control_script.name.lower(), pkg_name)) + output = subprocess.check_output([p_full, control_script.argument], + stderr=subprocess.STDOUT).decode("utf-8") + bb.note(output) + except subprocess.CalledProcessError as e: + bb.note("%s for package %s failed with %d:\n%s" % + (control_script.name, pkg_name, e.returncode, + e.output.decode("utf-8"))) + failed_pkgs.append(pkg_name) + break + + if len(failed_pkgs): + self.mark_packages("unpacked", failed_pkgs) + + def update(self): + os.environ['APT_CONFIG'] = self.apt_conf_file + + self.deploy_dir_lock() + + cmd = "%s update" % self.apt_get_cmd + + try: + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to update the package index files. Command '%s' " + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) + + self.deploy_dir_unlock() + + def install(self, pkgs, attempt_only=False): + if attempt_only and len(pkgs) == 0: + return + + os.environ['APT_CONFIG'] = self.apt_conf_file + + cmd = "%s %s install --force-yes --allow-unauthenticated %s" % \ + (self.apt_get_cmd, self.apt_args, ' '.join(pkgs)) + + try: + bb.note("Installing the following packages: %s" % ' '.join(pkgs)) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " + "Command '%s' returned %d:\n%s" % + (cmd, e.returncode, e.output.decode("utf-8"))) + + # rename *.dpkg-new files/dirs + for root, dirs, files in os.walk(self.target_rootfs): + for dir in dirs: + new_dir = re.sub("\.dpkg-new", "", dir) + if dir != new_dir: + os.rename(os.path.join(root, dir), + os.path.join(root, new_dir)) + + for file in files: + new_file = re.sub("\.dpkg-new", "", file) + if file != new_file: + os.rename(os.path.join(root, file), + os.path.join(root, new_file)) + + + def remove(self, pkgs, with_dependencies=True): + if with_dependencies: + os.environ['APT_CONFIG'] = self.apt_conf_file + cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs)) + else: + cmd = "%s --admindir=%s/var/lib/dpkg --instdir=%s" \ + " -P --force-depends %s" % \ + (bb.utils.which(os.getenv('PATH'), "dpkg"), + self.target_rootfs, self.target_rootfs, ' '.join(pkgs)) + + try: + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Unable to remove packages. Command '%s' " + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) + + def write_index(self): + self.deploy_dir_lock() + + result = self.indexer.write_index() + + self.deploy_dir_unlock() + + if result is not None: + bb.fatal(result) + + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + if feed_uris == "": + return + + sources_conf = os.path.join("%s/etc/apt/sources.list" + % self.target_rootfs) + arch_list = [] + + if feed_archs is None: + for arch in self.all_arch_list: + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + else: + arch_list = feed_archs.split() + + feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + + with open(sources_conf, "w+") as sources_file: + for uri in feed_uris: + if arch_list: + for arch in arch_list: + bb.note('Adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s/%s ./\n" % + (uri, arch)) + else: + bb.note('Adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s ./\n" % uri) + + def _create_configs(self, archs, base_archs): + base_archs = re.sub("_", "-", base_archs) + + if os.path.exists(self.apt_conf_dir): + bb.utils.remove(self.apt_conf_dir, True) + + bb.utils.mkdirhier(self.apt_conf_dir) + bb.utils.mkdirhier(self.apt_conf_dir + "/lists/partial/") + bb.utils.mkdirhier(self.apt_conf_dir + "/apt.conf.d/") + bb.utils.mkdirhier(self.apt_conf_dir + "/preferences.d/") + + arch_list = [] + for arch in self.all_arch_list: + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + + with open(os.path.join(self.apt_conf_dir, "preferences"), "w+") as prefs_file: + priority = 801 + for arch in arch_list: + prefs_file.write( + "Package: *\n" + "Pin: release l=%s\n" + "Pin-Priority: %d\n\n" % (arch, priority)) + + priority += 5 + + pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or "" + for pkg in pkg_exclude.split(): + prefs_file.write( + "Package: %s\n" + "Pin: release *\n" + "Pin-Priority: -1\n\n" % pkg) + + arch_list.reverse() + + with open(os.path.join(self.apt_conf_dir, "sources.list"), "w+") as sources_file: + for arch in arch_list: + sources_file.write("deb file:%s/ ./\n" % + os.path.join(self.deploy_dir, arch)) + + base_arch_list = base_archs.split() + multilib_variants = self.d.getVar("MULTILIB_VARIANTS"); + for variant in multilib_variants.split(): + localdata = bb.data.createCopy(self.d) + variant_tune = localdata.getVar("DEFAULTTUNE_virtclass-multilib-" + variant, False) + orig_arch = localdata.getVar("DPKG_ARCH") + localdata.setVar("DEFAULTTUNE", variant_tune) + variant_arch = localdata.getVar("DPKG_ARCH") + if variant_arch not in base_arch_list: + base_arch_list.append(variant_arch) + + with open(self.apt_conf_file, "w+") as apt_conf: + with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample: + for line in apt_conf_sample.read().split("\n"): + match_arch = re.match(" Architecture \".*\";$", line) + architectures = "" + if match_arch: + for base_arch in base_arch_list: + architectures += "\"%s\";" % base_arch + apt_conf.write(" Architectures {%s};\n" % architectures); + apt_conf.write(" Architecture \"%s\";\n" % base_archs) + else: + line = re.sub("#ROOTFS#", self.target_rootfs, line) + line = re.sub("#APTCONF#", self.apt_conf_dir, line) + apt_conf.write(line + "\n") + + target_dpkg_dir = "%s/var/lib/dpkg" % self.target_rootfs + bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "info")) + + bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "updates")) + + if not os.path.exists(os.path.join(target_dpkg_dir, "status")): + open(os.path.join(target_dpkg_dir, "status"), "w+").close() + if not os.path.exists(os.path.join(target_dpkg_dir, "available")): + open(os.path.join(target_dpkg_dir, "available"), "w+").close() + + def remove_packaging_data(self): + bb.utils.remove(os.path.join(self.target_rootfs, + self.d.getVar('opkglibdir')), True) + bb.utils.remove(self.target_rootfs + "/var/lib/dpkg/", True) + + def fix_broken_dependencies(self): + os.environ['APT_CONFIG'] = self.apt_conf_file + + cmd = "%s %s -f install" % (self.apt_get_cmd, self.apt_args) + + try: + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Cannot fix broken dependencies. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + def list_installed(self): + return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() + + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s show %s" % (self.apt_cache_cmd, pkg) + pkg_info = super(DpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["pkgarch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + + return tmp_dir + +def generate_index_files(d): + classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split() + + indexer_map = { + "rpm": (RpmIndexer, d.getVar('DEPLOY_DIR_RPM')), + "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK')), + "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB')) + } + + result = None + + for pkg_class in classes: + if not pkg_class in indexer_map: + continue + + if os.path.exists(indexer_map[pkg_class][1]): + result = indexer_map[pkg_class][0](d, indexer_map[pkg_class][1]).write_index() + + if result is not None: + bb.fatal(result) + +if __name__ == "__main__": + """ + We should be able to run this as a standalone script, from outside bitbake + environment. + """ + """ + TBD + """ diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index 918e7ad2cc..32e5c82a94 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -1,6 +1,5 @@ -import os -import bb.data import codecs +import os def packaged(pkg, d): return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) @@ -9,12 +8,12 @@ def read_pkgdatafile(fn): pkgdata = {} def decode(str): - c = codecs.getdecoder("string_escape") + c = codecs.getdecoder("unicode_escape") return c(str)[0] if os.access(fn, os.R_OK): import re - f = file(fn, 'r') + f = open(fn, 'r') lines = f.readlines() f.close() r = re.compile("([^:]+):\s*(.*)") @@ -26,15 +25,7 @@ def read_pkgdatafile(fn): return pkgdata def get_subpkgedata_fn(pkg, d): - archs = bb.data.expand("${PACKAGE_ARCHS}", d).split(" ") - archs.reverse() - pkgdata = bb.data.expand('${TMPDIR}/pkgdata/', d) - targetdir = bb.data.expand('${TARGET_VENDOR}-${TARGET_OS}/runtime/', d) - for arch in archs: - fn = pkgdata + arch + targetdir + pkg - if os.path.exists(fn): - return fn - return bb.data.expand('${PKGDATA_DIR}/runtime/%s' % pkg, d) + return d.expand('${PKGDATA_DIR}/runtime/%s' % pkg) def has_subpkgdata(pkg, d): return os.access(get_subpkgedata_fn(pkg, d), os.R_OK) @@ -43,11 +34,11 @@ def read_subpkgdata(pkg, d): return read_pkgdatafile(get_subpkgedata_fn(pkg, d)) def has_pkgdata(pn, d): - fn = bb.data.expand('${PKGDATA_DIR}/%s' % pn, d) + fn = d.expand('${PKGDATA_DIR}/%s' % pn) return os.access(fn, os.R_OK) def read_pkgdata(pn, d): - fn = bb.data.expand('${PKGDATA_DIR}/%s' % pn, d) + fn = d.expand('${PKGDATA_DIR}/%s' % pn) return read_pkgdatafile(fn) # @@ -66,29 +57,24 @@ def read_subpkgdata_dict(pkg, d): def _pkgmap(d): """Return a dictionary mapping package to recipe name.""" - target_os = d.getVar("TARGET_OS", True) - target_vendor = d.getVar("TARGET_VENDOR", True) - basedir = os.path.dirname(d.getVar("PKGDATA_DIR", True)) - - dirs = ("%s%s-%s" % (arch, target_vendor, target_os) - for arch in d.getVar("PACKAGE_ARCHS", True).split()) + pkgdatadir = d.getVar("PKGDATA_DIR") pkgmap = {} - for pkgdatadir in (os.path.join(basedir, sys) for sys in dirs): + try: + files = os.listdir(pkgdatadir) + except OSError: + bb.warn("No files in %s?" % pkgdatadir) + files = [] + + for pn in [f for f in files if not os.path.isdir(os.path.join(pkgdatadir, f))]: try: - files = os.listdir(pkgdatadir) + pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn)) except OSError: continue - for pn in filter(lambda f: not os.path.isdir(os.path.join(pkgdatadir, f)), files): - try: - pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn)) - except OSError: - continue - - packages = pkgdata.get("PACKAGES") or "" - for pkg in packages.split(): - pkgmap[pkg] = pn + packages = pkgdata.get("PACKAGES") or "" + for pkg in packages.split(): + pkgmap[pkg] = pn return pkgmap diff --git a/meta/lib/oe/packagegroup.py b/meta/lib/oe/packagegroup.py index b04c45a1af..4bc5d3e4bb 100644 --- a/meta/lib/oe/packagegroup.py +++ b/meta/lib/oe/packagegroup.py @@ -1,19 +1,26 @@ import itertools -def is_optional(group, d): - return bool(d.getVarFlag("PACKAGE_GROUP_%s" % group, "optional")) +def is_optional(feature, d): + packages = d.getVar("FEATURE_PACKAGES_%s" % feature) + if packages: + return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional")) + else: + return bool(d.getVarFlag("PACKAGE_GROUP_%s" % feature, "optional")) -def packages(groups, d): - for group in groups: - for pkg in (d.getVar("PACKAGE_GROUP_%s" % group, True) or "").split(): +def packages(features, d): + for feature in features: + packages = d.getVar("FEATURE_PACKAGES_%s" % feature) + if not packages: + packages = d.getVar("PACKAGE_GROUP_%s" % feature) + for pkg in (packages or "").split(): yield pkg -def required_packages(groups, d): - req = filter(lambda group: not is_optional(group, d), groups) +def required_packages(features, d): + req = [feature for feature in features if not is_optional(feature, d)] return packages(req, d) -def optional_packages(groups, d): - opt = filter(lambda group: is_optional(group, d), groups) +def optional_packages(features, d): + opt = [feature for feature in features if is_optional(feature, d)] return packages(opt, d) def active_packages(features, d): diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py index f4ccb3e183..f1ab3dd809 100644 --- a/meta/lib/oe/patch.py +++ b/meta/lib/oe/patch.py @@ -1,25 +1,25 @@ import oe.path -import os -import bb.utils, bb.msg, bb.data, bb.fetch2 -class NotFoundError(Exception): +class NotFoundError(bb.BBHandledException): def __init__(self, path): self.path = path def __str__(self): return "Error: %s not found." % self.path -class CmdError(Exception): - def __init__(self, exitstatus, output): +class CmdError(bb.BBHandledException): + def __init__(self, command, exitstatus, output): + self.command = command self.status = exitstatus self.output = output def __str__(self): - return "Command Error: exit status: %d Output:\n%s" % (self.status, self.output) + return "Command Error: '%s' exited with %d Output:\n%s" % \ + (self.command, self.status, self.output) def runcmd(args, dir = None): - import commands + import pipes if dir: olddir = os.path.abspath(os.curdir) @@ -29,12 +29,12 @@ def runcmd(args, dir = None): # print("cwd: %s -> %s" % (olddir, dir)) try: - args = [ commands.mkarg(str(arg)) for arg in args ] + args = [ pipes.quote(str(arg)) for arg in args ] cmd = " ".join(args) # print("cmd: %s" % cmd) - (exitstatus, output) = commands.getstatusoutput(cmd) + (exitstatus, output) = oe.utils.getstatusoutput(cmd) if exitstatus != 0: - raise CmdError(exitstatus >> 8, output) + raise CmdError(cmd, exitstatus >> 8, output) return output finally: @@ -81,7 +81,7 @@ class PatchSet(object): patch[param] = PatchSet.defaults[param] if patch.get("remote"): - patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d) + patch["file"] = self.d.expand(bb.fetch2.localpath(patch["remote"], self.d)) patch["filemd5"] = bb.utils.md5_file(patch["file"]) @@ -94,11 +94,112 @@ class PatchSet(object): def Refresh(self, remote = None, all = None): raise NotImplementedError() + @staticmethod + def getPatchedFiles(patchfile, striplevel, srcdir=None): + """ + Read a patch file and determine which files it will modify. + Params: + patchfile: the patch file to read + striplevel: the strip level at which the patch is going to be applied + srcdir: optional path to join onto the patched file paths + Returns: + A list of tuples of file path and change mode ('A' for add, + 'D' for delete or 'M' for modify) + """ + + def patchedpath(patchline): + filepth = patchline.split()[1] + if filepth.endswith('/dev/null'): + return '/dev/null' + filesplit = filepth.split(os.sep) + if striplevel > len(filesplit): + bb.error('Patch %s has invalid strip level %d' % (patchfile, striplevel)) + return None + return os.sep.join(filesplit[striplevel:]) + + for encoding in ['utf-8', 'latin-1']: + try: + copiedmode = False + filelist = [] + with open(patchfile) as f: + for line in f: + if line.startswith('--- '): + patchpth = patchedpath(line) + if not patchpth: + break + if copiedmode: + addedfile = patchpth + else: + removedfile = patchpth + elif line.startswith('+++ '): + addedfile = patchedpath(line) + if not addedfile: + break + elif line.startswith('*** '): + copiedmode = True + removedfile = patchedpath(line) + if not removedfile: + break + else: + removedfile = None + addedfile = None + + if addedfile and removedfile: + if removedfile == '/dev/null': + mode = 'A' + elif addedfile == '/dev/null': + mode = 'D' + else: + mode = 'M' + if srcdir: + fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) + else: + fullpath = addedfile + filelist.append((fullpath, mode)) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to decode %s' % patchfile) + + return filelist + class PatchTree(PatchSet): def __init__(self, dir, d): PatchSet.__init__(self, dir, d) - + self.patchdir = os.path.join(self.dir, 'patches') + self.seriespath = os.path.join(self.dir, 'patches', 'series') + bb.utils.mkdirhier(self.patchdir) + + def _appendPatchFile(self, patch, strippath): + with open(self.seriespath, 'a') as f: + f.write(os.path.basename(patch) + "," + strippath + "\n") + shellcmd = ["cat", patch, ">" , self.patchdir + "/" + os.path.basename(patch)] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + + def _removePatch(self, p): + patch = {} + patch['file'] = p.split(",")[0] + patch['strippath'] = p.split(",")[1] + self._applypatch(patch, False, True) + + def _removePatchFile(self, all = False): + if not os.path.exists(self.seriespath): + return + with open(self.seriespath, 'r+') as f: + patches = f.readlines() + if all: + for p in reversed(patches): + self._removePatch(os.path.join(self.patchdir, p.strip())) + patches = [] + else: + self._removePatch(os.path.join(self.patchdir, patches[-1].strip())) + patches.pop() + with open(self.seriespath, 'w') as f: + for p in patches: + f.write(p) + def Import(self, patch, force = None): """""" PatchSet.Import(self, patch, force) @@ -120,13 +221,21 @@ class PatchTree(PatchSet): if not force: shellcmd.append('--dry-run') - output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + try: + output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - if force: - return + if force: + return + + shellcmd.pop(len(shellcmd) - 1) + output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + except CmdError as err: + raise bb.BBHandledException("Applying '%s' failed:\n%s" % + (os.path.basename(patch['file']), err.output)) + + if not reverse: + self._appendPatchFile(patch['file'], patch['strippath']) - shellcmd.pop(len(shellcmd) - 1) - output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) return output def Push(self, force = False, all = False, run = True): @@ -134,52 +243,315 @@ class PatchTree(PatchSet): bb.note("patches is %s" % self.patches) if all: for i in self.patches: - if self._current is not None: - self._current = self._current + 1 - else: - self._current = 0 bb.note("applying patch %s" % i) self._applypatch(i, force) + self._current = i else: if self._current is not None: - self._current = self._current + 1 + next = self._current + 1 else: - self._current = 0 - bb.note("applying patch %s" % self.patches[self._current]) - return self._applypatch(self.patches[self._current], force) + next = 0 + bb.note("applying patch %s" % self.patches[next]) + ret = self._applypatch(self.patches[next], force) + + self._current = next + return ret def Pop(self, force = None, all = None): if all: - for i in self.patches: - self._applypatch(i, force, True) + self._removePatchFile(True) + self._current = None else: - self._applypatch(self.patches[self._current], force, True) + self._removePatchFile(False) + + if self._current == 0: + self._current = None + + if self._current is not None: + self._current = self._current - 1 def Clean(self): """""" + self.Pop(all=True) class GitApplyTree(PatchTree): + patch_line_prefix = '%% original patch' + ignore_commit_prefix = '%% ignore' + def __init__(self, dir, d): PatchTree.__init__(self, dir, d) + self.commituser = d.getVar('PATCH_GIT_USER_NAME') + self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL') - def _applypatch(self, patch, force = False, reverse = False, run = True): - shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']] - - if reverse: - shellcmd.append('-R') - - shellcmd.append(patch['file']) - - if not run: - return "sh" + "-c" + " ".join(shellcmd) + @staticmethod + def extractPatchHeader(patchfile): + """ + Extract just the header lines from the top of a patch file + """ + for encoding in ['utf-8', 'latin-1']: + lines = [] + try: + with open(patchfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): + break + lines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % patchfile) + return lines + + @staticmethod + def decodeAuthor(line): + from email.header import decode_header + authorval = line.split(':', 1)[1].strip().replace('"', '') + result = decode_header(authorval)[0][0] + if hasattr(result, 'decode'): + result = result.decode('utf-8') + return result + + @staticmethod + def interpretPatchHeader(headerlines): + import re + author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') + from_commit_re = re.compile('^From [a-z0-9]{40} .*') + outlines = [] + author = None + date = None + subject = None + for line in headerlines: + if line.startswith('Subject: '): + subject = line.split(':', 1)[1] + # Remove any [PATCH][oe-core] etc. + subject = re.sub(r'\[.+?\]\s*', '', subject) + continue + elif line.startswith('From: ') or line.startswith('Author: '): + authorval = GitApplyTree.decodeAuthor(line) + # git is fussy about author formatting i.e. it must be Name <email@domain> + if author_re.match(authorval): + author = authorval + continue + elif line.startswith('Date: '): + if date is None: + dateval = line.split(':', 1)[1].strip() + # Very crude check for date format, since git will blow up if it's not in the right + # format. Without e.g. a python-dateutils dependency we can't do a whole lot more + if len(dateval) > 12: + date = dateval + continue + elif not author and line.lower().startswith('signed-off-by: '): + authorval = GitApplyTree.decodeAuthor(line) + # git is fussy about author formatting i.e. it must be Name <email@domain> + if author_re.match(authorval): + author = authorval + elif from_commit_re.match(line): + # We don't want the From <commit> line - if it's present it will break rebasing + continue + outlines.append(line) + + if not subject: + firstline = None + for line in headerlines: + line = line.strip() + if firstline: + if line: + # Second line is not blank, the first line probably isn't usable + firstline = None + break + elif line: + firstline = line + if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100: + subject = firstline + + return outlines, author, date, subject + + @staticmethod + def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None): + if d: + commituser = d.getVar('PATCH_GIT_USER_NAME') + commitemail = d.getVar('PATCH_GIT_USER_EMAIL') + if commituser: + cmd += ['-c', 'user.name="%s"' % commituser] + if commitemail: + cmd += ['-c', 'user.email="%s"' % commitemail] + + @staticmethod + def prepareCommit(patchfile, commituser=None, commitemail=None): + """ + Prepare a git commit command line based on the header from a patch file + (typically this is useful for patches that cannot be applied with "git am" due to formatting) + """ + import tempfile + # Process patch header and extract useful information + lines = GitApplyTree.extractPatchHeader(patchfile) + outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) + if not author or not subject or not date: + try: + shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile] + out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) + except CmdError: + out = None + if out: + _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) + if not author: + # If we're setting the author then the date should be set as well + author = newauthor + date = newdate + elif not date: + # If we don't do this we'll get the current date, at least this will be closer + date = newdate + if not subject: + subject = newsubject + if subject and outlines and not outlines[0].strip() == subject: + outlines.insert(0, '%s\n\n' % subject.strip()) + + # Write out commit message to a file + with tempfile.NamedTemporaryFile('w', delete=False) as tf: + tmpfile = tf.name + for line in outlines: + tf.write(line) + # Prepare git command + cmd = ["git"] + GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) + cmd += ["commit", "-F", tmpfile] + # git doesn't like plain email addresses as authors + if author and '<' in author: + cmd.append('--author="%s"' % author) + if date: + cmd.append('--date="%s"' % date) + return (tmpfile, cmd) + + @staticmethod + def extractPatches(tree, startcommit, outdir, paths=None): + import tempfile + import shutil + import re + tempdir = tempfile.mkdtemp(prefix='oepatch') + try: + shellcmd = ["git", "format-patch", startcommit, "-o", tempdir] + if paths: + shellcmd.append('--') + shellcmd.extend(paths) + out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) + if out: + for srcfile in out.split(): + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + outfile = None + try: + with open(srcfile, 'r', encoding=encoding) as f: + for line in f: + checkline = line + if checkline.startswith('Subject: '): + checkline = re.sub(r'\[.+?\]\s*', '', checkline[9:]) + if checkline.startswith(GitApplyTree.patch_line_prefix): + outfile = line.split()[-1].strip() + continue + if checkline.startswith(GitApplyTree.ignore_commit_prefix): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + + if not outfile: + outfile = os.path.basename(srcfile) + with open(os.path.join(outdir, outfile), 'w') as of: + for line in patchlines: + of.write(line) + finally: + shutil.rmtree(tempdir) - return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + def _applypatch(self, patch, force = False, reverse = False, run = True): + import shutil + + def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True): + if reverse: + shellcmd.append('-R') + + shellcmd.append(patch['file']) + + if not run: + return "sh" + "-c" + " ".join(shellcmd) + + return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + + # Add hooks which add a pointer to the original patch file name in the commit message + reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip() + if not reporoot: + raise Exception("Cannot get repository root for directory %s" % self.dir) + hooks_dir = os.path.join(reporoot, '.git', 'hooks') + hooks_dir_backup = hooks_dir + '.devtool-orig' + if os.path.lexists(hooks_dir_backup): + raise Exception("Git hooks backup directory already exists: %s" % hooks_dir_backup) + if os.path.lexists(hooks_dir): + shutil.move(hooks_dir, hooks_dir_backup) + os.mkdir(hooks_dir) + commithook = os.path.join(hooks_dir, 'commit-msg') + applyhook = os.path.join(hooks_dir, 'applypatch-msg') + with open(commithook, 'w') as f: + # NOTE: the formatting here is significant; if you change it you'll also need to + # change other places which read it back + f.write('echo >> $1\n') + f.write('echo "%s: $PATCHFILE" >> $1\n' % GitApplyTree.patch_line_prefix) + os.chmod(commithook, 0o755) + shutil.copy2(commithook, applyhook) + try: + patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) + try: + shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] + self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) + shellcmd += ["am", "-3", "--keep-cr", "-p%s" % patch['strippath']] + return _applypatchhelper(shellcmd, patch, force, reverse, run) + except CmdError: + # Need to abort the git am, or we'll still be within it at the end + try: + shellcmd = ["git", "--work-tree=%s" % reporoot, "am", "--abort"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + except CmdError: + pass + # git am won't always clean up after itself, sadly, so... + shellcmd = ["git", "--work-tree=%s" % reporoot, "reset", "--hard", "HEAD"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Also need to take care of any stray untracked files + shellcmd = ["git", "--work-tree=%s" % reporoot, "clean", "-f"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + + # Fall back to git apply + shellcmd = ["git", "--git-dir=%s" % reporoot, "apply", "-p%s" % patch['strippath']] + try: + output = _applypatchhelper(shellcmd, patch, force, reverse, run) + except CmdError: + # Fall back to patch + output = PatchTree._applypatch(self, patch, force, reverse, run) + # Add all files + shellcmd = ["git", "add", "-f", "-A", "."] + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Exclude the patches directory + shellcmd = ["git", "reset", "HEAD", self.patchdir] + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Commit the result + (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) + try: + shellcmd.insert(0, patchfilevar) + output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + finally: + os.remove(tmpfile) + return output + finally: + shutil.rmtree(hooks_dir) + if os.path.lexists(hooks_dir_backup): + shutil.move(hooks_dir_backup, hooks_dir) class QuiltTree(PatchSet): def _runcmd(self, args, run = True): - quiltrc = self.d.getVar('QUILTRCFILE', 1) + quiltrc = self.d.getVar('QUILTRCFILE') if not run: return ["quilt"] + ["--quiltrc"] + [quiltrc] + args runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir) @@ -207,18 +579,17 @@ class QuiltTree(PatchSet): # read series -> self.patches seriespath = os.path.join(self.dir, 'patches', 'series') if not os.path.exists(self.dir): - raise Exception("Error: %s does not exist." % self.dir) + raise NotFoundError(self.dir) if os.path.exists(seriespath): - series = file(seriespath, 'r') - for line in series.readlines(): - patch = {} - parts = line.strip().split() - patch["quiltfile"] = self._quiltpatchpath(parts[0]) - patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) - if len(parts) > 1: - patch["strippath"] = parts[1][2:] - self.patches.append(patch) - series.close() + with open(seriespath, 'r') as f: + for line in f.readlines(): + patch = {} + parts = line.strip().split() + patch["quiltfile"] = self._quiltpatchpath(parts[0]) + patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) + if len(parts) > 1: + patch["strippath"] = parts[1][2:] + self.patches.append(patch) # determine which patches are applied -> self._current try: @@ -228,7 +599,7 @@ class QuiltTree(PatchSet): if sys.exc_value.output.strip() == "No patches applied": return else: - raise sys.exc_value + raise output = [val for val in output.split('\n') if not val.startswith('#')] for patch in self.patches: if os.path.basename(patch["quiltfile"]) == output[-1]: @@ -239,10 +610,9 @@ class QuiltTree(PatchSet): if not self.initialized: self.InitFromDir() PatchSet.Import(self, patch, force) - oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"])) - f = open(os.path.join(self.dir, "patches","series"), "a"); - f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"]+"\n") - f.close() + oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"]), force=True) + with open(os.path.join(self.dir, "patches", "series"), "a") as f: + f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"] + "\n") patch["quiltfile"] = self._quiltpatchpath(patch["file"]) patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) @@ -292,7 +662,7 @@ class QuiltTree(PatchSet): patch = self.patches[kwargs["patch"]] if not patch: raise PatchError("No patch found at index %s in patchset." % kwargs["patch"]) - (type, host, path, user, pswd, parm) = bb.decodeurl(patch["remote"]) + (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(patch["remote"]) if type == "file": import shutil if not patch.get("file") and patch.get("remote"): @@ -336,7 +706,7 @@ class NOOPResolver(Resolver): except Exception: import sys os.chdir(olddir) - raise sys.exc_value + raise # Patch resolver which relies on the user doing all the work involved in the # resolution, with the exception of refreshing the remote copy of the patch @@ -353,24 +723,23 @@ class UserResolver(Resolver): os.chdir(self.patchset.dir) try: self.patchset.Push(False) - except CmdError, v: + except CmdError as v: # Patch application failed patchcmd = self.patchset.Push(True, False, False) - t = self.patchset.d.getVar('T', 1) + t = self.patchset.d.getVar('T') if not t: bb.msg.fatal("Build", "T not set") bb.utils.mkdirhier(t) import random rcfile = "%s/bashrc.%s.%s" % (t, str(os.getpid()), random.random()) - f = open(rcfile, "w") - f.write("echo '*** Manual patch resolution mode ***'\n") - f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n") - f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") - f.write("echo ''\n") - f.write(" ".join(patchcmd) + "\n") - f.close() - os.chmod(rcfile, 0775) + with open(rcfile, "w") as f: + f.write("echo '*** Manual patch resolution mode ***'\n") + f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n") + f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") + f.write("echo ''\n") + f.write(" ".join(patchcmd) + "\n") + os.chmod(rcfile, 0o775) self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d) @@ -400,3 +769,110 @@ class UserResolver(Resolver): os.chdir(olddir) raise os.chdir(olddir) + + +def patch_path(url, fetch, workdir, expand=True): + """Return the local path of a patch, or None if this isn't a patch""" + + local = fetch.localpath(url) + base, ext = os.path.splitext(os.path.basename(local)) + if ext in ('.gz', '.bz2', '.xz', '.Z'): + if expand: + local = os.path.join(workdir, base) + ext = os.path.splitext(base)[1] + + urldata = fetch.ud[url] + if "apply" in urldata.parm: + apply = oe.types.boolean(urldata.parm["apply"]) + if not apply: + return + elif ext not in (".diff", ".patch"): + return + + return local + +def src_patches(d, all=False, expand=True): + workdir = d.getVar('WORKDIR') + fetch = bb.fetch2.Fetch([], d) + patches = [] + sources = [] + for url in fetch.urls: + local = patch_path(url, fetch, workdir, expand) + if not local: + if all: + local = fetch.localpath(url) + sources.append(local) + continue + + urldata = fetch.ud[url] + parm = urldata.parm + patchname = parm.get('pname') or os.path.basename(local) + + apply, reason = should_apply(parm, d) + if not apply: + if reason: + bb.note("Patch %s %s" % (patchname, reason)) + continue + + patchparm = {'patchname': patchname} + if "striplevel" in parm: + striplevel = parm["striplevel"] + elif "pnum" in parm: + #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url) + striplevel = parm["pnum"] + else: + striplevel = '1' + patchparm['striplevel'] = striplevel + + patchdir = parm.get('patchdir') + if patchdir: + patchparm['patchdir'] = patchdir + + localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm)) + patches.append(localurl) + + if all: + return sources + + return patches + + +def should_apply(parm, d): + if "mindate" in parm or "maxdate" in parm: + pn = d.getVar('PN') + srcdate = d.getVar('SRCDATE_%s' % pn) + if not srcdate: + srcdate = d.getVar('SRCDATE') + + if srcdate == "now": + srcdate = d.getVar('DATE') + + if "maxdate" in parm and parm["maxdate"] < srcdate: + return False, 'is outdated' + + if "mindate" in parm and parm["mindate"] > srcdate: + return False, 'is predated' + + + if "minrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and srcrev < parm["minrev"]: + return False, 'applies to later revisions' + + if "maxrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and srcrev > parm["maxrev"]: + return False, 'applies to earlier revisions' + + if "rev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and parm["rev"] not in srcrev: + return False, "doesn't apply to revision" + + if "notrev" in parm: + srcrev = d.getVar('SRCREV') + if srcrev and parm["notrev"] in srcrev: + return False, "doesn't apply to revision" + + return True, None + diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py index 8eaa3c5da4..448a2b944e 100644 --- a/meta/lib/oe/path.py +++ b/meta/lib/oe/path.py @@ -1,9 +1,8 @@ -import bb import errno import glob -import os import shutil import subprocess +import os.path def join(*paths): """Like os.path.join but doesn't treat absolute RHS specially""" @@ -22,27 +21,59 @@ def relative(src, dest): foo/bar """ - if hasattr(os.path, "relpath"): - return os.path.relpath(dest, src) - else: - destlist = os.path.normpath(dest).split(os.path.sep) - srclist = os.path.normpath(src).split(os.path.sep) + return os.path.relpath(dest, src) + +def make_relative_symlink(path): + """ Convert an absolute symlink to a relative one """ + if not os.path.islink(path): + return + link = os.readlink(path) + if not os.path.isabs(link): + return + + # find the common ancestor directory + ancestor = path + depth = 0 + while ancestor and not link.startswith(ancestor): + ancestor = ancestor.rpartition('/')[0] + depth += 1 - # Find common section of the path - common = os.path.commonprefix([destlist, srclist]) - commonlen = len(common) + if not ancestor: + print("make_relative_symlink() Error: unable to find the common ancestor of %s and its target" % path) + return - # Climb back to the point where they differentiate - relpath = [ os.path.pardir ] * (len(srclist) - commonlen) - if commonlen < len(destlist): - # Add remaining portion - relpath += destlist[commonlen:] + base = link.partition(ancestor)[2].strip('/') + while depth > 1: + base = "../" + base + depth -= 1 - return os.path.sep.join(relpath) + os.remove(path) + os.symlink(base, path) + +def replace_absolute_symlinks(basedir, d): + """ + Walk basedir looking for absolute symlinks and replacing them with relative ones. + The absolute links are assumed to be relative to basedir + (compared to make_relative_symlink above which tries to compute common ancestors + using pattern matching instead) + """ + for walkroot, dirs, files in os.walk(basedir): + for file in files + dirs: + path = os.path.join(walkroot, file) + if not os.path.islink(path): + continue + link = os.readlink(path) + if not os.path.isabs(link): + continue + walkdir = os.path.dirname(path.rpartition(basedir)[2]) + base = os.path.relpath(link, walkdir) + bb.debug(2, "Replacing absolute path %s with relative path %s" % (link, base)) + os.remove(path) + os.symlink(base, path) def format_display(path, metadata): """ Prepare a path for display to the user. """ - rel = relative(metadata.getVar("TOPDIR", 1), path) + rel = relative(metadata.getVar("TOPDIR"), path) if len(rel) > len(path): return path else: @@ -55,16 +86,47 @@ def copytree(src, dst): # This way we also preserve hardlinks between files in the tree. bb.utils.mkdirhier(dst) - cmd = 'tar -cf - -C %s -ps . | tar -xf - -C %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) +def copyhardlinktree(src, dst): + """ Make the hard link when possible, otherwise copy. """ + bb.utils.mkdirhier(dst) + if os.path.isdir(src) and not len(os.listdir(src)): + return + + if (os.stat(src).st_dev == os.stat(dst).st_dev): + # Need to copy directories only with tar first since cp will error if two + # writers try and create a directory at the same time + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + source = '' + if os.path.isdir(src): + if len(glob.glob('%s/.??*' % src)) > 0: + source = './.??* ' + source += './*' + s_dir = src + else: + source = src + s_dir = os.getcwd() + cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst)) + subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) + else: + copytree(src, dst) def remove(path, recurse=True): - """Equivalent to rm -f or rm -rf""" + """ + Equivalent to rm -f or rm -rf + NOTE: be careful about passing paths that may contain filenames with + wildcards in them (as opposed to passing an actual wildcarded path) - + since we use glob.glob() to expand the path. Filenames containing + square brackets are particularly problematic since the they may not + actually expand to match the original filename. + """ for name in glob.glob(path): try: os.unlink(name) - except OSError, exc: + except OSError as exc: if recurse and exc.errno == errno.EISDIR: shutil.rmtree(name) elif exc.errno != errno.ENOENT: @@ -76,55 +138,102 @@ def symlink(source, destination, force=False): if force: remove(destination) os.symlink(source, destination) - except OSError, e: + except OSError as e: if e.errno != errno.EEXIST or os.readlink(destination) != source: raise -class CalledProcessError(Exception): - def __init__(self, retcode, cmd, output = None): - self.retcode = retcode - self.cmd = cmd - self.output = output - def __str__(self): - return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output) +def find(dir, **walkoptions): + """ Given a directory, recurses into that directory, + returning all files as absolute paths. """ -# Not needed when we move to python 2.7 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. + for root, dirs, files in os.walk(dir, **walkoptions): + for file in files: + yield os.path.join(root, file) - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - The arguments are the same as for the Popen constructor. Example: +## realpath() related functions +def __is_path_below(file, root): + return (file + os.path.sep).startswith(root) - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' +def __realpath_rel(start, rel_path, root, loop_cnt, assume_dir): + """Calculates real path of symlink 'start' + 'rel_path' below + 'root'; no part of 'start' below 'root' must contain symlinks. """ + have_dir = True - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. + for d in rel_path.split(os.path.sep): + if not have_dir and not assume_dir: + raise OSError(errno.ENOENT, "no such directory %s" % start) - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output + if d == os.path.pardir: # '..' + if len(start) >= len(root): + # do not follow '..' before root + start = os.path.dirname(start) + else: + # emit warning? + pass + else: + (start, have_dir) = __realpath(os.path.join(start, d), + root, loop_cnt, assume_dir) -def find(dir, **walkoptions): - """ Given a directory, recurses into that directory, - returning all files as absolute paths. """ + assert(__is_path_below(start, root)) - for root, dirs, files in os.walk(dir, **walkoptions): - for file in files: - yield os.path.join(root, file) + return start + +def __realpath(file, root, loop_cnt, assume_dir): + while os.path.islink(file) and len(file) >= len(root): + if loop_cnt == 0: + raise OSError(errno.ELOOP, file) + + loop_cnt -= 1 + target = os.path.normpath(os.readlink(file)) + + if not os.path.isabs(target): + tdir = os.path.dirname(file) + assert(__is_path_below(tdir, root)) + else: + tdir = root + + file = __realpath_rel(tdir, target, root, loop_cnt, assume_dir) + + try: + is_dir = os.path.isdir(file) + except: + is_dir = false + + return (file, is_dir) + +def realpath(file, root, use_physdir = True, loop_cnt = 100, assume_dir = False): + """ Returns the canonical path of 'file' with assuming a + toplevel 'root' directory. When 'use_physdir' is set, all + preceding path components of 'file' will be resolved first; + this flag should be set unless it is guaranteed that there is + no symlink in the path. When 'assume_dir' is not set, missing + path components will raise an ENOENT error""" + + root = os.path.normpath(root) + file = os.path.normpath(file) + + if not root.endswith(os.path.sep): + # letting root end with '/' makes some things easier + root = root + os.path.sep + + if not __is_path_below(file, root): + raise OSError(errno.EINVAL, "file '%s' is not below root" % file) + + try: + if use_physdir: + file = __realpath_rel(root, file[(len(root) - 1):], root, loop_cnt, assume_dir) + else: + file = __realpath(file, root, loop_cnt, assume_dir)[0] + except OSError as e: + if e.errno == errno.ELOOP: + # make ELOOP more readable; without catching it, there will + # be printed a backtrace with 100s of OSError exceptions + # else + raise OSError(errno.ELOOP, + "too much recursions while resolving '%s'; loop in '%s'" % + (file, e.strerror)) + + raise + + return file diff --git a/meta/lib/oe/process.py b/meta/lib/oe/process.py deleted file mode 100644 index 26c3e65910..0000000000 --- a/meta/lib/oe/process.py +++ /dev/null @@ -1,74 +0,0 @@ -import subprocess -import signal - -def subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - -class CmdError(RuntimeError): - def __init__(self, command): - self.command = command - - def __str__(self): - if not isinstance(self.command, basestring): - cmd = subprocess.list2cmdline(self.command) - else: - cmd = self.command - - return "Execution of '%s' failed" % cmd - -class NotFoundError(CmdError): - def __str__(self): - return CmdError.__str__(self) + ": command not found" - -class ExecutionError(CmdError): - def __init__(self, command, exitcode, stdout = None, stderr = None): - CmdError.__init__(self, command) - self.exitcode = exitcode - self.stdout = stdout - self.stderr = stderr - - def __str__(self): - message = "" - if self.stderr: - message += self.stderr - if self.stdout: - message += self.stdout - if message: - message = ":\n" + message - return (CmdError.__str__(self) + - " with exit code %s" % self.exitcode + message) - -class Popen(subprocess.Popen): - defaults = { - "close_fds": True, - "preexec_fn": subprocess_setup, - "stdout": subprocess.PIPE, - "stderr": subprocess.STDOUT, - "stdin": subprocess.PIPE, - "shell": False, - } - - def __init__(self, *args, **kwargs): - options = dict(self.defaults) - options.update(kwargs) - subprocess.Popen.__init__(self, *args, **options) - -def run(cmd, input=None, **options): - """Convenience function to run a command and return its output, raising an - exception when the command fails""" - - if isinstance(cmd, basestring) and not "shell" in options: - options["shell"] = True - try: - pipe = Popen(cmd, **options) - except OSError, exc: - if exc.errno == 2: - raise NotFoundError(cmd) - else: - raise - stdout, stderr = pipe.communicate(input) - if pipe.returncode != 0: - raise ExecutionError(cmd, pipe.returncode, stdout, stderr) - return stdout diff --git a/meta/lib/oe/prservice.py b/meta/lib/oe/prservice.py new file mode 100644 index 0000000000..32dfc15e88 --- /dev/null +++ b/meta/lib/oe/prservice.py @@ -0,0 +1,126 @@ + +def prserv_make_conn(d, check = False): + import prserv.serv + host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) + try: + conn = None + conn = prserv.serv.PRServerConnection(host_params[0], int(host_params[1])) + if check: + if not conn.ping(): + raise Exception('service not available') + d.setVar("__PRSERV_CONN",conn) + except Exception as exc: + bb.fatal("Connecting to PR service %s:%s failed: %s" % (host_params[0], host_params[1], str(exc))) + + return conn + +def prserv_dump_db(d): + if not d.getVar('PRSERV_HOST'): + bb.error("Not using network based PR service") + return None + + conn = d.getVar("__PRSERV_CONN") + if conn is None: + conn = prserv_make_conn(d) + if conn is None: + bb.error("Making connection failed to remote PR service") + return None + + #dump db + opt_version = d.getVar('PRSERV_DUMPOPT_VERSION') + opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH') + opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM') + opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL')) + return conn.export(opt_version, opt_pkgarch, opt_checksum, opt_col) + +def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None): + if not d.getVar('PRSERV_HOST'): + bb.error("Not using network based PR service") + return None + + conn = d.getVar("__PRSERV_CONN") + if conn is None: + conn = prserv_make_conn(d) + if conn is None: + bb.error("Making connection failed to remote PR service") + return None + #get the entry values + imported = [] + prefix = "PRAUTO$" + for v in d.keys(): + if v.startswith(prefix): + (remain, sep, checksum) = v.rpartition('$') + (remain, sep, pkgarch) = remain.rpartition('$') + (remain, sep, version) = remain.rpartition('$') + if (remain + '$' != prefix) or \ + (filter_version and filter_version != version) or \ + (filter_pkgarch and filter_pkgarch != pkgarch) or \ + (filter_checksum and filter_checksum != checksum): + continue + try: + value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum)) + except BaseException as exc: + bb.debug("Not valid value of %s:%s" % (v,str(exc))) + continue + ret = conn.importone(version,pkgarch,checksum,value) + if ret != value: + bb.error("importing(%s,%s,%s,%d) failed. DB may have larger value %d" % (version,pkgarch,checksum,value,ret)) + else: + imported.append((version,pkgarch,checksum,value)) + return imported + +def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): + import bb.utils + #initilize the output file + bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR')) + df = d.getVar('PRSERV_DUMPFILE') + #write data + lf = bb.utils.lockfile("%s.lock" % df) + f = open(df, "a") + if metainfo: + #dump column info + f.write("#PR_core_ver = \"%s\"\n\n" % metainfo['core_ver']); + f.write("#Table: %s\n" % metainfo['tbl_name']) + f.write("#Columns:\n") + f.write("#name \t type \t notn \t dflt \t pk\n") + f.write("#----------\t --------\t --------\t --------\t ----\n") + for i in range(len(metainfo['col_info'])): + f.write("#%10s\t %8s\t %8s\t %8s\t %4s\n" % + (metainfo['col_info'][i]['name'], + metainfo['col_info'][i]['type'], + metainfo['col_info'][i]['notnull'], + metainfo['col_info'][i]['dflt_value'], + metainfo['col_info'][i]['pk'])) + f.write("\n") + + if lockdown: + f.write("PRSERV_LOCKDOWN = \"1\"\n\n") + + if datainfo: + idx = {} + for i in range(len(datainfo)): + pkgarch = datainfo[i]['pkgarch'] + value = datainfo[i]['value'] + if pkgarch not in idx: + idx[pkgarch] = i + elif value > datainfo[idx[pkgarch]]['value']: + idx[pkgarch] = i + f.write("PRAUTO$%s$%s$%s = \"%s\"\n" % + (str(datainfo[i]['version']), pkgarch, str(datainfo[i]['checksum']), str(value))) + if not nomax: + for i in idx: + f.write("PRAUTO_%s_%s = \"%s\"\n" % (str(datainfo[idx[i]]['version']),str(datainfo[idx[i]]['pkgarch']),str(datainfo[idx[i]]['value']))) + f.close() + bb.utils.unlockfile(lf) + +def prserv_check_avail(d): + host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f]) + try: + if len(host_params) != 2: + raise TypeError + else: + int(host_params[1]) + except TypeError: + bb.fatal('Undefined/incorrect PRSERV_HOST value. Format: "host:port"') + else: + prserv_make_conn(d, True) diff --git a/meta/lib/oe/qa.py b/meta/lib/oe/qa.py index d3800128ed..3231e60cea 100644 --- a/meta/lib/oe/qa.py +++ b/meta/lib/oe/qa.py @@ -1,3 +1,8 @@ +import os, struct, mmap + +class NotELFFileError(Exception): + pass + class ELFFile: EI_NIDENT = 16 @@ -7,6 +12,8 @@ class ELFFile: EI_OSABI = 7 EI_ABIVERSION = 8 + E_MACHINE = 0x12 + # possible values for EI_CLASS ELFCLASSNONE = 0 ELFCLASS32 = 1 @@ -16,74 +23,149 @@ class ELFFile: EV_CURRENT = 1 # possible values for EI_DATA - ELFDATANONE = 0 - ELFDATA2LSB = 1 - ELFDATA2MSB = 2 + EI_DATA_NONE = 0 + EI_DATA_LSB = 1 + EI_DATA_MSB = 2 + + PT_INTERP = 3 def my_assert(self, expectation, result): if not expectation == result: #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name) - raise Exception("This does not work as expected") + raise NotELFFileError("%s is not an ELF" % self.name) - def __init__(self, name, bits = 0): + def __init__(self, name): self.name = name - self.bits = bits + self.objdump_output = {} + + # Context Manager functions to close the mmap explicitly + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.data.close() def open(self): - self.file = file(self.name, "r") - self.data = self.file.read(ELFFile.EI_NIDENT+4) - - self.my_assert(len(self.data), ELFFile.EI_NIDENT+4) - self.my_assert(self.data[0], chr(0x7f) ) - self.my_assert(self.data[1], 'E') - self.my_assert(self.data[2], 'L') - self.my_assert(self.data[3], 'F') - if self.bits == 0: - if self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS32): - self.bits = 32 - elif self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS64): - self.bits = 64 - else: - # Not 32-bit or 64.. lets assert - raise Exception("ELF but not 32 or 64 bit.") - elif self.bits == 32: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS32)) - elif self.bits == 64: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS64)) - else: - raise Exception("Must specify unknown, 32 or 64 bit size.") - self.my_assert(self.data[ELFFile.EI_VERSION], chr(ELFFile.EV_CURRENT) ) - - self.sex = self.data[ELFFile.EI_DATA] - if self.sex == chr(ELFFile.ELFDATANONE): - raise Exception("self.sex == ELFDATANONE") - elif self.sex == chr(ELFFile.ELFDATA2LSB): - self.sex = "<" - elif self.sex == chr(ELFFile.ELFDATA2MSB): - self.sex = ">" + with open(self.name, "rb") as f: + try: + self.data = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + except ValueError: + # This means the file is empty + raise NotELFFileError("%s is empty" % self.name) + + # Check the file has the minimum number of ELF table entries + if len(self.data) < ELFFile.EI_NIDENT + 4: + raise NotELFFileError("%s is not an ELF" % self.name) + + # ELF header + self.my_assert(self.data[0], 0x7f) + self.my_assert(self.data[1], ord('E')) + self.my_assert(self.data[2], ord('L')) + self.my_assert(self.data[3], ord('F')) + if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: + self.bits = 32 + elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: + self.bits = 64 else: - raise Exception("Unknown self.sex") + # Not 32-bit or 64.. lets assert + raise NotELFFileError("ELF but not 32 or 64 bit.") + self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT) + + self.endian = self.data[ELFFile.EI_DATA] + if self.endian not in (ELFFile.EI_DATA_LSB, ELFFile.EI_DATA_MSB): + raise NotELFFileError("Unexpected EI_DATA %x" % self.endian) def osAbi(self): - return ord(self.data[ELFFile.EI_OSABI]) + return self.data[ELFFile.EI_OSABI] def abiVersion(self): - return ord(self.data[ELFFile.EI_ABIVERSION]) + return self.data[ELFFile.EI_ABIVERSION] def abiSize(self): - return self.bits + return self.bits def isLittleEndian(self): - return self.sex == "<" + return self.endian == ELFFile.EI_DATA_LSB + + def isBigEndian(self): + return self.endian == ELFFile.EI_DATA_MSB + + def getStructEndian(self): + return {ELFFile.EI_DATA_LSB: "<", + ELFFile.EI_DATA_MSB: ">"}[self.endian] + + def getShort(self, offset): + return struct.unpack_from(self.getStructEndian() + "H", self.data, offset)[0] - def isBigEngian(self): - return self.sex == ">" + def getWord(self, offset): + return struct.unpack_from(self.getStructEndian() + "i", self.data, offset)[0] + + def isDynamic(self): + """ + Return True if there is a .interp segment (therefore dynamically + linked), otherwise False (statically linked). + """ + offset = self.getWord(self.bits == 32 and 0x1C or 0x20) + size = self.getShort(self.bits == 32 and 0x2A or 0x36) + count = self.getShort(self.bits == 32 and 0x2C or 0x38) + + for i in range(0, count): + p_type = self.getWord(offset + i * size) + if p_type == ELFFile.PT_INTERP: + return True + return False def machine(self): """ - We know the sex stored in self.sex and we + We know the endian stored in self.endian and we know the position """ - import struct - (a,) = struct.unpack(self.sex+"H", self.data[18:20]) - return a + return self.getShort(ELFFile.E_MACHINE) + + def run_objdump(self, cmd, d): + import bb.process + import sys + + if cmd in self.objdump_output: + return self.objdump_output[cmd] + + objdump = d.getVar('OBJDUMP') + + env = os.environ.copy() + env["LC_ALL"] = "C" + env["PATH"] = d.getVar('PATH') + + try: + bb.note("%s %s %s" % (objdump, cmd, self.name)) + self.objdump_output[cmd] = bb.process.run([objdump, cmd, self.name], env=env, shell=False)[0] + return self.objdump_output[cmd] + except Exception as e: + bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e)) + return "" + +def elf_machine_to_string(machine): + """ + Return the name of a given ELF e_machine field or the hex value as a string + if it isn't recognised. + """ + try: + return { + 0x02: "SPARC", + 0x03: "x86", + 0x08: "MIPS", + 0x14: "PowerPC", + 0x28: "ARM", + 0x2A: "SuperH", + 0x32: "IA-64", + 0x3E: "x86-64", + 0xB7: "AArch64" + }[machine] + except: + return "Unknown (%s)" % repr(machine) + +if __name__ == "__main__": + import sys + + with ELFFile(sys.argv[1]) as elf: + elf.open() + print(elf.isDynamic()) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py new file mode 100644 index 0000000000..a7fdd36e40 --- /dev/null +++ b/meta/lib/oe/recipeutils.py @@ -0,0 +1,918 @@ +# Utility functions for reading and modifying recipes +# +# Some code borrowed from the OE layer index +# +# Copyright (C) 2013-2016 Intel Corporation +# + +import sys +import os +import os.path +import tempfile +import textwrap +import difflib +from . import utils +import shutil +import re +import fnmatch +import glob +from collections import OrderedDict, defaultdict + + +# Help us to find places to insert values +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()'] +# Variables that sometimes are a bit long but shouldn't be wrapped +nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]'] +list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] +meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] + + +def pn_to_recipe(cooker, pn, mc=''): + """Convert a recipe name (PN) to the path to the recipe file""" + + best = cooker.findBestProvider(pn, mc) + return best[3] + + +def get_unavailable_reasons(cooker, pn): + """If a recipe could not be found, find out why if possible""" + import bb.taskdata + taskdata = bb.taskdata.TaskData(None, skiplist=cooker.skiplist) + return taskdata.get_reasons(pn) + + +def parse_recipe(cooker, fn, appendfiles): + """ + Parse an individual recipe file, optionally with a list of + bbappend files. + """ + import bb.cache + parser = bb.cache.NoCache(cooker.databuilder) + envdata = parser.loadDataFull(fn, appendfiles) + return envdata + + +def get_var_files(fn, varlist, d): + """Find the file in which each of a list of variables is set. + Note: requires variable history to be enabled when parsing. + """ + varfiles = {} + for v in varlist: + history = d.varhistory.variable(v) + files = [] + for event in history: + if 'file' in event and not 'flag' in event: + files.append(event['file']) + if files: + actualfile = files[-1] + else: + actualfile = None + varfiles[v] = actualfile + + return varfiles + + +def split_var_value(value, assignment=True): + """ + Split a space-separated variable's value into a list of items, + taking into account that some of the items might be made up of + expressions containing spaces that should not be split. + Parameters: + value: + The string value to split + assignment: + True to assume that the value represents an assignment + statement, False otherwise. If True, and an assignment + statement is passed in the first item in + the returned list will be the part of the assignment + statement up to and including the opening quote character, + and the last item will be the closing quote. + """ + inexpr = 0 + lastchar = None + out = [] + buf = '' + for char in value: + if char == '{': + if lastchar == '$': + inexpr += 1 + elif char == '}': + inexpr -= 1 + elif assignment and char in '"\'' and inexpr == 0: + if buf: + out.append(buf) + out.append(char) + char = '' + buf = '' + elif char.isspace() and inexpr == 0: + char = '' + if buf: + out.append(buf) + buf = '' + buf += char + lastchar = char + if buf: + out.append(buf) + + # Join together assignment statement and opening quote + outlist = out + if assignment: + assigfound = False + for idx, item in enumerate(out): + if '=' in item: + assigfound = True + if assigfound: + if '"' in item or "'" in item: + outlist = [' '.join(out[:idx+1])] + outlist.extend(out[idx+1:]) + break + return outlist + + +def patch_recipe_lines(fromlines, values, trailing_newline=True): + """Update or insert variable values into lines from a recipe. + Note that some manual inspection/intervention may be required + since this cannot handle all situations. + """ + + import bb.utils + + if trailing_newline: + newline = '\n' + else: + newline = '' + + recipe_progression_res = [] + recipe_progression_restrs = [] + for item in recipe_progression: + if item.endswith('()'): + key = item[:-2] + else: + key = item + restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key + if item.endswith('()'): + recipe_progression_restrs.append(restr + '()') + else: + recipe_progression_restrs.append(restr) + recipe_progression_res.append(re.compile('^%s$' % restr)) + + def get_recipe_pos(variable): + for i, p in enumerate(recipe_progression_res): + if p.match(variable): + return i + return -1 + + remainingnames = {} + for k in values.keys(): + remainingnames[k] = get_recipe_pos(k) + remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1])) + + modifying = False + + def outputvalue(name, lines, rewindcomments=False): + if values[name] is None: + return + rawtext = '%s = "%s"%s' % (name, values[name], newline) + addlines = [] + if name in nowrap_vars: + addlines.append(rawtext) + elif name in list_vars: + splitvalue = split_var_value(values[name], assignment=False) + if len(splitvalue) > 1: + linesplit = ' \\\n' + (' ' * (len(name) + 4)) + addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline)) + else: + addlines.append(rawtext) + else: + wrapped = textwrap.wrap(rawtext) + for wrapline in wrapped[:-1]: + addlines.append('%s \\%s' % (wrapline, newline)) + addlines.append('%s%s' % (wrapped[-1], newline)) + if rewindcomments: + # Ensure we insert the lines before any leading comments + # (that we'd want to ensure remain leading the next value) + for i, ln in reversed(list(enumerate(lines))): + if not ln.startswith('#'): + lines[i+1:i+1] = addlines + break + else: + lines.extend(addlines) + else: + lines.extend(addlines) + + existingnames = [] + def patch_recipe_varfunc(varname, origvalue, op, newlines): + if modifying: + # Insert anything that should come before this variable + pos = get_recipe_pos(varname) + for k in list(remainingnames): + if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: + outputvalue(k, newlines, rewindcomments=True) + del remainingnames[k] + # Now change this variable, if it needs to be changed + if varname in existingnames and op in ['+=', '=', '=+']: + if varname in remainingnames: + outputvalue(varname, newlines) + del remainingnames[varname] + return None, None, 0, True + else: + if varname in values: + existingnames.append(varname) + return origvalue, None, 0, True + + # First run - establish which values we want to set are already in the file + varlist = [re.escape(item) for item in values.keys()] + bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc) + # Second run - actually set everything + modifying = True + varlist.extend(recipe_progression_restrs) + changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) + + if remainingnames: + if tolines and tolines[-1].strip() != '': + tolines.append('\n') + for k in remainingnames.keys(): + outputvalue(k, tolines) + + return changed, tolines + + +def patch_recipe_file(fn, values, patch=False, relpath=''): + """Update or insert variable values into a recipe file (assuming you + have already identified the exact file you want to update.) + Note that some manual inspection/intervention may be required + since this cannot handle all situations. + """ + + with open(fn, 'r') as f: + fromlines = f.readlines() + + _, tolines = patch_recipe_lines(fromlines, values) + + if patch: + relfn = os.path.relpath(fn, relpath) + diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) + return diff + else: + with open(fn, 'w') as f: + f.writelines(tolines) + return None + + +def localise_file_vars(fn, varfiles, varlist): + """Given a list of variables and variable history (fetched with get_var_files()) + find where each variable should be set/changed. This handles for example where a + recipe includes an inc file where variables might be changed - in most cases + we want to update the inc file when changing the variable value rather than adding + it to the recipe itself. + """ + fndir = os.path.dirname(fn) + os.sep + + first_meta_file = None + for v in meta_vars: + f = varfiles.get(v, None) + if f: + actualdir = os.path.dirname(f) + os.sep + if actualdir.startswith(fndir): + first_meta_file = f + break + + filevars = defaultdict(list) + for v in varlist: + f = varfiles[v] + # Only return files that are in the same directory as the recipe or in some directory below there + # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable + # in if we were going to set a value specific to this recipe) + if f: + actualfile = f + else: + # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it + if first_meta_file: + actualfile = first_meta_file + else: + actualfile = fn + + actualdir = os.path.dirname(actualfile) + os.sep + if not actualdir.startswith(fndir): + actualfile = fn + filevars[actualfile].append(v) + + return filevars + +def patch_recipe(d, fn, varvalues, patch=False, relpath=''): + """Modify a list of variable values in the specified recipe. Handles inc files if + used by the recipe. + """ + varlist = varvalues.keys() + varfiles = get_var_files(fn, varlist, d) + locs = localise_file_vars(fn, varfiles, varlist) + patches = [] + for f,v in locs.items(): + vals = {k: varvalues[k] for k in v} + patchdata = patch_recipe_file(f, vals, patch, relpath) + if patch: + patches.append(patchdata) + + if patch: + return patches + else: + return None + + + +def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): + """Copy (local) recipe files, including both files included via include/require, + and files referred to in the SRC_URI variable.""" + import bb.fetch2 + import oe.path + + # FIXME need a warning if the unexpanded SRC_URI value contains variable references + + uris = (d.getVar('SRC_URI') or "").split() + fetch = bb.fetch2.Fetch(uris, d) + if download: + fetch.download() + + # Copy local files to target directory and gather any remote files + bb_dir = os.path.dirname(d.getVar('FILE')) + os.sep + remotes = [] + copied = [] + includes = [path for path in d.getVar('BBINCLUDED').split() if + path.startswith(bb_dir) and os.path.exists(path)] + for path in fetch.localpaths() + includes: + # Only import files that are under the meta directory + if path.startswith(bb_dir): + if not whole_dir: + relpath = os.path.relpath(path, bb_dir) + subdir = os.path.join(tgt_dir, os.path.dirname(relpath)) + if not os.path.exists(subdir): + os.makedirs(subdir) + shutil.copy2(path, os.path.join(tgt_dir, relpath)) + copied.append(relpath) + else: + remotes.append(path) + # Simply copy whole meta dir, if requested + if whole_dir: + shutil.copytree(bb_dir, tgt_dir) + + return copied, remotes + + +def get_recipe_local_files(d, patches=False, archives=False): + """Get a list of local files in SRC_URI within a recipe.""" + import oe.patch + uris = (d.getVar('SRC_URI') or "").split() + fetch = bb.fetch2.Fetch(uris, d) + # FIXME this list should be factored out somewhere else (such as the + # fetcher) though note that this only encompasses actual container formats + # i.e. that can contain multiple files as opposed to those that only + # contain a compressed stream (i.e. .tar.gz as opposed to just .gz) + archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z'] + ret = {} + for uri in uris: + if fetch.ud[uri].type == 'file': + if (not patches and + oe.patch.patch_path(uri, fetch, '', expand=False)): + continue + # Skip files that are referenced by absolute path + fname = fetch.ud[uri].basepath + if os.path.isabs(fname): + continue + # Handle subdir= + subdir = fetch.ud[uri].parm.get('subdir', '') + if subdir: + if os.path.isabs(subdir): + continue + fname = os.path.join(subdir, fname) + localpath = fetch.localpath(uri) + if not archives: + # Ignore archives that will be unpacked + if localpath.endswith(tuple(archive_exts)): + unpack = fetch.ud[uri].parm.get('unpack', True) + if unpack: + continue + ret[fname] = localpath + return ret + + +def get_recipe_patches(d): + """Get a list of the patches included in SRC_URI within a recipe.""" + import oe.patch + patches = oe.patch.src_patches(d, expand=False) + patchfiles = [] + for patch in patches: + _, _, local, _, _, parm = bb.fetch.decodeurl(patch) + patchfiles.append(local) + return patchfiles + + +def get_recipe_patched_files(d): + """ + Get the list of patches for a recipe along with the files each patch modifies. + Params: + d: the datastore for the recipe + Returns: + a dict mapping patch file path to a list of tuples of changed files and + change mode ('A' for add, 'D' for delete or 'M' for modify) + """ + import oe.patch + patches = oe.patch.src_patches(d, expand=False) + patchedfiles = {} + for patch in patches: + _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch) + striplevel = int(parm['striplevel']) + patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', ''))) + return patchedfiles + + +def validate_pn(pn): + """Perform validation on a recipe name (PN) for a new recipe.""" + reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] + if not re.match('^[0-9a-z-.+]+$', pn): + return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn + elif pn in reserved_names: + return 'Recipe name "%s" is invalid: is a reserved keyword' % pn + elif pn.startswith('pn-'): + return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn + elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')): + return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn + return '' + + +def get_bbfile_path(d, destdir, extrapathhint=None): + """ + Determine the correct path for a recipe within a layer + Parameters: + d: Recipe-specific datastore + destdir: destination directory. Can be the path to the base of the layer or a + partial path somewhere within the layer. + extrapathhint: a path relative to the base of the layer to try + """ + import bb.cookerdata + + destdir = os.path.abspath(destdir) + destlayerdir = find_layerdir(destdir) + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + pn = d.getVar('PN') + + bbfilespecs = (confdata.getVar('BBFILES') or '').split() + if destdir == destlayerdir: + for bbfilespec in bbfilespecs: + if not bbfilespec.endswith('.bbappend'): + for match in glob.glob(bbfilespec): + splitext = os.path.splitext(os.path.basename(match)) + if splitext[1] == '.bb': + mpn = splitext[0].split('_')[0] + if mpn == pn: + return os.path.dirname(match) + + # Try to make up a path that matches BBFILES + # this is a little crude, but better than nothing + bpn = d.getVar('BPN') + recipefn = os.path.basename(d.getVar('FILE')) + pathoptions = [destdir] + if extrapathhint: + pathoptions.append(os.path.join(destdir, extrapathhint)) + if destdir == destlayerdir: + pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) + pathoptions.append(os.path.join(destdir, 'recipes', bpn)) + pathoptions.append(os.path.join(destdir, bpn)) + elif not destdir.endswith(('/' + pn, '/' + bpn)): + pathoptions.append(os.path.join(destdir, bpn)) + closepath = '' + for pathoption in pathoptions: + bbfilepath = os.path.join(pathoption, 'test.bb') + for bbfilespec in bbfilespecs: + if fnmatch.fnmatchcase(bbfilepath, bbfilespec): + return pathoption + return None + +def get_bbappend_path(d, destlayerdir, wildcardver=False): + """Determine how a bbappend for a recipe should be named and located within another layer""" + + import bb.cookerdata + + destlayerdir = os.path.abspath(destlayerdir) + recipefile = d.getVar('FILE') + recipefn = os.path.splitext(os.path.basename(recipefile))[0] + if wildcardver and '_' in recipefn: + recipefn = recipefn.split('_', 1)[0] + '_%' + appendfn = recipefn + '.bbappend' + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + + origlayerdir = find_layerdir(recipefile) + if not origlayerdir: + return (None, False) + # Now join this to the path where the bbappend is going and check if it is covered by BBFILES + appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn) + closepath = '' + pathok = True + for bbfilespec in confdata.getVar('BBFILES').split(): + if fnmatch.fnmatchcase(appendpath, bbfilespec): + # Our append path works, we're done + break + elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)): + # Try to find the longest matching path + if len(bbfilespec) > len(closepath): + closepath = bbfilespec + else: + # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure + if closepath: + # bbappend layer's layer.conf at least has a spec that picks up .bbappend files + # Now we just need to substitute out any wildcards + appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir) + if 'recipes-*' in appendsubdir: + # Try to copy this part from the original recipe path + res = re.search('/recipes-[^/]+/', recipefile) + if res: + appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0)) + # This is crude, but we have to do something + appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0]) + appendsubdir = appendsubdir.replace('?', 'a') + appendpath = os.path.join(destlayerdir, appendsubdir, appendfn) + else: + pathok = False + return (appendpath, pathok) + + +def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None): + """ + Writes a bbappend file for a recipe + Parameters: + rd: data dictionary for the recipe + destlayerdir: base directory of the layer to place the bbappend in + (subdirectory path from there will be determined automatically) + srcfiles: dict of source files to add to SRC_URI, where the value + is the full path to the file to be added, and the value is the + original filename as it would appear in SRC_URI or None if it + isn't already present. You may pass None for this parameter if + you simply want to specify your own content via the extralines + parameter. + install: dict mapping entries in srcfiles to a tuple of two elements: + install path (*without* ${D} prefix) and permission value (as a + string, e.g. '0644'). + wildcardver: True to use a % wildcard in the bbappend filename, or + False to make the bbappend specific to the recipe version. + machine: + If specified, make the changes in the bbappend specific to this + machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}" + to be added to the bbappend. + extralines: + Extra lines to add to the bbappend. This may be a dict of name + value pairs, or simply a list of the lines. + removevalues: + Variable values to remove - a dict of names/values. + """ + + if not removevalues: + removevalues = {} + + # Determine how the bbappend should be named + appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) + if not appendpath: + bb.error('Unable to determine layer directory containing %s' % recipefile) + return (None, None) + if not pathok: + bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) + + appenddir = os.path.dirname(appendpath) + bb.utils.mkdirhier(appenddir) + + # FIXME check if the bbappend doesn't get overridden by a higher priority layer? + + layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] + if not os.path.abspath(destlayerdir) in layerdirs: + bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') + + bbappendlines = [] + if extralines: + if isinstance(extralines, dict): + for name, value in extralines.items(): + bbappendlines.append((name, '=', value)) + else: + # Do our best to split it + for line in extralines: + if line[-1] == '\n': + line = line[:-1] + splitline = line.split(None, 2) + if len(splitline) == 3: + bbappendlines.append(tuple(splitline)) + else: + raise Exception('Invalid extralines value passed') + + def popline(varname): + for i in range(0, len(bbappendlines)): + if bbappendlines[i][0] == varname: + line = bbappendlines.pop(i) + return line + return None + + def appendline(varname, op, value): + for i in range(0, len(bbappendlines)): + item = bbappendlines[i] + if item[0] == varname: + bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value) + break + else: + bbappendlines.append((varname, op, value)) + + destsubdir = rd.getVar('PN') + if srcfiles: + bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:')) + + appendoverride = '' + if machine: + bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}')) + appendoverride = '_%s' % machine + copyfiles = {} + if srcfiles: + instfunclines = [] + for newfile, origsrcfile in srcfiles.items(): + srcfile = origsrcfile + srcurientry = None + if not srcfile: + srcfile = os.path.basename(newfile) + srcurientry = 'file://%s' % srcfile + # Double-check it's not there already + # FIXME do we care if the entry is added by another bbappend that might go away? + if not srcurientry in rd.getVar('SRC_URI').split(): + if machine: + appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry) + else: + appendline('SRC_URI', '+=', srcurientry) + copyfiles[newfile] = srcfile + if install: + institem = install.pop(newfile, None) + if institem: + (destpath, perms) = institem + instdestpath = replace_dir_vars(destpath, rd) + instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath) + if not instdirline in instfunclines: + instfunclines.append(instdirline) + instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath)) + if instfunclines: + bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines)) + + bb.note('Writing append file %s' % appendpath) + + if os.path.exists(appendpath): + # Work around lack of nonlocal in python 2 + extvars = {'destsubdir': destsubdir} + + def appendfile_varfunc(varname, origvalue, op, newlines): + if varname == 'FILESEXTRAPATHS_prepend': + if origvalue.startswith('${THISDIR}/'): + popline('FILESEXTRAPATHS_prepend') + extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':')) + elif varname == 'PACKAGE_ARCH': + if machine: + popline('PACKAGE_ARCH') + return (machine, None, 4, False) + elif varname.startswith('do_install_append'): + func = popline(varname) + if func: + instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()] + for line in func[2]: + if not line in instfunclines: + instfunclines.append(line) + return (instfunclines, None, 4, False) + else: + splitval = split_var_value(origvalue, assignment=False) + changed = False + removevar = varname + if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]: + removevar = 'SRC_URI' + line = popline(varname) + if line: + if line[2] not in splitval: + splitval.append(line[2]) + changed = True + else: + line = popline(varname) + if line: + splitval = [line[2]] + changed = True + + if removevar in removevalues: + remove = removevalues[removevar] + if isinstance(remove, str): + if remove in splitval: + splitval.remove(remove) + changed = True + else: + for removeitem in remove: + if removeitem in splitval: + splitval.remove(removeitem) + changed = True + + if changed: + newvalue = splitval + if len(newvalue) == 1: + # Ensure it's written out as one line + if '_append' in varname: + newvalue = ' ' + newvalue[0] + else: + newvalue = newvalue[0] + if not newvalue and (op in ['+=', '.='] or '_append' in varname): + # There's no point appending nothing + newvalue = None + if varname.endswith('()'): + indent = 4 + else: + indent = -1 + return (newvalue, None, indent, True) + return (origvalue, None, 4, False) + + varnames = [item[0] for item in bbappendlines] + if removevalues: + varnames.extend(list(removevalues.keys())) + + with open(appendpath, 'r') as f: + (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) + + destsubdir = extvars['destsubdir'] + else: + updated = False + newlines = [] + + if bbappendlines: + for line in bbappendlines: + if line[0].endswith('()'): + newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2]))) + else: + newlines.append('%s %s "%s"\n\n' % line) + updated = True + + if updated: + with open(appendpath, 'w') as f: + f.writelines(newlines) + + if copyfiles: + if machine: + destsubdir = os.path.join(destsubdir, machine) + for newfile, srcfile in copyfiles.items(): + filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile)) + if os.path.abspath(newfile) != os.path.abspath(filedest): + if newfile.startswith(tempfile.gettempdir()): + newfiledisp = os.path.basename(newfile) + else: + newfiledisp = newfile + bb.note('Copying %s to %s' % (newfiledisp, filedest)) + bb.utils.mkdirhier(os.path.dirname(filedest)) + shutil.copyfile(newfile, filedest) + + return (appendpath, os.path.join(appenddir, destsubdir)) + + +def find_layerdir(fn): + """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" + pth = fn + layerdir = '' + while pth: + if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): + layerdir = pth + break + pth = os.path.dirname(pth) + if pth == '/': + return None + return layerdir + + +def replace_dir_vars(path, d): + """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})""" + dirvars = {} + # Sort by length so we get the variables we're interested in first + for var in sorted(list(d.keys()), key=len): + if var.endswith('dir') and var.lower() == var: + value = d.getVar(var) + if value.startswith('/') and not '\n' in value and value not in dirvars: + dirvars[value] = var + for dirpath in sorted(list(dirvars.keys()), reverse=True): + path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) + return path + +def get_recipe_pv_without_srcpv(pv, uri_type): + """ + Get PV without SRCPV common in SCM's for now only + support git. + + Returns tuple with pv, prefix and suffix. + """ + pfx = '' + sfx = '' + + if uri_type == 'git': + git_regex = re.compile("(?P<pfx>v?)(?P<ver>[^\+]*)((?P<sfx>\+(git)?r?(AUTOINC\+))(?P<rev>.*))?") + m = git_regex.match(pv) + + if m: + pv = m.group('ver') + pfx = m.group('pfx') + sfx = m.group('sfx') + else: + regex = re.compile("(?P<pfx>(v|r)?)(?P<ver>.*)") + m = regex.match(pv) + if m: + pv = m.group('ver') + pfx = m.group('pfx') + + return (pv, pfx, sfx) + +def get_recipe_upstream_version(rd): + """ + Get upstream version of recipe using bb.fetch2 methods with support for + http, https, ftp and git. + + bb.fetch2 exceptions can be raised, + FetchError when don't have network access or upstream site don't response. + NoMethodError when uri latest_versionstring method isn't implemented. + + Returns a dictonary with version, type and datetime. + Type can be A for Automatic, M for Manual and U for Unknown. + """ + from bb.fetch2 import decodeurl + from datetime import datetime + + ru = {} + ru['version'] = '' + ru['type'] = 'U' + ru['datetime'] = '' + + pv = rd.getVar('PV') + + # XXX: If don't have SRC_URI means that don't have upstream sources so + # returns the current recipe version, so that upstream version check + # declares a match. + src_uris = rd.getVar('SRC_URI') + if not src_uris: + ru['version'] = pv + ru['type'] = 'M' + ru['datetime'] = datetime.now() + return ru + + # XXX: we suppose that the first entry points to the upstream sources + src_uri = src_uris.split()[0] + uri_type, _, _, _, _, _ = decodeurl(src_uri) + + manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION") + if manual_upstream_version: + # manual tracking of upstream version. + ru['version'] = manual_upstream_version + ru['type'] = 'M' + + manual_upstream_date = rd.getVar("CHECK_DATE") + if manual_upstream_date: + date = datetime.strptime(manual_upstream_date, "%b %d, %Y") + else: + date = datetime.now() + ru['datetime'] = date + + elif uri_type == "file": + # files are always up-to-date + ru['version'] = pv + ru['type'] = 'A' + ru['datetime'] = datetime.now() + else: + ud = bb.fetch2.FetchData(src_uri, rd) + pupver = ud.method.latest_versionstring(ud, rd) + (upversion, revision) = pupver + + # format git version version+gitAUTOINC+HASH + if uri_type == 'git': + (pv, pfx, sfx) = get_recipe_pv_without_srcpv(pv, uri_type) + + # if contains revision but not upversion use current pv + if upversion == '' and revision: + upversion = pv + + if upversion: + tmp = upversion + upversion = '' + + if pfx: + upversion = pfx + tmp + else: + upversion = tmp + + if sfx: + upversion = upversion + sfx + revision[:10] + + if upversion: + ru['version'] = upversion + ru['type'] = 'A' + + ru['datetime'] = datetime.now() + + return ru diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py new file mode 100644 index 0000000000..96591f370e --- /dev/null +++ b/meta/lib/oe/rootfs.py @@ -0,0 +1,1030 @@ +from abc import ABCMeta, abstractmethod +from oe.utils import execute_pre_post_process +from oe.package_manager import * +from oe.manifest import * +import oe.path +import filecmp +import shutil +import os +import subprocess +import re + + +class Rootfs(object, metaclass=ABCMeta): + """ + This is an abstract class. Do not instantiate this directly. + """ + + def __init__(self, d, progress_reporter=None, logcatcher=None): + self.d = d + self.pm = None + self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') + self.deploydir = self.d.getVar('IMGDEPLOYDIR') + self.progress_reporter = progress_reporter + self.logcatcher = logcatcher + + self.install_order = Manifest.INSTALL_ORDER + + @abstractmethod + def _create(self): + pass + + @abstractmethod + def _get_delayed_postinsts(self): + pass + + @abstractmethod + def _save_postinsts(self): + pass + + @abstractmethod + def _log_check(self): + pass + + def _log_check_common(self, type, match): + # Ignore any lines containing log_check to avoid recursion, and ignore + # lines beginning with a + since sh -x may emit code which isn't + # actually executed, but may contain error messages + excludes = [ 'log_check', r'^\+' ] + if hasattr(self, 'log_check_expected_regexes'): + excludes.extend(self.log_check_expected_regexes) + excludes = [re.compile(x) for x in excludes] + r = re.compile(match) + log_path = self.d.expand("${T}/log.do_rootfs") + messages = [] + with open(log_path, 'r') as log: + for line in log: + if self.logcatcher and self.logcatcher.contains(line.rstrip()): + continue + for ee in excludes: + m = ee.search(line) + if m: + break + if m: + continue + + m = r.search(line) + if m: + messages.append('[log_check] %s' % line) + if messages: + if len(messages) == 1: + msg = '1 %s message' % type + else: + msg = '%d %s messages' % (len(messages), type) + msg = '[log_check] %s: found %s in the logfile:\n%s' % \ + (self.d.getVar('PN'), msg, ''.join(messages)) + if type == 'error': + bb.fatal(msg) + else: + bb.warn(msg) + + def _log_check_warn(self): + self._log_check_common('warning', '^(warn|Warn|WARNING:)') + + def _log_check_error(self): + self._log_check_common('error', self.log_check_regex) + + def _insert_feed_uris(self): + if bb.utils.contains("IMAGE_FEATURES", "package-management", + True, False, self.d): + self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", + self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", + self.d.getVar('PACKAGE_FEED_ARCHS')) + + + @abstractmethod + def _handle_intercept_failure(self, failed_script): + pass + + """ + The _cleanup() method should be used to clean-up stuff that we don't really + want to end up on target. For example, in the case of RPM, the DB locks. + The method is called, once, at the end of create() method. + """ + @abstractmethod + def _cleanup(self): + pass + + def _setup_dbg_rootfs(self, dirs): + gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' + if gen_debugfs != '1': + return + + bb.note(" Renaming the original rootfs...") + try: + shutil.rmtree(self.image_rootfs + '-orig') + except: + pass + os.rename(self.image_rootfs, self.image_rootfs + '-orig') + + bb.note(" Creating debug rootfs...") + bb.utils.mkdirhier(self.image_rootfs) + + bb.note(" Copying back package database...") + for dir in dirs: + if not os.path.isdir(self.image_rootfs + '-orig' + dir): + continue + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) + shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) + + cpath = oe.cachedpath.CachedPath() + # Copy files located in /usr/lib/debug or /usr/src/debug + for dir in ["/usr/lib/debug", "/usr/src/debug"]: + src = self.image_rootfs + '-orig' + dir + if cpath.exists(src): + dst = self.image_rootfs + dir + bb.utils.mkdirhier(os.path.dirname(dst)) + shutil.copytree(src, dst) + + # Copy files with suffix '.debug' or located in '.debug' dir. + for root, dirs, files in cpath.walk(self.image_rootfs + '-orig'): + relative_dir = root[len(self.image_rootfs + '-orig'):] + for f in files: + if f.endswith('.debug') or '/.debug' in relative_dir: + bb.utils.mkdirhier(self.image_rootfs + relative_dir) + shutil.copy(os.path.join(root, f), + self.image_rootfs + relative_dir) + + bb.note(" Install complementary '*-dbg' packages...") + self.pm.install_complementary('*-dbg') + + bb.note(" Rename debug rootfs...") + try: + shutil.rmtree(self.image_rootfs + '-dbg') + except: + pass + os.rename(self.image_rootfs, self.image_rootfs + '-dbg') + + bb.note(" Restoreing original rootfs...") + os.rename(self.image_rootfs + '-orig', self.image_rootfs) + + def _exec_shell_cmd(self, cmd): + fakerootcmd = self.d.getVar('FAKEROOT') + if fakerootcmd is not None: + exec_cmd = [fakerootcmd, cmd] + else: + exec_cmd = cmd + + try: + subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + + return None + + def create(self): + bb.note("###### Generate rootfs #######") + pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") + post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") + rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') + + postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR") + if not postinst_intercepts_dir: + postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts") + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + + bb.utils.remove(intercepts_dir, True) + + bb.utils.mkdirhier(self.image_rootfs) + + bb.utils.mkdirhier(self.deploydir) + + shutil.copytree(postinst_intercepts_dir, intercepts_dir) + + execute_pre_post_process(self.d, pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + # call the package manager dependent create method + self._create() + + sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') + bb.utils.mkdirhier(sysconfdir) + with open(sysconfdir + "/version", "w+") as ver: + ver.write(self.d.getVar('BUILDNAME') + "\n") + + execute_pre_post_process(self.d, rootfs_post_install_cmds) + + self._run_intercepts() + + execute_pre_post_process(self.d, post_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", + True, False, self.d): + delayed_postinsts = self._get_delayed_postinsts() + if delayed_postinsts is not None: + bb.fatal("The following packages could not be configured " + "offline and rootfs is read-only: %s" % + delayed_postinsts) + + if self.d.getVar('USE_DEVFS') != "1": + self._create_devfs() + + self._uninstall_unneeded() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._insert_feed_uris() + + self._run_ldconfig() + + if self.d.getVar('USE_DEPMOD') != "0": + self._generate_kernel_module_deps() + + self._cleanup() + self._log_check() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + + def _uninstall_unneeded(self): + # Remove unneeded init script symlinks + delayed_postinsts = self._get_delayed_postinsts() + if delayed_postinsts is None: + if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")): + self._exec_shell_cmd(["update-rc.d", "-f", "-r", + self.d.getVar('IMAGE_ROOTFS'), + "run-postinsts", "remove"]) + + image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", + True, False, self.d) + image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') + + if image_rorfs or image_rorfs_force == "1": + # Remove components that we don't need if it's a read-only rootfs + unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() + pkgs_installed = image_list_installed_packages(self.d) + # Make sure update-alternatives is last on the command line, so + # that it is removed last. This makes sure that its database is + # available while uninstalling packages, allowing alternative + # symlinks of packages to be uninstalled to be managed correctly. + provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") + pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) + + if len(pkgs_to_remove) > 0: + self.pm.remove(pkgs_to_remove, False) + + if delayed_postinsts: + self._save_postinsts() + if image_rorfs: + bb.warn("There are post install scripts " + "in a read-only rootfs") + + post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") + execute_pre_post_process(self.d, post_uninstall_cmds) + + runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", + True, False, self.d) + if not runtime_pkgmanage: + # Remove the package manager data files + self.pm.remove_packaging_data() + + def _run_intercepts(self): + intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), + "intercept_scripts") + + bb.note("Running intercept scripts:") + os.environ['D'] = self.image_rootfs + os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE') + for script in os.listdir(intercepts_dir): + script_full = os.path.join(intercepts_dir, script) + + if script == "postinst_intercept" or not os.access(script_full, os.X_OK): + continue + + bb.note("> Executing %s intercept ..." % script) + + try: + subprocess.check_output(script_full) + except subprocess.CalledProcessError as e: + bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details! (Output: %s)" % + (script, e.returncode, e.output)) + + with open(script_full) as intercept: + registered_pkgs = None + for line in intercept.read().split("\n"): + m = re.match("^##PKGS:(.*)", line) + if m is not None: + registered_pkgs = m.group(1).strip() + break + + if registered_pkgs is not None: + bb.warn("The postinstalls for the following packages " + "will be postponed for first boot: %s" % + registered_pkgs) + + # call the backend dependent handler + self._handle_intercept_failure(registered_pkgs) + + def _run_ldconfig(self): + if self.d.getVar('LDCONFIGDEPEND'): + bb.note("Executing: ldconfig -r" + self.image_rootfs + "-c new -v") + self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', + 'new', '-v']) + + def _check_for_kernel_modules(self, modules_dir): + for root, dirs, files in os.walk(modules_dir, topdown=True): + for name in files: + found_ko = name.endswith(".ko") + if found_ko: + return found_ko + return False + + def _generate_kernel_module_deps(self): + modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') + # if we don't have any modules don't bother to do the depmod + if not self._check_for_kernel_modules(modules_dir): + bb.note("No Kernel Modules found, not running depmod") + return + + kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod", + 'kernel-abiversion') + if not os.path.exists(kernel_abi_ver_file): + bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) + + kernel_ver = open(kernel_abi_ver_file).read().strip(' \n') + versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) + + bb.utils.mkdirhier(versioned_modules_dir) + + self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver]) + + """ + Create devfs: + * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file + * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached + for in the BBPATH + If neither are specified then the default name of files/device_table-minimal.txt + is searched for in the BBPATH (same as the old version.) + """ + def _create_devfs(self): + devtable_list = [] + devtable = self.d.getVar('IMAGE_DEVICE_TABLE') + if devtable is not None: + devtable_list.append(devtable) + else: + devtables = self.d.getVar('IMAGE_DEVICE_TABLES') + if devtables is None: + devtables = 'files/device_table-minimal.txt' + for devtable in devtables.split(): + devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) + + for devtable in devtable_list: + self._exec_shell_cmd(["makedevs", "-r", + self.image_rootfs, "-D", devtable]) + + +class RpmRootfs(Rootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(RpmRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ + '|exit 1|ERROR: |Error: |Error |ERROR '\ + '|Failed |Failed: |Failed$|Failed\(\d+\):)' + self.manifest = RpmManifest(d, manifest_dir) + + self.pm = RpmPM(d, + d.getVar('IMAGE_ROOTFS'), + self.d.getVar('TARGET_VENDOR') + ) + + self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN') + if self.inc_rpm_image_gen != "1": + bb.utils.remove(self.image_rootfs, True) + else: + self.pm.recovery_packaging_data() + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) + + self.pm.create_configs() + + ''' + While rpm incremental image generation is enabled, it will remove the + unneeded pkgs by comparing the new install solution manifest and the + old installed manifest. + ''' + def _create_incremental(self, pkgs_initial_install): + if self.inc_rpm_image_gen == "1": + + pkgs_to_install = list() + for pkg_type in pkgs_initial_install: + pkgs_to_install += pkgs_initial_install[pkg_type] + + installed_manifest = self.pm.load_old_install_solution() + solution_manifest = self.pm.dump_install_solution(pkgs_to_install) + + pkg_to_remove = list() + for pkg in installed_manifest: + if pkg not in solution_manifest: + pkg_to_remove.append(pkg) + + self.pm.update() + + bb.note('incremental update -- upgrade packages in place ') + self.pm.upgrade() + if pkg_to_remove != []: + bb.note('incremental removed: %s' % ' '.join(pkg_to_remove)) + self.pm.remove(pkg_to_remove) + + self.pm.autoremove() + + def _create(self): + pkgs_to_install = self.manifest.parse_initial_manifest() + rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') + rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') + + # update PM index files + self.pm.write_index() + + execute_pre_post_process(self.d, rpm_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + if self.inc_rpm_image_gen == "1": + self._create_incremental(pkgs_to_install) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self.pm.update() + + pkgs = [] + pkgs_attempt = [] + for pkg_type in pkgs_to_install: + if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY: + pkgs_attempt += pkgs_to_install[pkg_type] + else: + pkgs += pkgs_to_install[pkg_type] + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self.pm.install(pkgs) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self.pm.install(pkgs_attempt, True) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self.pm.install_complementary() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) + + execute_pre_post_process(self.d, rpm_post_process_cmds) + + if self.inc_rpm_image_gen == "1": + self.pm.backup_packaging_data() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + + @staticmethod + def _depends_list(): + return ['DEPLOY_DIR_RPM', 'INC_RPM_IMAGE_GEN', 'RPM_PREPROCESS_COMMANDS', + 'RPM_POSTPROCESS_COMMANDS', 'RPM_PREFER_ELF_ARCH'] + + def _get_delayed_postinsts(self): + postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/rpm-postinsts") + if os.path.isdir(postinst_dir): + files = os.listdir(postinst_dir) + for f in files: + bb.note('Delayed package scriptlet: %s' % f) + return files + + return None + + def _save_postinsts(self): + # this is just a stub. For RPM, the failed postinstalls are + # already saved in /etc/rpm-postinsts + pass + + def _log_check(self): + self._log_check_warn() + self._log_check_error() + + def _handle_intercept_failure(self, registered_pkgs): + rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') + bb.utils.mkdirhier(rpm_postinsts_dir) + + # Save the package postinstalls in /etc/rpm-postinsts + for pkg in registered_pkgs.split(): + self.pm.save_rpmpostinst(pkg) + + def _cleanup(self): + pass + +class DpkgOpkgRootfs(Rootfs): + def __init__(self, d, progress_reporter=None, logcatcher=None): + super(DpkgOpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + + def _get_pkgs_postinsts(self, status_file): + def _get_pkg_depends_list(pkg_depends): + pkg_depends_list = [] + # filter version requirements like libc (>= 1.1) + for dep in pkg_depends.split(', '): + m_dep = re.match("^(.*) \(.*\)$", dep) + if m_dep: + dep = m_dep.group(1) + pkg_depends_list.append(dep) + + return pkg_depends_list + + pkgs = {} + pkg_name = "" + pkg_status_match = False + pkg_depends = "" + + with open(status_file) as status: + data = status.read() + status.close() + for line in data.split('\n'): + m_pkg = re.match("^Package: (.*)", line) + m_status = re.match("^Status:.*unpacked", line) + m_depends = re.match("^Depends: (.*)", line) + + if m_pkg is not None: + if pkg_name and pkg_status_match: + pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) + + pkg_name = m_pkg.group(1) + pkg_status_match = False + pkg_depends = "" + elif m_status is not None: + pkg_status_match = True + elif m_depends is not None: + pkg_depends = m_depends.group(1) + + # remove package dependencies not in postinsts + pkg_names = list(pkgs.keys()) + for pkg_name in pkg_names: + deps = pkgs[pkg_name][:] + + for d in deps: + if d not in pkg_names: + pkgs[pkg_name].remove(d) + + return pkgs + + def _get_delayed_postinsts_common(self, status_file): + def _dep_resolve(graph, node, resolved, seen): + seen.append(node) + + for edge in graph[node]: + if edge not in resolved: + if edge in seen: + raise RuntimeError("Packages %s and %s have " \ + "a circular dependency in postinsts scripts." \ + % (node, edge)) + _dep_resolve(graph, edge, resolved, seen) + + resolved.append(node) + + pkg_list = [] + + pkgs = None + if not self.d.getVar('PACKAGE_INSTALL').strip(): + bb.note("Building empty image") + else: + pkgs = self._get_pkgs_postinsts(status_file) + if pkgs: + root = "__packagegroup_postinst__" + pkgs[root] = list(pkgs.keys()) + _dep_resolve(pkgs, root, pkg_list, []) + pkg_list.remove(root) + + if len(pkg_list) == 0: + return None + + return pkg_list + + def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir): + num = 0 + for p in self._get_delayed_postinsts(): + bb.utils.mkdirhier(dst_postinst_dir) + + if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): + shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), + os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) + + num += 1 + +class DpkgRootfs(DpkgOpkgRootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(DpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '^E:' + self.log_check_expected_regexes = \ + [ + "^E: Unmet dependencies." + ] + + bb.utils.remove(self.image_rootfs, True) + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) + self.manifest = DpkgManifest(d, manifest_dir) + self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS'), + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH')) + + + def _create(self): + pkgs_to_install = self.manifest.parse_initial_manifest() + deb_pre_process_cmds = self.d.getVar('DEB_PREPROCESS_COMMANDS') + deb_post_process_cmds = self.d.getVar('DEB_POSTPROCESS_COMMANDS') + + alt_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/alternatives") + bb.utils.mkdirhier(alt_dir) + + # update PM index files + self.pm.write_index() + + execute_pre_post_process(self.d, deb_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + # Don't support incremental, so skip that + self.progress_reporter.next_stage() + + self.pm.update() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + for pkg_type in self.install_order: + if pkg_type in pkgs_to_install: + self.pm.install(pkgs_to_install[pkg_type], + [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + + if self.progress_reporter: + # Don't support attemptonly, so skip that + self.progress_reporter.next_stage() + self.progress_reporter.next_stage() + + self.pm.install_complementary() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/var/lib/dpkg']) + + self.pm.fix_broken_dependencies() + + self.pm.mark_packages("installed") + + self.pm.run_pre_post_installs() + + execute_pre_post_process(self.d, deb_post_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + @staticmethod + def _depends_list(): + return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS'] + + def _get_delayed_postinsts(self): + status_file = self.image_rootfs + "/var/lib/dpkg/status" + return self._get_delayed_postinsts_common(status_file) + + def _save_postinsts(self): + dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/deb-postinsts") + src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/info") + return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) + + def _handle_intercept_failure(self, registered_pkgs): + self.pm.mark_packages("unpacked", registered_pkgs.split()) + + def _log_check(self): + self._log_check_warn() + self._log_check_error() + + def _cleanup(self): + pass + + +class OpkgRootfs(DpkgOpkgRootfs): + def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): + super(OpkgRootfs, self).__init__(d, progress_reporter, logcatcher) + self.log_check_regex = '(exit 1|Collected errors)' + + self.manifest = OpkgManifest(d, manifest_dir) + self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") + self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS") + + self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN') or "" + if self._remove_old_rootfs(): + bb.utils.remove(self.image_rootfs, True) + self.pm = OpkgPM(d, + self.image_rootfs, + self.opkg_conf, + self.pkg_archs) + else: + self.pm = OpkgPM(d, + self.image_rootfs, + self.opkg_conf, + self.pkg_archs) + self.pm.recover_packaging_data() + + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) + + def _prelink_file(self, root_dir, filename): + bb.note('prelink %s in %s' % (filename, root_dir)) + prelink_cfg = oe.path.join(root_dir, + self.d.expand('${sysconfdir}/prelink.conf')) + if not os.path.exists(prelink_cfg): + shutil.copy(self.d.expand('${STAGING_DIR_NATIVE}${sysconfdir_native}/prelink.conf'), + prelink_cfg) + + cmd_prelink = self.d.expand('${STAGING_DIR_NATIVE}${sbindir_native}/prelink') + self._exec_shell_cmd([cmd_prelink, + '--root', + root_dir, + '-amR', + '-N', + '-c', + self.d.expand('${sysconfdir}/prelink.conf')]) + + ''' + Compare two files with the same key twice to see if they are equal. + If they are not equal, it means they are duplicated and come from + different packages. + 1st: Comapre them directly; + 2nd: While incremental image creation is enabled, one of the + files could be probaly prelinked in the previous image + creation and the file has been changed, so we need to + prelink the other one and compare them. + ''' + def _file_equal(self, key, f1, f2): + + # Both of them are not prelinked + if filecmp.cmp(f1, f2): + return True + + if self.image_rootfs not in f1: + self._prelink_file(f1.replace(key, ''), f1) + + if self.image_rootfs not in f2: + self._prelink_file(f2.replace(key, ''), f2) + + # Both of them are prelinked + if filecmp.cmp(f1, f2): + return True + + # Not equal + return False + + """ + This function was reused from the old implementation. + See commit: "image.bbclass: Added variables for multilib support." by + Lianhao Lu. + """ + def _multilib_sanity_test(self, dirs): + + allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") + if allow_replace is None: + allow_replace = "" + + allow_rep = re.compile(re.sub("\|$", "", allow_replace)) + error_prompt = "Multilib check error:" + + files = {} + for dir in dirs: + for root, subfolders, subfiles in os.walk(dir): + for file in subfiles: + item = os.path.join(root, file) + key = str(os.path.join("/", os.path.relpath(item, dir))) + + valid = True + if key in files: + #check whether the file is allow to replace + if allow_rep.match(key): + valid = True + else: + if os.path.exists(files[key]) and \ + os.path.exists(item) and \ + not self._file_equal(key, files[key], item): + valid = False + bb.fatal("%s duplicate files %s %s is not the same\n" % + (error_prompt, item, files[key])) + + #pass the check, add to list + if valid: + files[key] = item + + def _multilib_test_install(self, pkgs): + ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") + bb.utils.mkdirhier(ml_temp) + + dirs = [self.image_rootfs] + + for variant in self.d.getVar("MULTILIB_VARIANTS").split(): + ml_target_rootfs = os.path.join(ml_temp, variant) + + bb.utils.remove(ml_target_rootfs, True) + + ml_opkg_conf = os.path.join(ml_temp, + variant + "-" + os.path.basename(self.opkg_conf)) + + ml_pm = OpkgPM(self.d, ml_target_rootfs, ml_opkg_conf, self.pkg_archs) + + ml_pm.update() + ml_pm.install(pkgs) + + dirs.append(ml_target_rootfs) + + self._multilib_sanity_test(dirs) + + ''' + While ipk incremental image generation is enabled, it will remove the + unneeded pkgs by comparing the old full manifest in previous existing + image and the new full manifest in the current image. + ''' + def _remove_extra_packages(self, pkgs_initial_install): + if self.inc_opkg_image_gen == "1": + # Parse full manifest in previous existing image creation session + old_full_manifest = self.manifest.parse_full_manifest() + + # Create full manifest for the current image session, the old one + # will be replaced by the new one. + self.manifest.create_full(self.pm) + + # Parse full manifest in current image creation session + new_full_manifest = self.manifest.parse_full_manifest() + + pkg_to_remove = list() + for pkg in old_full_manifest: + if pkg not in new_full_manifest: + pkg_to_remove.append(pkg) + + if pkg_to_remove != []: + bb.note('decremental removed: %s' % ' '.join(pkg_to_remove)) + self.pm.remove(pkg_to_remove) + + ''' + Compare with previous existing image creation, if some conditions + triggered, the previous old image should be removed. + The conditions include any of 'PACKAGE_EXCLUDE, NO_RECOMMENDATIONS + and BAD_RECOMMENDATIONS' has been changed. + ''' + def _remove_old_rootfs(self): + if self.inc_opkg_image_gen != "1": + return True + + vars_list_file = self.d.expand('${T}/vars_list') + + old_vars_list = "" + if os.path.exists(vars_list_file): + old_vars_list = open(vars_list_file, 'r+').read() + + new_vars_list = '%s:%s:%s\n' % \ + ((self.d.getVar('BAD_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('NO_RECOMMENDATIONS') or '').strip(), + (self.d.getVar('PACKAGE_EXCLUDE') or '').strip()) + open(vars_list_file, 'w+').write(new_vars_list) + + if old_vars_list != new_vars_list: + return True + + return False + + def _create(self): + pkgs_to_install = self.manifest.parse_initial_manifest() + opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') + opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') + + # update PM index files, unless users provide their own feeds + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": + self.pm.write_index() + + execute_pre_post_process(self.d, opkg_pre_process_cmds) + + if self.progress_reporter: + self.progress_reporter.next_stage() + # Steps are a bit different in order, skip next + self.progress_reporter.next_stage() + + self.pm.update() + + self.pm.handle_bad_recommendations() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + if self.inc_opkg_image_gen == "1": + self._remove_extra_packages(pkgs_to_install) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + for pkg_type in self.install_order: + if pkg_type in pkgs_to_install: + # For multilib, we perform a sanity test before final install + # If sanity test fails, it will automatically do a bb.fatal() + # and the installation will stop + if pkg_type == Manifest.PKG_TYPE_MULTILIB: + self._multilib_test_install(pkgs_to_install[pkg_type]) + + self.pm.install(pkgs_to_install[pkg_type], + [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + + if self.progress_reporter: + self.progress_reporter.next_stage() + + self.pm.install_complementary() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') + self._setup_dbg_rootfs([opkg_dir]) + + execute_pre_post_process(self.d, opkg_post_process_cmds) + + if self.inc_opkg_image_gen == "1": + self.pm.backup_packaging_data() + + if self.progress_reporter: + self.progress_reporter.next_stage() + + @staticmethod + def _depends_list(): + return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] + + def _get_delayed_postinsts(self): + status_file = os.path.join(self.image_rootfs, + self.d.getVar('OPKGLIBDIR').strip('/'), + "opkg", "status") + return self._get_delayed_postinsts_common(status_file) + + def _save_postinsts(self): + dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") + src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") + return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) + + def _handle_intercept_failure(self, registered_pkgs): + self.pm.mark_packages("unpacked", registered_pkgs.split()) + + def _log_check(self): + self._log_check_warn() + self._log_check_error() + + def _cleanup(self): + self.pm.remove_lists() + +def get_class_for_type(imgtype): + return {"rpm": RpmRootfs, + "ipk": OpkgRootfs, + "deb": DpkgRootfs}[imgtype] + +def variable_depends(d, manifest_dir=None): + img_type = d.getVar('IMAGE_PKGTYPE') + cls = get_class_for_type(img_type) + return cls._depends_list() + +def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): + env_bkp = os.environ.copy() + + img_type = d.getVar('IMAGE_PKGTYPE') + if img_type == "rpm": + RpmRootfs(d, manifest_dir, progress_reporter, logcatcher).create() + elif img_type == "ipk": + OpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() + elif img_type == "deb": + DpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create() + + os.environ.clear() + os.environ.update(env_bkp) + + +def image_list_installed_packages(d, rootfs_dir=None): + if not rootfs_dir: + rootfs_dir = d.getVar('IMAGE_ROOTFS') + + img_type = d.getVar('IMAGE_PKGTYPE') + if img_type == "rpm": + return RpmPkgsList(d, rootfs_dir).list_pkgs() + elif img_type == "ipk": + return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET")).list_pkgs() + elif img_type == "deb": + return DpkgPkgsList(d, rootfs_dir).list_pkgs() + +if __name__ == "__main__": + """ + We should be able to run this as a standalone script, from outside bitbake + environment. + """ + """ + TBD + """ diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py new file mode 100644 index 0000000000..deb823b6ec --- /dev/null +++ b/meta/lib/oe/sdk.py @@ -0,0 +1,376 @@ +from abc import ABCMeta, abstractmethod +from oe.utils import execute_pre_post_process +from oe.manifest import * +from oe.package_manager import * +import os +import shutil +import glob +import traceback + + +class Sdk(object, metaclass=ABCMeta): + def __init__(self, d, manifest_dir): + self.d = d + self.sdk_output = self.d.getVar('SDK_OUTPUT') + self.sdk_native_path = self.d.getVar('SDKPATHNATIVE').strip('/') + self.target_path = self.d.getVar('SDKTARGETSYSROOT').strip('/') + self.sysconfdir = self.d.getVar('sysconfdir').strip('/') + + self.sdk_target_sysroot = os.path.join(self.sdk_output, self.target_path) + self.sdk_host_sysroot = self.sdk_output + + if manifest_dir is None: + self.manifest_dir = self.d.getVar("SDK_DIR") + else: + self.manifest_dir = manifest_dir + + self.remove(self.sdk_output, True) + + self.install_order = Manifest.INSTALL_ORDER + + @abstractmethod + def _populate(self): + pass + + def populate(self): + self.mkdirhier(self.sdk_output) + + # call backend dependent implementation + self._populate() + + # Don't ship any libGL in the SDK + self.remove(os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('libdir_nativesdk').strip('/'), + "libGL*")) + + # Fix or remove broken .la files + self.remove(os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('libdir_nativesdk').strip('/'), + "*.la")) + + # Link the ld.so.cache file into the hosts filesystem + link_name = os.path.join(self.sdk_output, self.sdk_native_path, + self.sysconfdir, "ld.so.cache") + self.mkdirhier(os.path.dirname(link_name)) + os.symlink("/etc/ld.so.cache", link_name) + + execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND')) + + def movefile(self, sourcefile, destdir): + try: + # FIXME: this check of movefile's return code to None should be + # fixed within the function to use only exceptions to signal when + # something goes wrong + if (bb.utils.movefile(sourcefile, destdir) == None): + raise OSError("moving %s to %s failed" + %(sourcefile, destdir)) + #FIXME: using umbrella exc catching because bb.utils method raises it + except Exception as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.error("unable to place %s in final SDK location" % sourcefile) + + def mkdirhier(self, dirpath): + try: + bb.utils.mkdirhier(dirpath) + except OSError as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.fatal("cannot make dir for SDK: %s" % dirpath) + + def remove(self, path, recurse=False): + try: + bb.utils.remove(path, recurse) + #FIXME: using umbrella exc catching because bb.utils method raises it + except Exception as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.warn("cannot remove SDK dir: %s" % path) + +class RpmSdk(Sdk): + def __init__(self, d, manifest_dir=None): + super(RpmSdk, self).__init__(d, manifest_dir) + + self.target_manifest = RpmManifest(d, self.manifest_dir, + Manifest.MANIFEST_TYPE_SDK_TARGET) + self.host_manifest = RpmManifest(d, self.manifest_dir, + Manifest.MANIFEST_TYPE_SDK_HOST) + + target_providename = ['/bin/sh', + '/bin/bash', + '/usr/bin/env', + '/usr/bin/perl', + 'pkgconfig' + ] + + self.target_pm = RpmPM(d, + self.sdk_target_sysroot, + self.d.getVar('TARGET_VENDOR'), + 'target', + target_providename + ) + + sdk_providename = ['/bin/sh', + '/bin/bash', + '/usr/bin/env', + '/usr/bin/perl', + 'pkgconfig', + 'libGL.so()(64bit)', + 'libGL.so' + ] + + self.host_pm = RpmPM(d, + self.sdk_host_sysroot, + self.d.getVar('SDK_VENDOR'), + 'host', + sdk_providename, + "SDK_PACKAGE_ARCHS", + "SDK_OS" + ) + + def _populate_sysroot(self, pm, manifest): + pkgs_to_install = manifest.parse_initial_manifest() + + pm.create_configs() + pm.write_index() + pm.update() + + pkgs = [] + pkgs_attempt = [] + for pkg_type in pkgs_to_install: + if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY: + pkgs_attempt += pkgs_to_install[pkg_type] + else: + pkgs += pkgs_to_install[pkg_type] + + pm.install(pkgs) + + pm.install(pkgs_attempt, True) + + def _populate(self): + bb.note("Installing TARGET packages") + self._populate_sysroot(self.target_pm, self.target_manifest) + + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() + + bb.note("Installing NATIVESDK packages") + self._populate_sysroot(self.host_pm, self.host_manifest) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() + + # Move host RPM library data + native_rpm_state_dir = os.path.join(self.sdk_output, + self.sdk_native_path, + self.d.getVar('localstatedir_nativesdk').strip('/'), + "lib", + "rpm" + ) + self.mkdirhier(native_rpm_state_dir) + for f in glob.glob(os.path.join(self.sdk_output, + "var", + "lib", + "rpm", + "*")): + self.movefile(f, native_rpm_state_dir) + + self.remove(os.path.join(self.sdk_output, "var"), True) + + # Move host sysconfig data + native_sysconf_dir = os.path.join(self.sdk_output, + self.sdk_native_path, + self.d.getVar('sysconfdir', + True).strip('/'), + ) + self.mkdirhier(native_sysconf_dir) + for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")): + self.movefile(f, native_sysconf_dir) + for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", "*")): + self.movefile(f, native_sysconf_dir) + self.remove(os.path.join(self.sdk_output, "etc"), True) + + +class OpkgSdk(Sdk): + def __init__(self, d, manifest_dir=None): + super(OpkgSdk, self).__init__(d, manifest_dir) + + self.target_conf = self.d.getVar("IPKGCONF_TARGET") + self.host_conf = self.d.getVar("IPKGCONF_SDK") + + self.target_manifest = OpkgManifest(d, self.manifest_dir, + Manifest.MANIFEST_TYPE_SDK_TARGET) + self.host_manifest = OpkgManifest(d, self.manifest_dir, + Manifest.MANIFEST_TYPE_SDK_HOST) + + self.target_pm = OpkgPM(d, self.sdk_target_sysroot, self.target_conf, + self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) + + self.host_pm = OpkgPM(d, self.sdk_host_sysroot, self.host_conf, + self.d.getVar("SDK_PACKAGE_ARCHS")) + + def _populate_sysroot(self, pm, manifest): + pkgs_to_install = manifest.parse_initial_manifest() + + if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1": + pm.write_index() + + pm.update() + + for pkg_type in self.install_order: + if pkg_type in pkgs_to_install: + pm.install(pkgs_to_install[pkg_type], + [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + + def _populate(self): + bb.note("Installing TARGET packages") + self._populate_sysroot(self.target_pm, self.target_manifest) + + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() + + bb.note("Installing NATIVESDK packages") + self._populate_sysroot(self.host_pm, self.host_manifest) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() + + target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) + host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) + + self.mkdirhier(target_sysconfdir) + shutil.copy(self.target_conf, target_sysconfdir) + os.chmod(os.path.join(target_sysconfdir, + os.path.basename(self.target_conf)), 0o644) + + self.mkdirhier(host_sysconfdir) + shutil.copy(self.host_conf, host_sysconfdir) + os.chmod(os.path.join(host_sysconfdir, + os.path.basename(self.host_conf)), 0o644) + + native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('localstatedir_nativesdk').strip('/'), + "lib", "opkg") + self.mkdirhier(native_opkg_state_dir) + for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "opkg", "*")): + self.movefile(f, native_opkg_state_dir) + + self.remove(os.path.join(self.sdk_output, "var"), True) + + +class DpkgSdk(Sdk): + def __init__(self, d, manifest_dir=None): + super(DpkgSdk, self).__init__(d, manifest_dir) + + self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt") + self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt-sdk") + + self.target_manifest = DpkgManifest(d, self.manifest_dir, + Manifest.MANIFEST_TYPE_SDK_TARGET) + self.host_manifest = DpkgManifest(d, self.manifest_dir, + Manifest.MANIFEST_TYPE_SDK_HOST) + + self.target_pm = DpkgPM(d, self.sdk_target_sysroot, + self.d.getVar("PACKAGE_ARCHS"), + self.d.getVar("DPKG_ARCH"), + self.target_conf_dir) + + self.host_pm = DpkgPM(d, self.sdk_host_sysroot, + self.d.getVar("SDK_PACKAGE_ARCHS"), + self.d.getVar("DEB_SDK_ARCH"), + self.host_conf_dir) + + def _copy_apt_dir_to(self, dst_dir): + staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE") + + self.remove(dst_dir, True) + + shutil.copytree(os.path.join(staging_etcdir_native, "apt"), dst_dir) + + def _populate_sysroot(self, pm, manifest): + pkgs_to_install = manifest.parse_initial_manifest() + + pm.write_index() + pm.update() + + for pkg_type in self.install_order: + if pkg_type in pkgs_to_install: + pm.install(pkgs_to_install[pkg_type], + [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + + def _populate(self): + bb.note("Installing TARGET packages") + self._populate_sysroot(self.target_pm, self.target_manifest) + + self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) + + self._copy_apt_dir_to(os.path.join(self.sdk_target_sysroot, "etc", "apt")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() + + bb.note("Installing NATIVESDK packages") + self._populate_sysroot(self.host_pm, self.host_manifest) + + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) + + self._copy_apt_dir_to(os.path.join(self.sdk_output, self.sdk_native_path, + "etc", "apt")) + + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() + + native_dpkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, + "var", "lib", "dpkg") + self.mkdirhier(native_dpkg_state_dir) + for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "dpkg", "*")): + self.movefile(f, native_dpkg_state_dir) + self.remove(os.path.join(self.sdk_output, "var"), True) + + + +def sdk_list_installed_packages(d, target, rootfs_dir=None): + if rootfs_dir is None: + sdk_output = d.getVar('SDK_OUTPUT') + target_path = d.getVar('SDKTARGETSYSROOT').strip('/') + + rootfs_dir = [sdk_output, os.path.join(sdk_output, target_path)][target is True] + + img_type = d.getVar('IMAGE_PKGTYPE') + if img_type == "rpm": + arch_var = ["SDK_PACKAGE_ARCHS", None][target is True] + os_var = ["SDK_OS", None][target is True] + return RpmPkgsList(d, rootfs_dir).list_pkgs() + elif img_type == "ipk": + conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True] + return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs() + elif img_type == "deb": + return DpkgPkgsList(d, rootfs_dir).list_pkgs() + +def populate_sdk(d, manifest_dir=None): + env_bkp = os.environ.copy() + + img_type = d.getVar('IMAGE_PKGTYPE') + if img_type == "rpm": + RpmSdk(d, manifest_dir).populate() + elif img_type == "ipk": + OpkgSdk(d, manifest_dir).populate() + elif img_type == "deb": + DpkgSdk(d, manifest_dir).populate() + + os.environ.clear() + os.environ.update(env_bkp) + +if __name__ == "__main__": + pass diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py new file mode 100644 index 0000000000..f087a019e1 --- /dev/null +++ b/meta/lib/oe/sstatesig.py @@ -0,0 +1,366 @@ +import bb.siggen + +def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): + # Return True if we should keep the dependency, False to drop it + def isNative(x): + return x.endswith("-native") + def isCross(x): + return "-cross-" in x + def isNativeSDK(x): + return x.startswith("nativesdk-") + def isKernel(fn): + inherits = " ".join(dataCache.inherits[fn]) + return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1 + def isPackageGroup(fn): + inherits = " ".join(dataCache.inherits[fn]) + return "/packagegroup.bbclass" in inherits + def isAllArch(fn): + inherits = " ".join(dataCache.inherits[fn]) + return "/allarch.bbclass" in inherits + def isImage(fn): + return "/image.bbclass" in " ".join(dataCache.inherits[fn]) + + # Always include our own inter-task dependencies + if recipename == depname: + return True + + # Quilt (patch application) changing isn't likely to affect anything + excludelist = ['quilt-native', 'subversion-native', 'git-native'] + if depname in excludelist and recipename != depname: + return False + + # Exclude well defined recipe->dependency + if "%s->%s" % (recipename, depname) in siggen.saferecipedeps: + return False + + # Don't change native/cross/nativesdk recipe dependencies any further + if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename): + return True + + # Only target packages beyond here + + # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes + if isPackageGroup(fn) and isAllArch(fn) and not isNative(depname): + return False + + # Exclude well defined machine specific configurations which don't change ABI + if depname in siggen.abisaferecipes and not isImage(fn): + return False + + # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum + # if we're just doing an RRECOMMENDS_xxx = "kernel-module-*", not least because the checksum + # is machine specific. + # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes) + # and we reccomend a kernel-module, we exclude the dependency. + depfn = dep.rsplit(".", 1)[0] + if dataCache and isKernel(depfn) and not isKernel(fn): + for pkg in dataCache.runrecs[fn]: + if " ".join(dataCache.runrecs[fn][pkg]).find("kernel-module-") != -1: + return False + + # Default to keep dependencies + return True + +def sstate_lockedsigs(d): + sigs = {} + types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split() + for t in types: + siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t + lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split() + for ls in lockedsigs: + pn, task, h = ls.split(":", 2) + if pn not in sigs: + sigs[pn] = {} + sigs[pn][task] = [h, siggen_lockedsigs_var] + return sigs + +class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): + name = "OEBasic" + def init_rundepcheck(self, data): + self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() + pass + def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): + return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) + +class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): + name = "OEBasicHash" + def init_rundepcheck(self, data): + self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() + self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() + self.lockedsigs = sstate_lockedsigs(data) + self.lockedhashes = {} + self.lockedpnmap = {} + self.lockedhashfn = {} + self.machine = data.getVar("MACHINE") + self.mismatch_msgs = [] + self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or + "").split() + self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } + pass + + def tasks_resolved(self, virtmap, virtpnmap, dataCache): + # Translate virtual/xxx entries to PN values + newabisafe = [] + for a in self.abisaferecipes: + if a in virtpnmap: + newabisafe.append(virtpnmap[a]) + else: + newabisafe.append(a) + self.abisaferecipes = newabisafe + newsafedeps = [] + for a in self.saferecipedeps: + a1, a2 = a.split("->") + if a1 in virtpnmap: + a1 = virtpnmap[a1] + if a2 in virtpnmap: + a2 = virtpnmap[a2] + newsafedeps.append(a1 + "->" + a2) + self.saferecipedeps = newsafedeps + + def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): + return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) + + def get_taskdata(self): + data = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskdata() + return (data, self.lockedpnmap, self.lockedhashfn) + + def set_taskdata(self, data): + coredata, self.lockedpnmap, self.lockedhashfn = data + super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata) + + def dump_sigs(self, dataCache, options): + sigfile = os.getcwd() + "/locked-sigs.inc" + bb.plain("Writing locked sigs to %s" % sigfile) + self.dump_lockedsigs(sigfile) + return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) + + def get_taskhash(self, fn, task, deps, dataCache): + h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache) + + recipename = dataCache.pkg_fn[fn] + self.lockedpnmap[fn] = recipename + self.lockedhashfn[fn] = dataCache.hashfn[fn] + + unlocked = False + if recipename in self.unlockedrecipes: + unlocked = True + else: + def recipename_from_dep(dep): + # The dep entry will look something like + # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task, + # ... + fn = dep.rsplit('.', 1)[0] + return dataCache.pkg_fn[fn] + + # If any unlocked recipe is in the direct dependencies then the + # current recipe should be unlocked as well. + depnames = [ recipename_from_dep(x) for x in deps ] + if any(x in y for y in depnames for x in self.unlockedrecipes): + self.unlockedrecipes[recipename] = '' + unlocked = True + + if not unlocked and recipename in self.lockedsigs: + if task in self.lockedsigs[recipename]: + k = fn + "." + task + h_locked = self.lockedsigs[recipename][task][0] + var = self.lockedsigs[recipename][task][1] + self.lockedhashes[k] = h_locked + self.taskhash[k] = h_locked + #bb.warn("Using %s %s %s" % (recipename, task, h)) + + if h != h_locked: + self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' + % (recipename, task, h, h_locked, var)) + + return h_locked + #bb.warn("%s %s %s" % (recipename, task, h)) + return h + + def dump_sigtask(self, fn, task, stampbase, runtime): + k = fn + "." + task + if k in self.lockedhashes: + return + super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) + + def dump_lockedsigs(self, sigfile, taskfilter=None): + types = {} + for k in self.runtaskdeps: + if taskfilter: + if not k in taskfilter: + continue + fn = k.rsplit(".",1)[0] + t = self.lockedhashfn[fn].split(" ")[1].split(":")[5] + t = 't-' + t.replace('_', '-') + if t not in types: + types[t] = [] + types[t].append(k) + + with open(sigfile, "w") as f: + l = sorted(types) + for t in l: + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) + types[t].sort() + sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]]) + for k in sortedk: + fn = k.rsplit(".",1)[0] + task = k.rsplit(".",1)[1] + if k not in self.taskhash: + continue + f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n") + f.write(' "\n') + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(l))) + + def dump_siglist(self, sigfile): + with open(sigfile, "w") as f: + tasks = [] + for taskitem in self.taskhash: + (fn, task) = taskitem.rsplit(".", 1) + pn = self.lockedpnmap[fn] + tasks.append((pn, task, fn, self.taskhash[taskitem])) + for (pn, task, fn, taskhash) in sorted(tasks): + f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash)) + + def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): + warn_msgs = [] + error_msgs = [] + sstate_missing_msgs = [] + + for task in range(len(sq_fn)): + if task not in ret: + for pn in self.lockedsigs: + if sq_hash[task] in iter(self.lockedsigs[pn].values()): + if sq_task[task] == 'do_shared_workdir': + continue + sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" + % (pn, sq_task[task], sq_hash[task])) + + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") + if checklevel == 'warn': + warn_msgs += self.mismatch_msgs + elif checklevel == 'error': + error_msgs += self.mismatch_msgs + + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK") + if checklevel == 'warn': + warn_msgs += sstate_missing_msgs + elif checklevel == 'error': + error_msgs += sstate_missing_msgs + + if warn_msgs: + bb.warn("\n".join(warn_msgs)) + if error_msgs: + bb.fatal("\n".join(error_msgs)) + + +# Insert these classes into siggen's namespace so it can see and select them +bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic +bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash + + +def find_siginfo(pn, taskname, taskhashlist, d): + """ Find signature data files for comparison purposes """ + + import fnmatch + import glob + + if not taskname: + # We have to derive pn and taskname + key = pn + splitit = key.split('.bb.') + taskname = splitit[1] + pn = os.path.basename(splitit[0]).split('_')[0] + if key.startswith('virtual:native:'): + pn = pn + '-native' + + hashfiles = {} + filedates = {} + + def get_hashval(siginfo): + if siginfo.endswith('.siginfo'): + return siginfo.rpartition(':')[2].partition('_')[0] + else: + return siginfo.rpartition('.')[2] + + # First search in stamps dir + localdata = d.createCopy() + localdata.setVar('MULTIMACH_TARGET_SYS', '*') + localdata.setVar('PN', pn) + localdata.setVar('PV', '*') + localdata.setVar('PR', '*') + localdata.setVar('EXTENDPE', '') + stamp = localdata.getVar('STAMP') + if pn.startswith("gcc-source"): + # gcc-source shared workdir is a special case :( + stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") + + filespec = '%s.%s.sigdata.*' % (stamp, taskname) + foundall = False + import glob + for fullpath in glob.glob(filespec): + match = False + if taskhashlist: + for taskhash in taskhashlist: + if fullpath.endswith('.%s' % taskhash): + hashfiles[taskhash] = fullpath + if len(hashfiles) == len(taskhashlist): + foundall = True + break + else: + try: + filedates[fullpath] = os.stat(fullpath).st_mtime + except OSError: + continue + hashval = get_hashval(fullpath) + hashfiles[hashval] = fullpath + + if not taskhashlist or (len(filedates) < 2 and not foundall): + # That didn't work, look in sstate-cache + hashes = taskhashlist or ['*'] + localdata = bb.data.createCopy(d) + for hashval in hashes: + localdata.setVar('PACKAGE_ARCH', '*') + localdata.setVar('TARGET_VENDOR', '*') + localdata.setVar('TARGET_OS', '*') + localdata.setVar('PN', pn) + localdata.setVar('PV', '*') + localdata.setVar('PR', '*') + localdata.setVar('BB_TASKHASH', hashval) + swspec = localdata.getVar('SSTATE_SWSPEC') + if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec: + localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}') + elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: + localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") + sstatename = taskname[3:] + filespec = '%s_%s.*.siginfo' % (localdata.getVar('SSTATE_PKG'), sstatename) + + matchedfiles = glob.glob(filespec) + for fullpath in matchedfiles: + actual_hashval = get_hashval(fullpath) + if actual_hashval in hashfiles: + continue + hashfiles[hashval] = fullpath + if not taskhashlist: + try: + filedates[fullpath] = os.stat(fullpath).st_mtime + except: + continue + + if taskhashlist: + return hashfiles + else: + return filedates + +bb.siggen.find_siginfo = find_siginfo + + +def sstate_get_manifest_filename(task, d): + """ + Return the sstate manifest file path for a particular task. + Also returns the datastore that can be used to query related variables. + """ + d2 = d.createCopy() + extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info') + if extrainf: + d2.setVar("SSTATE_MANMACH", extrainf) + return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2) diff --git a/meta/lib/oe/terminal.py b/meta/lib/oe/terminal.py index 43639d5ddd..0426e15834 100644 --- a/meta/lib/oe/terminal.py +++ b/meta/lib/oe/terminal.py @@ -1,17 +1,18 @@ import logging -import os import oe.classutils import shlex from bb.process import Popen, ExecutionError +from distutils.version import LooseVersion logger = logging.getLogger('BitBake.OE.Terminal') -class UnsupportedTerminal(StandardError): +class UnsupportedTerminal(Exception): pass -class NoSupportedTerminals(StandardError): - pass +class NoSupportedTerminals(Exception): + def __init__(self, terms): + self.terms = terms class Registry(oe.classutils.ClassRegistry): @@ -25,14 +26,11 @@ class Registry(oe.classutils.ClassRegistry): return bool(cls.command) -class Terminal(Popen): - __metaclass__ = Registry - - def __init__(self, command, title=None, env=None): - self.format_command(command, title) - +class Terminal(Popen, metaclass=Registry): + def __init__(self, sh_cmd, title=None, env=None, d=None): + fmt_sh_cmd = self.format_command(sh_cmd, title) try: - Popen.__init__(self, self.command, env=env) + Popen.__init__(self, fmt_sh_cmd, env=env) except OSError as exc: import errno if exc.errno == errno.ENOENT: @@ -40,49 +38,79 @@ class Terminal(Popen): else: raise - def format_command(self, command, title): - fmt = {'title': title or 'Terminal', 'command': command} - if isinstance(self.command, basestring): - self.command = shlex.split(self.command.format(**fmt)) + def format_command(self, sh_cmd, title): + fmt = {'title': title or 'Terminal', 'command': sh_cmd} + if isinstance(self.command, str): + return shlex.split(self.command.format(**fmt)) else: - self.command = [element.format(**fmt) for element in self.command] + return [element.format(**fmt) for element in self.command] class XTerminal(Terminal): - def __init__(self, command, title=None, env=None): - Terminal.__init__(self, command, title, env) + def __init__(self, sh_cmd, title=None, env=None, d=None): + Terminal.__init__(self, sh_cmd, title, env, d) if not os.environ.get('DISPLAY'): raise UnsupportedTerminal(self.name) class Gnome(XTerminal): - command = 'gnome-terminal --disable-factory -t "{title}" -x {command}' + command = 'gnome-terminal -t "{title}" -x {command}' + priority = 2 + + def __init__(self, sh_cmd, title=None, env=None, d=None): + # Recent versions of gnome-terminal does not support non-UTF8 charset: + # https://bugzilla.gnome.org/show_bug.cgi?id=732127; as a workaround, + # clearing the LC_ALL environment variable so it uses the locale. + # Once fixed on the gnome-terminal project, this should be removed. + if os.getenv('LC_ALL'): os.putenv('LC_ALL','') + + # We need to know when the command completes but gnome-terminal gives us no way + # to do this. We therefore write the pid to a file using a "phonehome" wrapper + # script, then monitor the pid until it exits. Thanks gnome! + import tempfile + pidfile = tempfile.NamedTemporaryFile(delete = False).name + try: + sh_cmd = "oe-gnome-terminal-phonehome " + pidfile + " " + sh_cmd + XTerminal.__init__(self, sh_cmd, title, env, d) + while os.stat(pidfile).st_size <= 0: + continue + with open(pidfile, "r") as f: + pid = int(f.readline()) + finally: + os.unlink(pidfile) + + import time + while True: + try: + os.kill(pid, 0) + time.sleep(0.1) + except OSError: + return + +class Mate(XTerminal): + command = 'mate-terminal -t "{title}" -x {command}' priority = 2 class Xfce(XTerminal): - command = 'Terminal -T "{title}" -e "{command}"' + command = 'xfce4-terminal -T "{title}" -e "{command}"' priority = 2 - def __init__(self, command, title=None, env=None): - # Upstream binary name is Terminal but Debian/Ubuntu use - # xfce4-terminal to avoid possible(?) conflicts - distro = distro_name() - if distro == 'ubuntu' or distro == 'debian': - cmd = 'xfce4-terminal -T "{title}" -e "{command}"' - else: - cmd = command - XTerminal.__init__(self, cmd, title, env) +class Terminology(XTerminal): + command = 'terminology -T="{title}" -e {command}' + priority = 2 class Konsole(XTerminal): - command = 'konsole -T "{title}" -e {command}' + command = 'konsole --separate --workdir . -p tabtitle="{title}" -e {command}' priority = 2 - def __init__(self, command, title=None, env=None): + def __init__(self, sh_cmd, title=None, env=None, d=None): # Check version - vernum = check_konsole_version("konsole") - if vernum: - if vernum.split('.')[0] == "2": - logger.debug(1, 'Konsole from KDE 4.x will not work as devshell, skipping') - raise UnsupportedTerminal(self.name) - XTerminal.__init__(self, command, title, env) + vernum = check_terminal_version("konsole") + if vernum and LooseVersion(vernum) < '2.0.0': + # Konsole from KDE 3.x + self.command = 'konsole -T "{title}" -e {command}' + elif vernum and LooseVersion(vernum) < '16.08.1': + # Konsole pre 16.08.01 Has nofork + self.command = 'konsole --nofork --workdir . -p tabtitle="{title}" -e {command}' + XTerminal.__init__(self, sh_cmd, title, env, d) class XTerm(XTerminal): command = 'xterm -T "{title}" -e {command}' @@ -95,27 +123,113 @@ class Rxvt(XTerminal): class Screen(Terminal): command = 'screen -D -m -t "{title}" -S devshell {command}' - def __init__(self, command, title=None, env=None): - Terminal.__init__(self, command, title, env) - logger.warn('Screen started. Please connect in another terminal with ' - '"screen -r devshell"') + def __init__(self, sh_cmd, title=None, env=None, d=None): + s_id = "devshell_%i" % os.getpid() + self.command = "screen -D -m -t \"{title}\" -S %s {command}" % s_id + Terminal.__init__(self, sh_cmd, title, env, d) + msg = 'Screen started. Please connect in another terminal with ' \ + '"screen -r %s"' % s_id + if (d): + bb.event.fire(bb.event.LogExecTTY(msg, "screen -r %s" % s_id, + 0.5, 10), d) + else: + logger.warn(msg) + +class TmuxRunning(Terminal): + """Open a new pane in the current running tmux window""" + name = 'tmux-running' + command = 'tmux split-window "{command}"' + priority = 2.75 + + def __init__(self, sh_cmd, title=None, env=None, d=None): + if not bb.utils.which(os.getenv('PATH'), 'tmux'): + raise UnsupportedTerminal('tmux is not installed') + + if not os.getenv('TMUX'): + raise UnsupportedTerminal('tmux is not running') + + if not check_tmux_pane_size('tmux'): + raise UnsupportedTerminal('tmux pane too small or tmux < 1.9 version is being used') + + Terminal.__init__(self, sh_cmd, title, env, d) + +class TmuxNewWindow(Terminal): + """Open a new window in the current running tmux session""" + name = 'tmux-new-window' + command = 'tmux new-window -n "{title}" "{command}"' + priority = 2.70 + + def __init__(self, sh_cmd, title=None, env=None, d=None): + if not bb.utils.which(os.getenv('PATH'), 'tmux'): + raise UnsupportedTerminal('tmux is not installed') + + if not os.getenv('TMUX'): + raise UnsupportedTerminal('tmux is not running') + + Terminal.__init__(self, sh_cmd, title, env, d) + +class Tmux(Terminal): + """Start a new tmux session and window""" + command = 'tmux new -d -s devshell -n devshell "{command}"' + priority = 0.75 + + def __init__(self, sh_cmd, title=None, env=None, d=None): + if not bb.utils.which(os.getenv('PATH'), 'tmux'): + raise UnsupportedTerminal('tmux is not installed') + + # TODO: consider using a 'devshell' session shared amongst all + # devshells, if it's already there, add a new window to it. + window_name = 'devshell-%i' % os.getpid() + + self.command = 'tmux new -d -s {0} -n {0} "{{command}}"'.format(window_name) + Terminal.__init__(self, sh_cmd, title, env, d) + + attach_cmd = 'tmux att -t {0}'.format(window_name) + msg = 'Tmux started. Please connect in another terminal with `tmux att -t {0}`'.format(window_name) + if d: + bb.event.fire(bb.event.LogExecTTY(msg, attach_cmd, 0.5, 10), d) + else: + logger.warn(msg) + +class Custom(Terminal): + command = 'false' # This is a placeholder + priority = 3 + + def __init__(self, sh_cmd, title=None, env=None, d=None): + self.command = d and d.getVar('OE_TERMINAL_CUSTOMCMD') + if self.command: + if not '{command}' in self.command: + self.command += ' {command}' + Terminal.__init__(self, sh_cmd, title, env, d) + logger.warn('Custom terminal was started.') + else: + logger.debug(1, 'No custom terminal (OE_TERMINAL_CUSTOMCMD) set') + raise UnsupportedTerminal('OE_TERMINAL_CUSTOMCMD not set') def prioritized(): return Registry.prioritized() -def spawn_preferred(command, title=None, env=None): +def get_cmd_list(): + terms = Registry.prioritized() + cmds = [] + for term in terms: + if term.command: + cmds.append(term.command) + return cmds + +def spawn_preferred(sh_cmd, title=None, env=None, d=None): """Spawn the first supported terminal, by priority""" for terminal in prioritized(): try: - spawn(terminal.name, command, title, env) + spawn(terminal.name, sh_cmd, title, env, d) break except UnsupportedTerminal: continue else: - raise NoSupportedTerminals() + raise NoSupportedTerminals(get_cmd_list()) -def spawn(name, command, title=None, env=None): +def spawn(name, sh_cmd, title=None, env=None, d=None): """Spawn the specified terminal, by name""" logger.debug(1, 'Attempting to spawn terminal "%s"', name) try: @@ -123,17 +237,46 @@ def spawn(name, command, title=None, env=None): except KeyError: raise UnsupportedTerminal(name) - pipe = terminal(command, title, env) + pipe = terminal(sh_cmd, title, env, d) output = pipe.communicate()[0] + if output: + output = output.decode("utf-8") if pipe.returncode != 0: - raise ExecutionError(pipe.command, pipe.returncode, output) + raise ExecutionError(sh_cmd, pipe.returncode, output) + +def check_tmux_pane_size(tmux): + import subprocess as sub + # On older tmux versions (<1.9), return false. The reason + # is that there is no easy way to get the height of the active panel + # on current window without nested formats (available from version 1.9) + vernum = check_terminal_version("tmux") + if vernum and LooseVersion(vernum) < '1.9': + return False + try: + p = sub.Popen('%s list-panes -F "#{?pane_active,#{pane_height},}"' % tmux, + shell=True,stdout=sub.PIPE,stderr=sub.PIPE) + out, err = p.communicate() + size = int(out.strip()) + except OSError as exc: + import errno + if exc.errno == errno.ENOENT: + return None + else: + raise -def check_konsole_version(konsole): + return size/2 >= 19 + +def check_terminal_version(terminalName): import subprocess as sub try: - p = sub.Popen(['sh', '-c', '%s --version' % konsole],stdout=sub.PIPE,stderr=sub.PIPE) + cmdversion = '%s --version' % terminalName + if terminalName.startswith('tmux'): + cmdversion = '%s -V' % terminalName + newenv = os.environ.copy() + newenv["LANG"] = "C" + p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv) out, err = p.communicate() - ver_info = out.rstrip().split('\n') + ver_info = out.decode().rstrip().split('\n') except OSError as exc: import errno if exc.errno == errno.ENOENT: @@ -144,6 +287,10 @@ def check_konsole_version(konsole): for ver in ver_info: if ver.startswith('Konsole'): vernum = ver.split(' ')[-1] + if ver.startswith('GNOME Terminal'): + vernum = ver.split(' ')[-1] + if ver.startswith('tmux'): + vernum = ver.split()[-1] return vernum def distro_name(): diff --git a/meta/lib/oe/test_types.py b/meta/lib/oe/test_types.py deleted file mode 100644 index 367cc30e45..0000000000 --- a/meta/lib/oe/test_types.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest -from oe.maketype import create, factory - -class TestTypes(unittest.TestCase): - def assertIsInstance(self, obj, cls): - return self.assertTrue(isinstance(obj, cls)) - - def assertIsNot(self, obj, other): - return self.assertFalse(obj is other) - - def assertFactoryCreated(self, value, type, **flags): - cls = factory(type) - self.assertIsNot(cls, None) - self.assertIsInstance(create(value, type, **flags), cls) - -class TestBooleanType(TestTypes): - def test_invalid(self): - self.assertRaises(ValueError, create, '', 'boolean') - self.assertRaises(ValueError, create, 'foo', 'boolean') - self.assertRaises(TypeError, create, object(), 'boolean') - - def test_true(self): - self.assertTrue(create('y', 'boolean')) - self.assertTrue(create('yes', 'boolean')) - self.assertTrue(create('1', 'boolean')) - self.assertTrue(create('t', 'boolean')) - self.assertTrue(create('true', 'boolean')) - self.assertTrue(create('TRUE', 'boolean')) - self.assertTrue(create('truE', 'boolean')) - - def test_false(self): - self.assertFalse(create('n', 'boolean')) - self.assertFalse(create('no', 'boolean')) - self.assertFalse(create('0', 'boolean')) - self.assertFalse(create('f', 'boolean')) - self.assertFalse(create('false', 'boolean')) - self.assertFalse(create('FALSE', 'boolean')) - self.assertFalse(create('faLse', 'boolean')) - - def test_bool_equality(self): - self.assertEqual(create('n', 'boolean'), False) - self.assertNotEqual(create('n', 'boolean'), True) - self.assertEqual(create('y', 'boolean'), True) - self.assertNotEqual(create('y', 'boolean'), False) - -class TestList(TestTypes): - def assertListEqual(self, value, valid, sep=None): - obj = create(value, 'list', separator=sep) - self.assertEqual(obj, valid) - if sep is not None: - self.assertEqual(obj.separator, sep) - self.assertEqual(str(obj), obj.separator.join(obj)) - - def test_list_nosep(self): - testlist = ['alpha', 'beta', 'theta'] - self.assertListEqual('alpha beta theta', testlist) - self.assertListEqual('alpha beta\ttheta', testlist) - self.assertListEqual('alpha', ['alpha']) - - def test_list_usersep(self): - self.assertListEqual('foo:bar', ['foo', 'bar'], ':') - self.assertListEqual('foo:bar:baz', ['foo', 'bar', 'baz'], ':') diff --git a/meta/lib/oe/types.py b/meta/lib/oe/types.py index ea31cf4219..4ae58acfac 100644 --- a/meta/lib/oe/types.py +++ b/meta/lib/oe/types.py @@ -1,4 +1,7 @@ +import errno import re +import os + class OEList(list): """OpenEmbedded 'list' type @@ -30,7 +33,7 @@ def choice(value, choices): Acts as a multiple choice for the user. To use this, set the variable type flag to 'choice', and set the 'choices' flag to a space separated list of valid values.""" - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("choice accepts a string, not '%s'" % type(value)) value = value.lower() @@ -40,6 +43,31 @@ def choice(value, choices): (value, choices)) return value +class NoMatch(object): + """Stub python regex pattern object which never matches anything""" + def findall(self, string, flags=0): + return None + + def finditer(self, string, flags=0): + return None + + def match(self, flags=0): + return None + + def search(self, string, flags=0): + return None + + def split(self, string, maxsplit=0): + return None + + def sub(pattern, repl, string, count=0): + return None + + def subn(pattern, repl, string, count=0): + return None + +NoMatch = NoMatch() + def regex(value, regexflags=None): """OpenEmbedded 'regex' type @@ -59,9 +87,15 @@ def regex(value, regexflags=None): except AttributeError: raise ValueError("Invalid regex flag '%s'" % flag) + if not value: + # Let's ensure that the default behavior for an undefined or empty + # variable is to match nothing. If the user explicitly wants to match + # anything, they can match '.*' instead. + return NoMatch + try: return re.compile(value, flagval) - except re.error, exc: + except re.error as exc: raise ValueError("Invalid regex value '%s': %s" % (value, exc.args[0])) @@ -72,7 +106,7 @@ def boolean(value): Valid values for false: 'no', 'n', 'false', 'f', '0' """ - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("boolean accepts a string, not '%s'" % type(value)) value = value.lower() @@ -102,3 +136,18 @@ def float(value, fromhex='false'): return _float.fromhex(value) else: return _float(value) + +def path(value, relativeto='', normalize='true', mustexist='false'): + value = os.path.join(relativeto, value) + + if boolean(normalize): + value = os.path.normpath(value) + + if boolean(mustexist): + try: + open(value, 'r') + except IOError as exc: + if exc.errno == errno.ENOENT: + raise ValueError("{0}: {1}".format(value, os.strerror(errno.ENOENT))) + + return value diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 95daace6c6..330a5ff94a 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -1,12 +1,14 @@ -import bb, bb.data +import subprocess def read_file(filename): try: - f = file( filename, "r" ) - except IOError, reason: + f = open( filename, "r" ) + except IOError as reason: return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M: else: - return f.read().strip() + data = f.read().strip() + f.close() + return data return None def ifelse(condition, iftrue = True, iffalse = False): @@ -16,51 +18,61 @@ def ifelse(condition, iftrue = True, iffalse = False): return iffalse def conditional(variable, checkvalue, truevalue, falsevalue, d): - if d.getVar(variable,1) == checkvalue: + if d.getVar(variable) == checkvalue: return truevalue else: return falsevalue def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - if float(d.getVar(variable,1)) <= float(checkvalue): + if float(d.getVar(variable)) <= float(checkvalue): return truevalue else: return falsevalue def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - result = bb.vercmp(d.getVar(variable,True), checkvalue) + result = bb.utils.vercmp_string(d.getVar(variable), checkvalue) if result <= 0: return truevalue else: return falsevalue -def contains(variable, checkvalues, truevalue, falsevalue, d): - val = d.getVar(variable, True) - if not val: - return falsevalue - val = set(val.split()) - if isinstance(checkvalues, basestring): - checkvalues = set(checkvalues.split()) - else: - checkvalues = set(checkvalues) - if checkvalues.issubset(val): - return truevalue - return falsevalue - def both_contain(variable1, variable2, checkvalue, d): - if d.getVar(variable1,1).find(checkvalue) != -1 and d.getVar(variable2,1).find(checkvalue) != -1: - return checkvalue + val1 = d.getVar(variable1) + val2 = d.getVar(variable2) + val1 = set(val1.split()) + val2 = set(val2.split()) + if isinstance(checkvalue, str): + checkvalue = set(checkvalue.split()) + else: + checkvalue = set(checkvalue) + if checkvalue.issubset(val1) and checkvalue.issubset(val2): + return " ".join(checkvalue) else: return "" +def set_intersect(variable1, variable2, d): + """ + Expand both variables, interpret them as lists of strings, and return the + intersection as a flattened string. + + For example: + s1 = "a b c" + s2 = "b c d" + s3 = set_intersect(s1, s2) + => s3 = "b c" + """ + val1 = set(d.getVar(variable1).split()) + val2 = set(d.getVar(variable2).split()) + return " ".join(val1 & val2) + def prune_suffix(var, suffixes, d): - # See if var ends with any of the suffixes listed and + # See if var ends with any of the suffixes listed and # remove it if found for suffix in suffixes: if var.endswith(suffix): var = var.replace(suffix, "") - prefix = d.getVar("MLPREFIX", True) + prefix = d.getVar("MLPREFIX") if prefix and var.startswith(prefix): var = var.replace(prefix, "") @@ -68,11 +80,11 @@ def prune_suffix(var, suffixes, d): def str_filter(f, str, d): from re import match - return " ".join(filter(lambda x: match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if match(f, x, 0)]) def str_filter_out(f, str, d): from re import match - return " ".join(filter(lambda x: not match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if not match(f, x, 0)]) def param_bool(cfg, field, dflt = None): """Lookup <field> in <cfg> map and convert it to a boolean; take @@ -85,6 +97,240 @@ def param_bool(cfg, field, dflt = None): return False raise ValueError("invalid value for boolean parameter '%s': '%s'" % (field, value)) +def build_depends_string(depends, task): + """Append a taskname to a string of dependencies as used by the [depends] flag""" + return " ".join(dep + ":" + task for dep in depends.split()) + def inherits(d, *classes): """Return True if the metadata inherits any of the specified classes""" return any(bb.data.inherits_class(cls, d) for cls in classes) + +def features_backfill(var,d): + # This construct allows the addition of new features to variable specified + # as var + # Example for var = "DISTRO_FEATURES" + # This construct allows the addition of new features to DISTRO_FEATURES + # that if not present would disable existing functionality, without + # disturbing distributions that have already set DISTRO_FEATURES. + # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should + # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED + features = (d.getVar(var) or "").split() + backfill = (d.getVar(var+"_BACKFILL") or "").split() + considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split() + + addfeatures = [] + for feature in backfill: + if feature not in features and feature not in considered: + addfeatures.append(feature) + + if addfeatures: + d.appendVar(var, " " + " ".join(addfeatures)) + + +def packages_filter_out_system(d): + """ + Return a list of packages from PACKAGES with the "system" packages such as + PN-dbg PN-doc PN-locale-eb-gb removed. + """ + pn = d.getVar('PN') + blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] + localepkg = pn + "-locale-" + pkgs = [] + + for pkg in d.getVar('PACKAGES').split(): + if pkg not in blacklist and localepkg not in pkg: + pkgs.append(pkg) + return pkgs + +def getstatusoutput(cmd): + return subprocess.getstatusoutput(cmd) + + +def trim_version(version, num_parts=2): + """ + Return just the first <num_parts> of <version>, split by periods. For + example, trim_version("1.2.3", 2) will return "1.2". + """ + if type(version) is not str: + raise TypeError("Version should be a string") + if num_parts < 1: + raise ValueError("Cannot split to parts < 1") + + parts = version.split(".") + trimmed = ".".join(parts[:num_parts]) + return trimmed + +def cpu_count(): + import multiprocessing + return multiprocessing.cpu_count() + +def execute_pre_post_process(d, cmds): + if cmds is None: + return + + for cmd in cmds.strip().split(';'): + cmd = cmd.strip() + if cmd != '': + bb.note("Executing %s ..." % cmd) + bb.build.exec_func(cmd, d) + +def multiprocess_exec(commands, function): + import signal + import multiprocessing + + if not commands: + return [] + + def init_worker(): + signal.signal(signal.SIGINT, signal.SIG_IGN) + + nproc = min(multiprocessing.cpu_count(), len(commands)) + pool = bb.utils.multiprocessingpool(nproc, init_worker) + imap = pool.imap(function, commands) + + try: + res = list(imap) + pool.close() + pool.join() + results = [] + for result in res: + if result is not None: + results.append(result) + return results + + except KeyboardInterrupt: + pool.terminate() + pool.join() + raise + +def squashspaces(string): + import re + return re.sub("\s+", " ", string).strip() + +def format_pkg_list(pkg_dict, ret_format=None): + output = [] + + if ret_format == "arch": + for pkg in sorted(pkg_dict): + output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"])) + elif ret_format == "file": + for pkg in sorted(pkg_dict): + output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"])) + elif ret_format == "ver": + for pkg in sorted(pkg_dict): + output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"])) + elif ret_format == "deps": + for pkg in sorted(pkg_dict): + for dep in pkg_dict[pkg]["deps"]: + output.append("%s|%s" % (pkg, dep)) + else: + for pkg in sorted(pkg_dict): + output.append(pkg) + + return '\n'.join(output) + +def host_gcc_version(d): + import re, subprocess + + compiler = d.getVar("BUILD_CC") + try: + env = os.environ.copy() + env["PATH"] = d.getVar("PATH") + output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) + + match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + if not match: + bb.fatal("Can't get compiler version from %s --version output" % compiler) + + version = match.group(1) + return "-%s" % version if version in ("4.8", "4.9") else "" + +# +# Python 2.7 doesn't have threaded pools (just multiprocessing) +# so implement a version here +# + +from queue import Queue +from threading import Thread + +class ThreadedWorker(Thread): + """Thread executing tasks from a given tasks queue""" + def __init__(self, tasks, worker_init, worker_end): + Thread.__init__(self) + self.tasks = tasks + self.daemon = True + + self.worker_init = worker_init + self.worker_end = worker_end + + def run(self): + from queue import Empty + + if self.worker_init is not None: + self.worker_init(self) + + while True: + try: + func, args, kargs = self.tasks.get(block=False) + except Empty: + if self.worker_end is not None: + self.worker_end(self) + break + + try: + func(self, *args, **kargs) + except Exception as e: + print(e) + finally: + self.tasks.task_done() + +class ThreadedPool: + """Pool of threads consuming tasks from a queue""" + def __init__(self, num_workers, num_tasks, worker_init=None, + worker_end=None): + self.tasks = Queue(num_tasks) + self.workers = [] + + for _ in range(num_workers): + worker = ThreadedWorker(self.tasks, worker_init, worker_end) + self.workers.append(worker) + + def start(self): + for worker in self.workers: + worker.start() + + def add_task(self, func, *args, **kargs): + """Add a task to the queue""" + self.tasks.put((func, args, kargs)) + + def wait_completion(self): + """Wait for completion of all the tasks in the queue""" + self.tasks.join() + for worker in self.workers: + worker.join() + +def write_ld_so_conf(d): + # Some utils like prelink may not have the correct target library paths + # so write an ld.so.conf to help them + ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf") + if os.path.exists(ldsoconf): + bb.utils.remove(ldsoconf) + bb.utils.mkdirhier(os.path.dirname(ldsoconf)) + with open(ldsoconf, "w") as f: + f.write(d.getVar("base_libdir") + '\n') + f.write(d.getVar("libdir") + '\n') + +class ImageQAFailed(bb.build.FuncFailed): + def __init__(self, description, name=None, logfile=None): + self.description = description + self.name = name + self.logfile=logfile + + def __str__(self): + msg = 'Function failed: %s' % self.name + if self.description: + msg = msg + ' (%s)' % self.description + + return msg |