diff options
Diffstat (limited to 'meta/lib/oe')
30 files changed, 8350 insertions, 633 deletions
diff --git a/meta/lib/oe/__init__.py b/meta/lib/oe/__init__.py index e69de29bb2..3ad9513f40 100644 --- a/meta/lib/oe/__init__.py +++ b/meta/lib/oe/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py new file mode 100644 index 0000000000..3a5b7b6b44 --- /dev/null +++ b/meta/lib/oe/buildhistory_analysis.py @@ -0,0 +1,607 @@ +# Report significant differences in the buildhistory repository since a specific revision +# +# Copyright (C) 2012-2013, 2016-2017 Intel Corporation +# Author: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Note: requires GitPython 0.3.1+ +# +# You can use this from the command line by running scripts/buildhistory-diff +# + +import sys +import os.path +import difflib +import git +import re +import hashlib +import collections +import bb.utils +import bb.tinfoil + + +# How to display fields +list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] +list_order_fields = ['PACKAGES'] +defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'} +numeric_fields = ['PKGSIZE', 'IMAGESIZE'] +# Fields to monitor +monitor_fields = ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RREPLACES', 'RCONFLICTS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE', 'PKG'] +ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR'] +# Percentage change to alert for numeric fields +monitor_numeric_threshold = 10 +# Image files to monitor (note that image-info.txt is handled separately) +img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt'] +# Related context fields for reporting (note: PE, PV & PR are always reported for monitored package fields) +related_fields = {} +related_fields['RDEPENDS'] = ['DEPENDS'] +related_fields['RRECOMMENDS'] = ['DEPENDS'] +related_fields['FILELIST'] = ['FILES'] +related_fields['PKGSIZE'] = ['FILELIST'] +related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND'] +related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'NO_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] + + +class ChangeRecord: + def __init__(self, path, fieldname, oldvalue, newvalue, monitored): + self.path = path + self.fieldname = fieldname + self.oldvalue = oldvalue + self.newvalue = newvalue + self.monitored = monitored + self.related = [] + self.filechanges = None + + def __str__(self): + return self._str_internal(True) + + def _str_internal(self, outer): + if outer: + if '/image-files/' in self.path: + prefix = '%s: ' % self.path.split('/image-files/')[0] + else: + prefix = '%s: ' % self.path + else: + prefix = '' + + def pkglist_combine(depver): + pkglist = [] + for k,v in depver.items(): + if v: + pkglist.append("%s (%s)" % (k,v)) + else: + pkglist.append(k) + return pkglist + + def detect_renamed_dirs(aitems, bitems): + adirs = set(map(os.path.dirname, aitems)) + bdirs = set(map(os.path.dirname, bitems)) + files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \ + for name in adirs - bdirs] + files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \ + for name in bdirs - adirs] + renamed_dirs = [(dir1, dir2) for dir1, files1 in files_ab for dir2, files2 in files_ba if files1 == files2] + # remove files that belong to renamed dirs from aitems and bitems + for dir1, dir2 in renamed_dirs: + aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)] + bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)] + return renamed_dirs, aitems, bitems + + if self.fieldname in list_fields or self.fieldname in list_order_fields: + renamed_dirs = [] + if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: + (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue) + aitems = pkglist_combine(depvera) + bitems = pkglist_combine(depverb) + else: + aitems = self.oldvalue.split() + bitems = self.newvalue.split() + if self.fieldname == 'FILELIST': + renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + + removed = list(set(aitems) - set(bitems)) + added = list(set(bitems) - set(aitems)) + + lines = [] + if renamed_dirs: + for dfrom, dto in renamed_dirs: + lines.append('directory renamed %s -> %s' % (dfrom, dto)) + if removed or added: + if removed and not bitems: + lines.append('removed all items "%s"' % ' '.join(removed)) + else: + if removed: + lines.append('removed "%s"' % ' '.join(removed)) + if added: + lines.append('added "%s"' % ' '.join(added)) + else: + lines.append('changed order') + + out = '%s: %s' % (self.fieldname, ', '.join(lines)) + + elif self.fieldname in numeric_fields: + aval = int(self.oldvalue or 0) + bval = int(self.newvalue or 0) + if aval != 0: + percentchg = ((bval - aval) / float(aval)) * 100 + else: + percentchg = 100 + out = '%s changed from %s to %s (%s%d%%)' % (self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg) + elif self.fieldname in defaultval_map: + out = '%s changed from %s to %s' % (self.fieldname, self.oldvalue, self.newvalue) + if self.fieldname == 'PKG' and '[default]' in self.newvalue: + out += ' - may indicate debian renaming failure' + elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']: + if self.oldvalue and self.newvalue: + out = '%s changed:\n ' % self.fieldname + elif self.newvalue: + out = '%s added:\n ' % self.fieldname + elif self.oldvalue: + out = '%s cleared:\n ' % self.fieldname + alines = self.oldvalue.splitlines() + blines = self.newvalue.splitlines() + diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='') + out += '\n '.join(list(diff)[2:]) + out += '\n --' + elif self.fieldname in img_monitor_files or '/image-files/' in self.path: + fieldname = self.fieldname + if '/image-files/' in self.path: + fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname) + out = 'Changes to %s:\n ' % fieldname + else: + if outer: + prefix = 'Changes to %s ' % self.path + out = '(%s):\n ' % self.fieldname + if self.filechanges: + out += '\n '.join(['%s' % i for i in self.filechanges]) + else: + alines = self.oldvalue.splitlines() + blines = self.newvalue.splitlines() + diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='') + out += '\n '.join(list(diff)) + out += '\n --' + else: + out = '%s changed from "%s" to "%s"' % (self.fieldname, self.oldvalue, self.newvalue) + + if self.related: + for chg in self.related: + if not outer and chg.fieldname in ['PE', 'PV', 'PR']: + continue + for line in chg._str_internal(False).splitlines(): + out += '\n * %s' % line + + return '%s%s' % (prefix, out) + +class FileChange: + changetype_add = 'A' + changetype_remove = 'R' + changetype_type = 'T' + changetype_perms = 'P' + changetype_ownergroup = 'O' + changetype_link = 'L' + + def __init__(self, path, changetype, oldvalue = None, newvalue = None): + self.path = path + self.changetype = changetype + self.oldvalue = oldvalue + self.newvalue = newvalue + + def _ftype_str(self, ftype): + if ftype == '-': + return 'file' + elif ftype == 'd': + return 'directory' + elif ftype == 'l': + return 'symlink' + elif ftype == 'c': + return 'char device' + elif ftype == 'b': + return 'block device' + elif ftype == 'p': + return 'fifo' + elif ftype == 's': + return 'socket' + else: + return 'unknown (%s)' % ftype + + def __str__(self): + if self.changetype == self.changetype_add: + return '%s was added' % self.path + elif self.changetype == self.changetype_remove: + return '%s was removed' % self.path + elif self.changetype == self.changetype_type: + return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue)) + elif self.changetype == self.changetype_perms: + return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue) + elif self.changetype == self.changetype_ownergroup: + return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue) + elif self.changetype == self.changetype_link: + return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue) + else: + return '%s changed (unknown)' % self.path + + +def blob_to_dict(blob): + alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] + adict = {} + for line in alines: + splitv = [i.strip() for i in line.split('=',1)] + if len(splitv) > 1: + adict[splitv[0]] = splitv[1] + return adict + + +def file_list_to_dict(lines): + adict = {} + for line in lines: + # Leave the last few fields intact so we handle file names containing spaces + splitv = line.split(None,4) + # Grab the path and remove the leading . + path = splitv[4][1:].strip() + # Handle symlinks + if(' -> ' in path): + target = path.split(' -> ')[1] + path = path.split(' -> ')[0] + adict[path] = splitv[0:3] + [target] + else: + adict[path] = splitv[0:3] + return adict + + +def compare_file_lists(alines, blines): + adict = file_list_to_dict(alines) + bdict = file_list_to_dict(blines) + filechanges = [] + for path, splitv in adict.items(): + newsplitv = bdict.pop(path, None) + if newsplitv: + # Check type + oldvalue = splitv[0][0] + newvalue = newsplitv[0][0] + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue)) + # Check permissions + oldvalue = splitv[0][1:] + newvalue = newsplitv[0][1:] + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue)) + # Check owner/group + oldvalue = '%s/%s' % (splitv[1], splitv[2]) + newvalue = '%s/%s' % (newsplitv[1], newsplitv[2]) + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue)) + # Check symlink target + if newsplitv[0][0] == 'l': + if len(splitv) > 3: + oldvalue = splitv[3] + else: + oldvalue = None + newvalue = newsplitv[3] + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue)) + else: + filechanges.append(FileChange(path, FileChange.changetype_remove)) + + # Whatever is left over has been added + for path in bdict: + filechanges.append(FileChange(path, FileChange.changetype_add)) + + return filechanges + + +def compare_lists(alines, blines): + removed = list(set(alines) - set(blines)) + added = list(set(blines) - set(alines)) + + filechanges = [] + for pkg in removed: + filechanges.append(FileChange(pkg, FileChange.changetype_remove)) + for pkg in added: + filechanges.append(FileChange(pkg, FileChange.changetype_add)) + + return filechanges + + +def compare_pkg_lists(astr, bstr): + depvera = bb.utils.explode_dep_versions2(astr) + depverb = bb.utils.explode_dep_versions2(bstr) + + # Strip out changes where the version has increased + remove = [] + for k in depvera: + if k in depverb: + dva = depvera[k] + dvb = depverb[k] + if dva and dvb and len(dva) == len(dvb): + # Since length is the same, sort so that prefixes (e.g. >=) will line up + dva.sort() + dvb.sort() + removeit = True + for dvai, dvbi in zip(dva, dvb): + if dvai != dvbi: + aiprefix = dvai.split(' ')[0] + biprefix = dvbi.split(' ')[0] + if aiprefix == biprefix and aiprefix in ['>=', '=']: + if bb.utils.vercmp(bb.utils.split_version(dvai), bb.utils.split_version(dvbi)) > 0: + removeit = False + break + else: + removeit = False + break + if removeit: + remove.append(k) + + for k in remove: + depvera.pop(k) + depverb.pop(k) + + return (depvera, depverb) + + +def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): + adict = blob_to_dict(ablob) + bdict = blob_to_dict(bblob) + + pkgname = os.path.basename(path) + + defaultvals = {} + defaultvals['PKG'] = pkgname + defaultvals['PKGE'] = '0' + + changes = [] + keys = list(set(adict.keys()) | set(bdict.keys()) | set(defaultval_map.keys())) + for key in keys: + astr = adict.get(key, '') + bstr = bdict.get(key, '') + if key in ver_monitor_fields: + monitored = report_ver or astr or bstr + else: + monitored = key in monitor_fields + mapped_key = defaultval_map.get(key, '') + if mapped_key: + if not astr: + astr = '%s [default]' % adict.get(mapped_key, defaultvals.get(key, '')) + if not bstr: + bstr = '%s [default]' % bdict.get(mapped_key, defaultvals.get(key, '')) + + if astr != bstr: + if (not report_all) and key in numeric_fields: + aval = int(astr or 0) + bval = int(bstr or 0) + if aval != 0: + percentchg = ((bval - aval) / float(aval)) * 100 + else: + percentchg = 100 + if abs(percentchg) < monitor_numeric_threshold: + continue + elif (not report_all) and key in list_fields: + if key == "FILELIST" and path.endswith("-dbg") and bstr.strip() != '': + continue + if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: + (depvera, depverb) = compare_pkg_lists(astr, bstr) + if depvera == depverb: + continue + alist = astr.split() + alist.sort() + blist = bstr.split() + blist.sort() + # We don't care about the removal of self-dependencies + if pkgname in alist and not pkgname in blist: + alist.remove(pkgname) + if ' '.join(alist) == ' '.join(blist): + continue + + if key == 'PKGR' and not report_all: + vers = [] + # strip leading 'r' and dots + for ver in (astr.split()[0], bstr.split()[0]): + if ver.startswith('r'): + ver = ver[1:] + vers.append(ver.replace('.', '')) + maxlen = max(len(vers[0]), len(vers[1])) + try: + # pad with '0' and convert to int + vers = [int(ver.ljust(maxlen, '0')) for ver in vers] + except ValueError: + pass + else: + # skip decrements and increments + if abs(vers[0] - vers[1]) == 1: + continue + + chg = ChangeRecord(path, key, astr, bstr, monitored) + changes.append(chg) + return changes + + +def compare_siglists(a_blob, b_blob, taskdiff=False): + # FIXME collapse down a recipe's tasks? + alines = a_blob.data_stream.read().decode('utf-8').splitlines() + blines = b_blob.data_stream.read().decode('utf-8').splitlines() + keys = [] + pnmap = {} + def readsigs(lines): + sigs = {} + for line in lines: + linesplit = line.split() + if len(linesplit) > 2: + sigs[linesplit[0]] = linesplit[2] + if not linesplit[0] in keys: + keys.append(linesplit[0]) + pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0] + return sigs + adict = readsigs(alines) + bdict = readsigs(blines) + out = [] + + changecount = 0 + addcount = 0 + removecount = 0 + if taskdiff: + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + + changes = collections.OrderedDict() + + def compare_hashfiles(pn, taskname, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data) + + if not taskname: + (pn, taskname) = pn.rsplit('.', 1) + pn = pnmap.get(pn, pn) + desc = '%s.%s' % (pn, taskname) + + if len(hashfiles) == 0: + out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2)) + elif not hash1 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1)) + elif not hash2 in hashfiles: + out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2)) + else: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True) + for line in out2: + m = hashlib.sha256() + m.update(line.encode('utf-8')) + entry = changes.get(m.hexdigest(), (line, [])) + if desc not in entry[1]: + changes[m.hexdigest()] = (line, entry[1] + [desc]) + + # Define recursion callback + def recursecb(key, hash1, hash2): + compare_hashfiles(key, None, hash1, hash2) + return [] + + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + changecount += 1 + (pn, taskname) = key.rsplit('.', 1) + compare_hashfiles(pn, taskname, siga, sigb) + elif siga is None: + addcount += 1 + elif sigb is None: + removecount += 1 + for key, item in changes.items(): + line, tasks = item + if len(tasks) == 1: + desc = tasks[0] + elif len(tasks) == 2: + desc = '%s and %s' % (tasks[0], tasks[1]) + else: + desc = '%s and %d others' % (tasks[-1], len(tasks)-1) + out.append('%s: %s' % (desc, line)) + else: + for key in keys: + siga = adict.get(key, None) + sigb = bdict.get(key, None) + if siga is not None and sigb is not None and siga != sigb: + out.append('%s changed from %s to %s' % (key, siga, sigb)) + changecount += 1 + elif siga is None: + out.append('%s was added' % key) + addcount += 1 + elif sigb is None: + out.append('%s was removed' % key) + removecount += 1 + out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100))) + return '\n'.join(out) + + +def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False): + repo = git.Repo(repopath) + assert repo.bare == False + commit = repo.commit(revision1) + diff = commit.diff(revision2) + + changes = [] + + if sigs or sigsdiff: + for d in diff.iter_change_type('M'): + if d.a_blob.path == 'siglist.txt': + changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff)) + return changes + + for d in diff.iter_change_type('M'): + path = os.path.dirname(d.a_blob.path) + if path.startswith('packages/'): + filename = os.path.basename(d.a_blob.path) + if filename == 'latest': + changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) + elif filename.startswith('latest.'): + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) + changes.append(chg) + elif path.startswith('images/'): + filename = os.path.basename(d.a_blob.path) + if filename in img_monitor_files: + if filename == 'files-in-image.txt': + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() + filechanges = compare_file_lists(alines,blines) + if filechanges: + chg = ChangeRecord(path, filename, None, None, True) + chg.filechanges = filechanges + changes.append(chg) + elif filename == 'installed-package-names.txt': + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() + filechanges = compare_lists(alines,blines) + if filechanges: + chg = ChangeRecord(path, filename, None, None, True) + chg.filechanges = filechanges + changes.append(chg) + else: + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) + changes.append(chg) + elif filename == 'image-info.txt': + changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) + elif '/image-files/' in path: + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) + changes.append(chg) + + # Look for added preinst/postinst/prerm/postrm + # (without reporting newly added recipes) + addedpkgs = [] + addedchanges = [] + for d in diff.iter_change_type('A'): + path = os.path.dirname(d.b_blob.path) + if path.startswith('packages/'): + filename = os.path.basename(d.b_blob.path) + if filename == 'latest': + addedpkgs.append(path) + elif filename.startswith('latest.'): + chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True) + addedchanges.append(chg) + for chg in addedchanges: + found = False + for pkg in addedpkgs: + if chg.path.startswith(pkg): + found = True + break + if not found: + changes.append(chg) + + # Look for cleared preinst/postinst/prerm/postrm + for d in diff.iter_change_type('D'): + path = os.path.dirname(d.a_blob.path) + if path.startswith('packages/'): + filename = os.path.basename(d.a_blob.path) + if filename != 'latest' and filename.startswith('latest.'): + chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) + changes.append(chg) + + # Link related changes + for chg in changes: + if chg.monitored: + for chg2 in changes: + # (Check dirname in the case of fields from recipe info files) + if chg.path == chg2.path or os.path.dirname(chg.path) == chg2.path: + if chg2.fieldname in related_fields.get(chg.fieldname, []): + chg.related.append(chg2) + elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']: + chg.related.append(chg2) + + if report_all: + return changes + else: + return [chg for chg in changes if chg.monitored] diff --git a/meta/lib/oe/cachedpath.py b/meta/lib/oe/cachedpath.py new file mode 100644 index 0000000000..0840cc4c3f --- /dev/null +++ b/meta/lib/oe/cachedpath.py @@ -0,0 +1,233 @@ +# +# Based on standard python library functions but avoid +# repeated stat calls. Its assumed the files will not change from under us +# so we can cache stat calls. +# + +import os +import errno +import stat as statmod + +class CachedPath(object): + def __init__(self): + self.statcache = {} + self.lstatcache = {} + self.normpathcache = {} + return + + def updatecache(self, x): + x = self.normpath(x) + if x in self.statcache: + del self.statcache[x] + if x in self.lstatcache: + del self.lstatcache[x] + + def normpath(self, path): + if path in self.normpathcache: + return self.normpathcache[path] + newpath = os.path.normpath(path) + self.normpathcache[path] = newpath + return newpath + + def _callstat(self, path): + if path in self.statcache: + return self.statcache[path] + try: + st = os.stat(path) + self.statcache[path] = st + return st + except os.error: + self.statcache[path] = False + return False + + # We might as well call lstat and then only + # call stat as well in the symbolic link case + # since this turns out to be much more optimal + # in real world usage of this cache + def callstat(self, path): + path = self.normpath(path) + self.calllstat(path) + return self.statcache[path] + + def calllstat(self, path): + path = self.normpath(path) + if path in self.lstatcache: + return self.lstatcache[path] + #bb.error("LStatpath:" + path) + try: + lst = os.lstat(path) + self.lstatcache[path] = lst + if not statmod.S_ISLNK(lst.st_mode): + self.statcache[path] = lst + else: + self._callstat(path) + return lst + except (os.error, AttributeError): + self.lstatcache[path] = False + self.statcache[path] = False + return False + + # This follows symbolic links, so both islink() and isdir() can be true + # for the same path ono systems that support symlinks + def isfile(self, path): + """Test whether a path is a regular file""" + st = self.callstat(path) + if not st: + return False + return statmod.S_ISREG(st.st_mode) + + # Is a path a directory? + # This follows symbolic links, so both islink() and isdir() + # can be true for the same path on systems that support symlinks + def isdir(self, s): + """Return true if the pathname refers to an existing directory.""" + st = self.callstat(s) + if not st: + return False + return statmod.S_ISDIR(st.st_mode) + + def islink(self, path): + """Test whether a path is a symbolic link""" + st = self.calllstat(path) + if not st: + return False + return statmod.S_ISLNK(st.st_mode) + + # Does a path exist? + # This is false for dangling symbolic links on systems that support them. + def exists(self, path): + """Test whether a path exists. Returns False for broken symbolic links""" + if self.callstat(path): + return True + return False + + def lexists(self, path): + """Test whether a path exists. Returns True for broken symbolic links""" + if self.calllstat(path): + return True + return False + + def stat(self, path): + return self.callstat(path) + + def lstat(self, path): + return self.calllstat(path) + + def walk(self, top, topdown=True, onerror=None, followlinks=False): + # Matches os.walk, not os.path.walk() + + # We may not have read permission for top, in which case we |