diff options
Diffstat (limited to 'scripts/oe-pkgdata-util')
| -rwxr-xr-x | scripts/oe-pkgdata-util | 564 |
1 files changed, 463 insertions, 101 deletions
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util index c63f87d7e6..6255662a4b 100755 --- a/scripts/oe-pkgdata-util +++ b/scripts/oe-pkgdata-util @@ -1,10 +1,10 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # OpenEmbedded pkgdata utility # # Written by: Paul Eggleton <paul.eggleton@linux.intel.com> # -# Copyright 2012 Intel Corporation +# Copyright 2012-2015 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,56 +19,63 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -# -# Currently only has two functions: -# 1) glob - mapping of packages to their dev/dbg/doc/locale etc. counterparts. -# 2) read-value - mapping of packagenames to their location in -# pkgdata and then returns value of selected variable (e.g. PKGSIZE) -# Could be extended in future to perform other useful querying functions on the -# pkgdata though. -# import sys import os import os.path import fnmatch import re +import argparse +import logging +from collections import defaultdict, OrderedDict -def usage(): - print("syntax: oe-pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"\n \ - read-value [-d] <pkgdatadir> <vendor-os> <value-name> \"<package-name>_<package_architecture>\""); +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('pkgdatautil') +def tinfoil_init(): + import bb.tinfoil + import logging + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(True) + tinfoil.logger.setLevel(logging.WARNING) + return tinfoil -def glob(args): - if len(args) < 4: - usage() - sys.exit(1) - pkgdata_dir = args[0] - target_suffix = args[1] - pkglist_file = args[2] - globs = args[3].split() +def glob(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + globs = [] + for globitem in args.glob: + globs.extend(globitem.split()) - if target_suffix.startswith("-"): - target_suffix = target_suffix[1:] + if not os.path.exists(args.pkglistfile): + logger.error('Unable to find package list file %s' % args.pkglistfile) + sys.exit(1) - skipregex = re.compile("-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-") + skipval = "-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-" + if args.exclude: + skipval += "|" + args.exclude + skipregex = re.compile(skipval) + skippedpkgs = set() mappedpkgs = set() - with open(pkglist_file, 'r') as f: + with open(args.pkglistfile, 'r') as f: for line in f: fields = line.rstrip().split() - if len(fields) < 2: + if not fields: continue pkg = fields[0] - arch = fields[1] - multimach_target_sys = "%s-%s" % (arch, target_suffix) + # We don't care about other args (used to need the package architecture but the + # new pkgdata structure avoids the need for that) # Skip packages for which there is no point applying globs if skipregex.search(pkg): - if debug: - print("%s -> !!" % pkg) + logger.debug("%s -> !!" % pkg) + skippedpkgs.add(pkg) continue # Skip packages that already match the globs, so if e.g. a dev package @@ -80,15 +87,15 @@ def glob(args): already = True break if already: - if debug: - print("%s -> !" % pkg) + skippedpkgs.add(pkg) + logger.debug("%s -> !" % pkg) continue # Define some functions def revpkgdata(pkgn): - return os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkgn) + return os.path.join(args.pkgdata_dir, "runtime-reverse", pkgn) def fwdpkgdata(pkgn): - return os.path.join(pkgdata_dir, multimach_target_sys, "runtime", pkgn) + return os.path.join(args.pkgdata_dir, "runtime", pkgn) def readpn(pkgdata_file): pn = "" with open(pkgdata_file, 'r') as f: @@ -138,92 +145,447 @@ def glob(args): mappedpkg = "" else: # Package doesn't even exist... - if debug: - print "%s is not a valid package!" % (pkg) + logger.debug("%s is not a valid package!" % (pkg)) break if mappedpkg: - if debug: - print "%s (%s) -> %s" % (pkg, g, mappedpkg) + logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg)) mappedpkgs.add(mappedpkg) else: - if debug: - print "%s (%s) -> ?" % (pkg, g) + logger.debug("%s (%s) -> ?" % (pkg, g)) - if debug: - print "------" + logger.debug("------") - print("\n".join(mappedpkgs)) + print("\n".join(mappedpkgs - skippedpkgs)) def read_value(args): - if len(args) < 4: - usage() - sys.exit(1) - - pkgdata_dir = args[0] - target_suffix = args[1] - var = args[2] - packages = args[3].split() - - if target_suffix.startswith("-"): - target_suffix = target_suffix[1:] + # Handle both multiple arguments and multiple values within an arg (old syntax) + packages = [] + if args.file: + with open(args.file, 'r') as f: + for line in f: + splitline = line.split() + if splitline: + packages.append(splitline[0]) + else: + for pkgitem in args.pkg: + packages.extend(pkgitem.split()) + if not packages: + logger.error("No packages specified") + sys.exit(1) - def readvar(pkgdata_file, var): + def readvar(pkgdata_file, valuename, mappedpkg): val = "" with open(pkgdata_file, 'r') as f: for line in f: - if line.startswith(var + ":"): - val = line.split(': ')[1].rstrip() + if (line.startswith(valuename + ":") or + line.startswith(valuename + "_" + mappedpkg + ":")): + val = line.split(': ', 1)[1].rstrip() return val - if debug: - print "read-value('%s', '%s', '%s' '%s'" % (pkgdata_dir, target_suffix, var, packages) + logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, args.valuename, packages)) for package in packages: pkg_split = package.split('_') pkg_name = pkg_split[0] - pkg_arch = '_'.join(pkg_split[1:]) - if debug: - print "package: name: '%s', arch: '%s'" % (pkg_name, pkg_arch) - multimach_target_sys = "%s-%s" % (pkg_arch, target_suffix) - revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name) - if debug: - print(revlink) - if not os.path.exists(revlink): - # [YOCTO #4227] try to drop -gnueabi from TARGET_OS - multimach_target_sys = '-'.join(multimach_target_sys.split('-')[:-1]) - revlink = os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkg_name) - if debug: - print(revlink) + logger.debug("package: '%s'" % pkg_name) + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg_name) + logger.debug(revlink) if os.path.exists(revlink): mappedpkg = os.path.basename(os.readlink(revlink)) - qvar = var + qvar = args.valuename + value = readvar(revlink, qvar, mappedpkg) if qvar == "PKGSIZE": - # append packagename - qvar = "%s_%s" % (var, mappedpkg) - print(readvar(revlink, qvar)) - -# Too lazy to use getopt -debug = False -noopt = False -args = [] -for arg in sys.argv[1:]: - if arg == "--": - noopt = True + # PKGSIZE is now in bytes, but we we want it in KB + pkgsize = (int(value) + 1024 // 2) // 1024 + value = "%d" % pkgsize + if args.prefix_name: + print('%s %s' % (pkg_name, value)) + else: + print(value) + else: + logger.debug("revlink %s does not exist", revlink) + +def lookup_pkglist(pkgs, pkgdata_dir, reverse): + if reverse: + mappings = OrderedDict() + for pkg in pkgs: + revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg) + logger.debug(revlink) + if os.path.exists(revlink): + mappings[pkg] = os.path.basename(os.readlink(revlink)) else: - if not noopt: - if arg == "-d": - debug = True - continue - args.append(arg) - -if len(args) < 1: - usage() - sys.exit(1) - -if args[0] == "glob": - glob(args[1:]) -elif args[0] == "read-value": - read_value(args[1:]) -else: - usage() - sys.exit(1) + mappings = defaultdict(list) + for pkg in pkgs: + pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PKG_%s' % pkg: + mappings[pkg].append(fields[1]) + break + return mappings + +def lookup_pkg(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + pkgs = [] + for pkgitem in args.pkg: + pkgs.extend(pkgitem.split()) + + mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse) + + if len(mappings) < len(pkgs): + missing = list(set(pkgs) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % ', '.join(missing)) + sys.exit(1) + + if args.reverse: + items = list(mappings.values()) + else: + items = [] + for pkg in pkgs: + items.extend(mappings.get(pkg, [])) + + print('\n'.join(items)) + +def lookup_recipe(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + pkgs = [] + for pkgitem in args.pkg: + pkgs.extend(pkgitem.split()) + + mappings = defaultdict(list) + for pkg in pkgs: + pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PN': + mappings[pkg].append(fields[1]) + break + if len(mappings) < len(pkgs): + missing = list(set(pkgs) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % ', '.join(missing)) + sys.exit(1) + + items = [] + for pkg in pkgs: + items.extend(mappings.get(pkg, [])) + print('\n'.join(items)) + +def package_info(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + packages = [] + if args.file: + with open(args.file, 'r') as f: + for line in f: + splitline = line.split() + if splitline: + packages.append(splitline[0]) + else: + for pkgitem in args.pkg: + packages.extend(pkgitem.split()) + if not packages: + logger.error("No packages specified") + sys.exit(1) + + mappings = defaultdict(lambda: defaultdict(str)) + for pkg in packages: + pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0].endswith("_" + pkg): + k = fields[0][:len(fields[0]) - len(pkg) - 1] + else: + k = fields[0] + v = fields[1] if len(fields) == 2 else "" + mappings[pkg][k] = v + + if len(mappings) < len(packages): + missing = list(set(packages) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % + ', '.join(missing)) + sys.exit(1) + + items = [] + for pkg in packages: + pkg_version = mappings[pkg]['PKGV'] + if mappings[pkg]['PKGE']: + pkg_version = mappings[pkg]['PKGE'] + ":" + pkg_version + if mappings[pkg]['PKGR']: + pkg_version = pkg_version + "-" + mappings[pkg]['PKGR'] + recipe = mappings[pkg]['PN'] + recipe_version = mappings[pkg]['PV'] + if mappings[pkg]['PE']: + recipe_version = mappings[pkg]['PE'] + ":" + recipe_version + if mappings[pkg]['PR']: + recipe_version = recipe_version + "-" + mappings[pkg]['PR'] + pkg_size = mappings[pkg]['PKGSIZE'] + + items.append("%s %s %s %s %s" % + (pkg, pkg_version, recipe, recipe_version, pkg_size)) + print('\n'.join(items)) + +def get_recipe_pkgs(pkgdata_dir, recipe, unpackaged): + recipedatafile = os.path.join(pkgdata_dir, recipe) + if not os.path.exists(recipedatafile): + logger.error("Unable to find packaged recipe with name %s" % recipe) + sys.exit(1) + packages = [] + with open(recipedatafile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PACKAGES': + packages = fields[1].split() + break + + if not unpackaged: + pkglist = [] + for pkg in packages: + if os.path.exists(os.path.join(pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + pkglist.append(pkg) + return pkglist + else: + return packages + +def list_pkgs(args): + found = False + + def matchpkg(pkg): + if args.pkgspec: + matched = False + for pkgspec in args.pkgspec: + if fnmatch.fnmatchcase(pkg, pkgspec): + matched = True + break + if not matched: + return False + if not args.unpackaged: + if args.runtime: + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if os.path.exists(revlink): + # We're unlikely to get here if the package was not packaged, but just in case + # we add the symlinks for unpackaged files in the future + mappedpkg = os.path.basename(os.readlink(revlink)) + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)): + return False + else: + return False + else: + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + return False + return True + + if args.recipe: + packages = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged) + + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = packages + + for pkg in pkglist: + if matchpkg(pkg): + found = True + print("%s" % pkg) + else: + if args.runtime: + searchdir = 'runtime-reverse' + else: + searchdir = 'runtime' + + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)): + for fn in files: + if fn.endswith('.packaged'): + continue + if matchpkg(fn): + found = True + print("%s" % fn) + if not found: + if args.pkgspec: + logger.error("Unable to find any package matching %s" % args.pkgspec) + else: + logger.error("No packages found") + sys.exit(1) + +def list_pkg_files(args): + import json + + if args.recipe: + if args.pkg: + logger.error("list-pkg-files: If -p/--recipe is specified then a package name cannot be specified") + sys.exit(1) + recipepkglist = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged) + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(recipepkglist, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = recipepkglist + else: + if not args.pkg: + logger.error("list-pkg-files: If -p/--recipe is not specified then at least one package name must be specified") + sys.exit(1) + pkglist = args.pkg + + for pkg in sorted(pkglist): + print("%s:" % pkg) + if args.runtime: + pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if not os.path.exists(pkgdatafile): + if args.recipe: + # This package was empty and thus never packaged, ignore + continue + logger.error("Unable to find any built runtime package named %s" % pkg) + sys.exit(1) + else: + pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", pkg) + if not os.path.exists(pkgdatafile): + logger.error("Unable to find any built recipe-space package named %s" % pkg) + sys.exit(1) + + with open(pkgdatafile, 'r') as f: + found = False + for line in f: + if line.startswith('FILES_INFO:'): + found = True + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in sorted(dictval): + print("\t%s" % fullpth) + break + if not found: + logger.error("Unable to find FILES_INFO entry in %s" % pkgdatafile) + sys.exit(1) + +def find_path(args): + import json + + found = False + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')): + for fn in files: + with open(os.path.join(root,fn)) as f: + for line in f: + if line.startswith('FILES_INFO:'): + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in dictval.keys(): + if fnmatch.fnmatchcase(fullpth, args.targetpath): + found = True + print("%s: %s" % (fn, fullpth)) + break + if not found: + logger.error("Unable to find any package producing path %s" % args.targetpath) + sys.exit(1) + + +def main(): + parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package", + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + subparsers.required = True + + parser_lookup_pkg = subparsers.add_parser('lookup-pkg', + help='Translate between recipe-space package names and runtime package names', + description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.') + parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up') + parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true') + parser_lookup_pkg.set_defaults(func=lookup_pkg) + + parser_list_pkgs = subparsers.add_parser('list-pkgs', + help='List packages', + description='Lists packages that have been built') + parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe') + parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true') + parser_list_pkgs.set_defaults(func=list_pkgs) + + parser_list_pkg_files = subparsers.add_parser('list-pkg-files', + help='List files within a package', + description='Lists files included in one or more packages') + parser_list_pkg_files.add_argument('pkg', nargs='*', help='Package name to report on (if -p/--recipe is not specified)') + parser_list_pkg_files.add_argument('-r', '--runtime', help='Specified package(s) are runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkg_files.add_argument('-p', '--recipe', help='Report on all packages produced by the specified recipe') + parser_list_pkg_files.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages (only useful with -p/--recipe)', action='store_true') + parser_list_pkg_files.set_defaults(func=list_pkg_files) + + parser_lookup_recipe = subparsers.add_parser('lookup-recipe', + help='Find recipe producing one or more packages', + description='Looks up the specified runtime package(s) to see which recipe they were produced by') + parser_lookup_recipe.add_argument('pkg', nargs='+', help='Runtime package name to look up') + parser_lookup_recipe.set_defaults(func=lookup_recipe) + + parser_package_info = subparsers.add_parser('package-info', + help='Show version, recipe and size information for one or more packages', + description='Looks up the specified runtime package(s) and display information') + parser_package_info.add_argument('pkg', nargs='*', help='Runtime package name to look up') + parser_package_info.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)') + parser_package_info.set_defaults(func=package_info) + + parser_find_path = subparsers.add_parser('find-path', + help='Find package providing a target path', + description='Finds the recipe-space package providing the specified target path') + parser_find_path.add_argument('targetpath', help='Path to find (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_find_path.set_defaults(func=find_path) + + parser_read_value = subparsers.add_parser('read-value', + help='Read any pkgdata value for one or more packages', + description='Reads the named value from the pkgdata files for the specified packages') + parser_read_value.add_argument('valuename', help='Name of the value to look up') + parser_read_value.add_argument('pkg', nargs='*', help='Runtime package name to look up') + parser_read_value.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)') + parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output with package name', action='store_true') + parser_read_value.set_defaults(func=read_value) + + parser_glob = subparsers.add_parser('glob', + help='Expand package name glob expression', + description='Expands one or more glob expressions over the packages listed in pkglistfile') + parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)') + parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev') + parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation') + parser_glob.set_defaults(func=glob) + + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + + if not args.pkgdata_dir: + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + logger.debug('Found bitbake path: %s' % bitbakepath) + tinfoil = tinfoil_init() + try: + args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + finally: + tinfoil.shutdown() + logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir) + if not args.pkgdata_dir: + logger.error('Unable to determine pkgdata directory from PKGDATA_DIR') + sys.exit(1) + + if not os.path.exists(args.pkgdata_dir): + logger.error('Unable to find pkgdata directory %s' % args.pkgdata_dir) + sys.exit(1) + + ret = args.func(args) + + return ret + + +if __name__ == "__main__": + main() |