diff options
Diffstat (limited to 'scripts/oe-pkgdata-util')
| -rwxr-xr-x | scripts/oe-pkgdata-util | 565 |
1 files changed, 412 insertions, 153 deletions
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util index bf8754749e..6255662a4b 100755 --- a/scripts/oe-pkgdata-util +++ b/scripts/oe-pkgdata-util @@ -1,10 +1,10 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # OpenEmbedded pkgdata utility # # Written by: Paul Eggleton <paul.eggleton@linux.intel.com> # -# Copyright 2012-2013 Intel Corporation +# Copyright 2012-2015 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -25,30 +25,45 @@ import os import os.path import fnmatch import re -import optparse -from collections import defaultdict - -def glob(args, usage, debug=False): - if len(args) < 3: - usage() - sys.exit(1) - - pkgdata_dir = args[0] - pkglist_file = args[1] - globs = args[2].split() - - if not os.path.exists(pkgdata_dir): - print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir) +import argparse +import logging +from collections import defaultdict, OrderedDict + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('pkgdatautil') + +def tinfoil_init(): + import bb.tinfoil + import logging + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(True) + + tinfoil.logger.setLevel(logging.WARNING) + return tinfoil + + +def glob(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + globs = [] + for globitem in args.glob: + globs.extend(globitem.split()) + + if not os.path.exists(args.pkglistfile): + logger.error('Unable to find package list file %s' % args.pkglistfile) sys.exit(1) - if not os.path.exists(pkglist_file): - print('ERROR: Unable to find package list file %s' % pkglist_file) - sys.exit(1) - - skipregex = re.compile("-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-") + skipval = "-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-" + if args.exclude: + skipval += "|" + args.exclude + skipregex = re.compile(skipval) + skippedpkgs = set() mappedpkgs = set() - with open(pkglist_file, 'r') as f: + with open(args.pkglistfile, 'r') as f: for line in f: fields = line.rstrip().split() if not fields: @@ -59,8 +74,8 @@ def glob(args, usage, debug=False): # Skip packages for which there is no point applying globs if skipregex.search(pkg): - if debug: - print("%s -> !!" % pkg) + logger.debug("%s -> !!" % pkg) + skippedpkgs.add(pkg) continue # Skip packages that already match the globs, so if e.g. a dev package @@ -72,15 +87,15 @@ def glob(args, usage, debug=False): already = True break if already: - if debug: - print("%s -> !" % pkg) + skippedpkgs.add(pkg) + logger.debug("%s -> !" % pkg) continue # Define some functions def revpkgdata(pkgn): - return os.path.join(pkgdata_dir, "runtime-reverse", pkgn) + return os.path.join(args.pkgdata_dir, "runtime-reverse", pkgn) def fwdpkgdata(pkgn): - return os.path.join(pkgdata_dir, "runtime", pkgn) + return os.path.join(args.pkgdata_dir, "runtime", pkgn) def readpn(pkgdata_file): pn = "" with open(pkgdata_file, 'r') as f: @@ -130,146 +145,331 @@ def glob(args, usage, debug=False): mappedpkg = "" else: # Package doesn't even exist... - if debug: - print "%s is not a valid package!" % (pkg) + logger.debug("%s is not a valid package!" % (pkg)) break if mappedpkg: - if debug: - print "%s (%s) -> %s" % (pkg, g, mappedpkg) + logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg)) mappedpkgs.add(mappedpkg) else: - if debug: - print "%s (%s) -> ?" % (pkg, g) + logger.debug("%s (%s) -> ?" % (pkg, g)) - if debug: - print "------" + logger.debug("------") - print("\n".join(mappedpkgs)) - -def read_value(args, usage, debug=False): - if len(args) < 3: - usage() - sys.exit(1) + print("\n".join(mappedpkgs - skippedpkgs)) - pkgdata_dir = args[0] - var = args[1] - packages = args[2].split() - - if not os.path.exists(pkgdata_dir): - print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir) - sys.exit(1) +def read_value(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + packages = [] + if args.file: + with open(args.file, 'r') as f: + for line in f: + splitline = line.split() + if splitline: + packages.append(splitline[0]) + else: + for pkgitem in args.pkg: + packages.extend(pkgitem.split()) + if not packages: + logger.error("No packages specified") + sys.exit(1) - def readvar(pkgdata_file, var): + def readvar(pkgdata_file, valuename, mappedpkg): val = "" with open(pkgdata_file, 'r') as f: for line in f: - if line.startswith(var + ":"): - val = line.split(': ')[1].rstrip() + if (line.startswith(valuename + ":") or + line.startswith(valuename + "_" + mappedpkg + ":")): + val = line.split(': ', 1)[1].rstrip() return val - if debug: - print "read-value('%s', '%s' '%s'" % (pkgdata_dir, var, packages) + logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, args.valuename, packages)) for package in packages: pkg_split = package.split('_') pkg_name = pkg_split[0] - if debug: - print "package: '%s'" % pkg_name - revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg_name) - if debug: - print(revlink) + logger.debug("package: '%s'" % pkg_name) + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg_name) + logger.debug(revlink) if os.path.exists(revlink): mappedpkg = os.path.basename(os.readlink(revlink)) - qvar = var + qvar = args.valuename + value = readvar(revlink, qvar, mappedpkg) if qvar == "PKGSIZE": - # append packagename - qvar = "%s_%s" % (var, mappedpkg) # PKGSIZE is now in bytes, but we we want it in KB - pkgsize = (int(readvar(revlink, qvar)) + 1024 // 2) // 1024 - print("%d" % pkgsize) + pkgsize = (int(value) + 1024 // 2) // 1024 + value = "%d" % pkgsize + if args.prefix_name: + print('%s %s' % (pkg_name, value)) else: - print(readvar(revlink, qvar)) + print(value) + else: + logger.debug("revlink %s does not exist", revlink) + +def lookup_pkglist(pkgs, pkgdata_dir, reverse): + if reverse: + mappings = OrderedDict() + for pkg in pkgs: + revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg) + logger.debug(revlink) + if os.path.exists(revlink): + mappings[pkg] = os.path.basename(os.readlink(revlink)) + else: + mappings = defaultdict(list) + for pkg in pkgs: + pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PKG_%s' % pkg: + mappings[pkg].append(fields[1]) + break + return mappings -def lookup_pkg(args, usage, debug=False): - if len(args) < 2: - usage() - sys.exit(1) +def lookup_pkg(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + pkgs = [] + for pkgitem in args.pkg: + pkgs.extend(pkgitem.split()) - pkgdata_dir = args[0] - pkgs = args[1].split() + mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse) - if not os.path.exists(pkgdata_dir): - print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir) + if len(mappings) < len(pkgs): + missing = list(set(pkgs) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % ', '.join(missing)) sys.exit(1) + if args.reverse: + items = list(mappings.values()) + else: + items = [] + for pkg in pkgs: + items.extend(mappings.get(pkg, [])) + + print('\n'.join(items)) + +def lookup_recipe(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + pkgs = [] + for pkgitem in args.pkg: + pkgs.extend(pkgitem.split()) + mappings = defaultdict(list) for pkg in pkgs: - pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg) + pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg) if os.path.exists(pkgfile): with open(pkgfile, 'r') as f: for line in f: fields = line.rstrip().split(': ') - if fields[0] == 'PKG_%s' % pkg: + if fields[0] == 'PN': mappings[pkg].append(fields[1]) break if len(mappings) < len(pkgs): missing = list(set(pkgs) - set(mappings.keys())) - sys.stderr.write("ERROR: the following packages could not be found: %s\n" % ', '.join(missing)) + logger.error("The following packages could not be found: %s" % ', '.join(missing)) sys.exit(1) items = [] for pkg in pkgs: items.extend(mappings.get(pkg, [])) - print '\n'.join(items) + print('\n'.join(items)) -def lookup_recipe(args, usage, debug=False): - if len(args) < 2: - usage() - sys.exit(1) - - pkgdata_dir = args[0] - pkgs = args[1].split() - - if not os.path.exists(pkgdata_dir): - print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir) - sys.exit(1) - - mappings = defaultdict(list) - for pkg in pkgs: - pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg) +def package_info(args): + # Handle both multiple arguments and multiple values within an arg (old syntax) + packages = [] + if args.file: + with open(args.file, 'r') as f: + for line in f: + splitline = line.split() + if splitline: + packages.append(splitline[0]) + else: + for pkgitem in args.pkg: + packages.extend(pkgitem.split()) + if not packages: + logger.error("No packages specified") + sys.exit(1) + + mappings = defaultdict(lambda: defaultdict(str)) + for pkg in packages: + pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg) if os.path.exists(pkgfile): with open(pkgfile, 'r') as f: for line in f: fields = line.rstrip().split(': ') - if fields[0] == 'PN': - mappings[pkg].append(fields[1]) - break - if len(mappings) < len(pkgs): - missing = list(set(pkgs) - set(mappings.keys())) - sys.stderr.write("ERROR: the following packages could not be found: %s\n" % ', '.join(missing)) + if fields[0].endswith("_" + pkg): + k = fields[0][:len(fields[0]) - len(pkg) - 1] + else: + k = fields[0] + v = fields[1] if len(fields) == 2 else "" + mappings[pkg][k] = v + + if len(mappings) < len(packages): + missing = list(set(packages) - set(mappings.keys())) + logger.error("The following packages could not be found: %s" % + ', '.join(missing)) sys.exit(1) items = [] - for pkg in pkgs: - items.extend(mappings.get(pkg, [])) - print '\n'.join(items) - -def find_path(args, usage, debug=False): - if len(args) < 2: - usage() + for pkg in packages: + pkg_version = mappings[pkg]['PKGV'] + if mappings[pkg]['PKGE']: + pkg_version = mappings[pkg]['PKGE'] + ":" + pkg_version + if mappings[pkg]['PKGR']: + pkg_version = pkg_version + "-" + mappings[pkg]['PKGR'] + recipe = mappings[pkg]['PN'] + recipe_version = mappings[pkg]['PV'] + if mappings[pkg]['PE']: + recipe_version = mappings[pkg]['PE'] + ":" + recipe_version + if mappings[pkg]['PR']: + recipe_version = recipe_version + "-" + mappings[pkg]['PR'] + pkg_size = mappings[pkg]['PKGSIZE'] + + items.append("%s %s %s %s %s" % + (pkg, pkg_version, recipe, recipe_version, pkg_size)) + print('\n'.join(items)) + +def get_recipe_pkgs(pkgdata_dir, recipe, unpackaged): + recipedatafile = os.path.join(pkgdata_dir, recipe) + if not os.path.exists(recipedatafile): + logger.error("Unable to find packaged recipe with name %s" % recipe) sys.exit(1) + packages = [] + with open(recipedatafile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PACKAGES': + packages = fields[1].split() + break + + if not unpackaged: + pkglist = [] + for pkg in packages: + if os.path.exists(os.path.join(pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + pkglist.append(pkg) + return pkglist + else: + return packages - pkgdata_dir = args[0] - targetpath = args[1] +def list_pkgs(args): + found = False - if not os.path.exists(pkgdata_dir): - print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir) + def matchpkg(pkg): + if args.pkgspec: + matched = False + for pkgspec in args.pkgspec: + if fnmatch.fnmatchcase(pkg, pkgspec): + matched = True + break + if not matched: + return False + if not args.unpackaged: + if args.runtime: + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if os.path.exists(revlink): + # We're unlikely to get here if the package was not packaged, but just in case + # we add the symlinks for unpackaged files in the future + mappedpkg = os.path.basename(os.readlink(revlink)) + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)): + return False + else: + return False + else: + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + return False + return True + + if args.recipe: + packages = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged) + + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = packages + + for pkg in pkglist: + if matchpkg(pkg): + found = True + print("%s" % pkg) + else: + if args.runtime: + searchdir = 'runtime-reverse' + else: + searchdir = 'runtime' + + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)): + for fn in files: + if fn.endswith('.packaged'): + continue + if matchpkg(fn): + found = True + print("%s" % fn) + if not found: + if args.pkgspec: + logger.error("Unable to find any package matching %s" % args.pkgspec) + else: + logger.error("No packages found") sys.exit(1) +def list_pkg_files(args): + import json + + if args.recipe: + if args.pkg: + logger.error("list-pkg-files: If -p/--recipe is specified then a package name cannot be specified") + sys.exit(1) + recipepkglist = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged) + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(recipepkglist, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = recipepkglist + else: + if not args.pkg: + logger.error("list-pkg-files: If -p/--recipe is not specified then at least one package name must be specified") + sys.exit(1) + pkglist = args.pkg + + for pkg in sorted(pkglist): + print("%s:" % pkg) + if args.runtime: + pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if not os.path.exists(pkgdatafile): + if args.recipe: + # This package was empty and thus never packaged, ignore + continue + logger.error("Unable to find any built runtime package named %s" % pkg) + sys.exit(1) + else: + pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", pkg) + if not os.path.exists(pkgdatafile): + logger.error("Unable to find any built recipe-space package named %s" % pkg) + sys.exit(1) + + with open(pkgdatafile, 'r') as f: + found = False + for line in f: + if line.startswith('FILES_INFO:'): + found = True + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in sorted(dictval): + print("\t%s" % fullpth) + break + if not found: + logger.error("Unable to find FILES_INFO entry in %s" % pkgdatafile) + sys.exit(1) + +def find_path(args): import json - import fnmatch - for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')): + found = False + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')): for fn in files: with open(os.path.join(root,fn)) as f: for line in f: @@ -277,55 +477,114 @@ def find_path(args, usage, debug=False): val = line.split(':', 1)[1].strip() dictval = json.loads(val) for fullpth in dictval.keys(): - if fnmatch.fnmatchcase(fullpth, targetpath): + if fnmatch.fnmatchcase(fullpth, args.targetpath): + found = True print("%s: %s" % (fn, fullpth)) break + if not found: + logger.error("Unable to find any package producing path %s" % args.targetpath) + sys.exit(1) def main(): - parser = optparse.OptionParser( - usage = '''%prog [options] <command> <arguments> - -Available commands: - glob <pkgdatadir> <pkglistfile> "<globs>" - expand one or more glob expressions over the packages listed in - pkglistfile (one package per line) - lookup-pkg <pkgdatadir> "<recipe-pkgs>" - look up the specified recipe-space package name(s) to see what the - final runtime package name is (e.g. eglibc becomes libc6) - lookup-recipe <pkgdatadir> "<pkgs>" - look up the specified package(s) to see which recipe they were - produced by - find-path <pkgdatadir> <path> - find the package providing the specified path (wildcards * ? allowed) - read-value <pkgdatadir> <value-name> "<pkgs>" - read the named value from the pkgdata files for the specified - packages''') - - parser.add_option("-d", "--debug", - help = "Enable debug output", - action="store_true", dest="debug", default=False) - - options, args = parser.parse_args(sys.argv) - args = args[1:] - - if len(args) < 1: - parser.print_help() + parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package", + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + subparsers.required = True + + parser_lookup_pkg = subparsers.add_parser('lookup-pkg', + help='Translate between recipe-space package names and runtime package names', + description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.') + parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up') + parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true') + parser_lookup_pkg.set_defaults(func=lookup_pkg) + + parser_list_pkgs = subparsers.add_parser('list-pkgs', + help='List packages', + description='Lists packages that have been built') + parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe') + parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true') + parser_list_pkgs.set_defaults(func=list_pkgs) + + parser_list_pkg_files = subparsers.add_parser('list-pkg-files', + help='List files within a package', + description='Lists files included in one or more packages') + parser_list_pkg_files.add_argument('pkg', nargs='*', help='Package name to report on (if -p/--recipe is not specified)') + parser_list_pkg_files.add_argument('-r', '--runtime', help='Specified package(s) are runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkg_files.add_argument('-p', '--recipe', help='Report on all packages produced by the specified recipe') + parser_list_pkg_files.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages (only useful with -p/--recipe)', action='store_true') + parser_list_pkg_files.set_defaults(func=list_pkg_files) + + parser_lookup_recipe = subparsers.add_parser('lookup-recipe', + help='Find recipe producing one or more packages', + description='Looks up the specified runtime package(s) to see which recipe they were produced by') + parser_lookup_recipe.add_argument('pkg', nargs='+', help='Runtime package name to look up') + parser_lookup_recipe.set_defaults(func=lookup_recipe) + + parser_package_info = subparsers.add_parser('package-info', + help='Show version, recipe and size information for one or more packages', + description='Looks up the specified runtime package(s) and display information') + parser_package_info.add_argument('pkg', nargs='*', help='Runtime package name to look up') + parser_package_info.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)') + parser_package_info.set_defaults(func=package_info) + + parser_find_path = subparsers.add_parser('find-path', + help='Find package providing a target path', + description='Finds the recipe-space package providing the specified target path') + parser_find_path.add_argument('targetpath', help='Path to find (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_find_path.set_defaults(func=find_path) + + parser_read_value = subparsers.add_parser('read-value', + help='Read any pkgdata value for one or more packages', + description='Reads the named value from the pkgdata files for the specified packages') + parser_read_value.add_argument('valuename', help='Name of the value to look up') + parser_read_value.add_argument('pkg', nargs='*', help='Runtime package name to look up') + parser_read_value.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)') + parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output with package name', action='store_true') + parser_read_value.set_defaults(func=read_value) + + parser_glob = subparsers.add_parser('glob', + help='Expand package name glob expression', + description='Expands one or more glob expressions over the packages listed in pkglistfile') + parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)') + parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev') + parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation') + parser_glob.set_defaults(func=glob) + + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + + if not args.pkgdata_dir: + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + logger.debug('Found bitbake path: %s' % bitbakepath) + tinfoil = tinfoil_init() + try: + args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + finally: + tinfoil.shutdown() + logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir) + if not args.pkgdata_dir: + logger.error('Unable to determine pkgdata directory from PKGDATA_DIR') + sys.exit(1) + + if not os.path.exists(args.pkgdata_dir): + logger.error('Unable to find pkgdata directory %s' % args.pkgdata_dir) sys.exit(1) - if args[0] == "glob": - glob(args[1:], parser.print_help, options.debug) - elif args[0] == "lookup-pkg": - lookup_pkg(args[1:], parser.print_help, options.debug) - elif args[0] == "lookup-recipe": - lookup_recipe(args[1:], parser.print_help, options.debug) - elif args[0] == "find-path": - find_path(args[1:], parser.print_help, options.debug) - elif args[0] == "read-value": - read_value(args[1:], parser.print_help, options.debug) - else: - parser.print_help() - sys.exit(1) + ret = args.func(args) + + return ret if __name__ == "__main__": |