diff options
Diffstat (limited to 'scripts/lib')
| -rw-r--r-- | scripts/lib/recipetool/create_buildsys_python.py | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index 9e4e1ebd4c..f4f4212559 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -41,6 +41,13 @@ def tinfoil_init(instance): class PythonRecipeHandler(RecipeHandler): + base_pkgdeps = ['python-core'] + excluded_pkgdeps = ['python-dbg'] + # os.path is provided by python-core + assume_provided = ['builtins', 'os.path'] + # Assumes that the host python builtin_module_names is sane for target too + assume_provided = assume_provided + list(sys.builtin_module_names) + bbvar_map = { 'Name': 'PN', 'Version': 'PV', @@ -273,6 +280,8 @@ class PythonRecipeHandler(RecipeHandler): mdinfo.append('{} = "{}"'.format(k, v)) lines_before[src_uri_line-1:src_uri_line-1] = mdinfo + mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) + extras_req = set() if 'Extras-require' in info: extras_req = info['Extras-require'] @@ -284,6 +293,8 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('# Uncomment this line to enable all the optional features.') lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) for feature, feature_reqs in extras_req.iteritems(): + unmapped_deps.difference_update(feature_reqs) + feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) @@ -293,11 +304,34 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('') inst_reqs = info['Install-requires'] if inst_reqs: + unmapped_deps.difference_update(inst_reqs) + inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') lines_after.append('# upstream names may not correspond exactly to bitbake package names.') lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + if mapped_deps: + name = info.get('Name') + if name and name[0] in mapped_deps: + # Attempt to avoid self-reference + mapped_deps.remove(name[0]) + mapped_deps -= set(self.excluded_pkgdeps) + if inst_reqs or extras_req: + lines_after.append('') + lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') + lines_after.append('# python sources, and might not be 100% accurate.') + lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + + unmapped_deps -= set(extensions) + unmapped_deps -= set(self.assume_provided) + if unmapped_deps: + if mapped_deps: + lines_after.append('') + lines_after.append('# WARNING: We were unable to map the following python package/module') + lines_after.append('# dependencies to the bitbake packages which include them:') + lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) + handled.append('buildsystem') def get_pkginfo(self, pkginfo_fn): @@ -425,6 +459,132 @@ class PythonRecipeHandler(RecipeHandler): if value != new_list: info[variable] = new_list + def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): + if 'Package-dir' in setup_info: + package_dir = setup_info['Package-dir'] + else: + package_dir = {} + + class PackageDir(distutils.command.build_py.build_py): + def __init__(self, package_dir): + self.package_dir = package_dir + + pd = PackageDir(package_dir) + to_scan = [] + if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): + if 'Py-modules' in setup_info: + for module in setup_info['Py-modules']: + try: + package, module = module.rsplit('.', 1) + except ValueError: + package, module = '.', module + module_path = os.path.join(pd.get_package_dir(package), module + '.py') + to_scan.append(module_path) + + if 'Packages' in setup_info: + for package in setup_info['Packages']: + to_scan.append(pd.get_package_dir(package)) + + if 'Scripts' in setup_info: + to_scan.extend(setup_info['Scripts']) + else: + logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") + + if not to_scan: + to_scan = ['.'] + + logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) + + provided_packages = self.parse_pkgdata_for_python_packages() + scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) + mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() + for dep in scanned_deps: + mapped = provided_packages.get(dep) + if mapped: + mapped_deps.add(mapped) + else: + unmapped_deps.add(dep) + return mapped_deps, unmapped_deps + + def scan_python_dependencies(self, paths): + deps = set() + try: + dep_output = self.run_command(['pythondeps', '-d'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + for line in dep_output.splitlines(): + line = line.rstrip() + dep, filename = line.split('\t', 1) + if filename.endswith('/setup.py'): + continue + deps.add(dep) + + try: + provides_output = self.run_command(['pythondeps', '-p'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + provides_lines = (l.rstrip() for l in provides_output.splitlines()) + provides = set(l for l in provides_lines if l and l != 'setup') + deps -= provides + + return deps + + def parse_pkgdata_for_python_packages(self): + suffixes = [t[0] for t in imp.get_suffixes()] + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + + ldata = tinfoil.config_data.createCopy() + bb.parse.handle('classes/python-dir.bbclass', ldata, True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True) + + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') + python_dirs = [python_sitedir + os.sep, + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, + os.path.dirname(python_sitedir) + os.sep] + packages = {} + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): + files_info = None + with open(pkgdatafile, 'r') as f: + for line in f.readlines(): + field, value = line.split(': ', 1) + if field == 'FILES_INFO': + files_info = ast.literal_eval(value) + break + else: + continue + + for fn in files_info.iterkeys(): + for suffix in suffixes: + if fn.endswith(suffix): + break + else: + continue + + if fn.startswith(dynload_dir + os.sep): + base = os.path.basename(fn) + provided = base.split('.', 1)[0] + packages[provided] = os.path.basename(pkgdatafile) + continue + + for python_dir in python_dirs: + if fn.startswith(python_dir): + relpath = fn[len(python_dir):] + relstart, _, relremaining = relpath.partition(os.sep) + if relstart.endswith('.egg'): + relpath = relremaining + base, _ = os.path.splitext(relpath) + + if '/.debug/' in base: + continue + if os.path.basename(base) == '__init__': + base = os.path.dirname(base) + base = base.replace(os.sep + os.sep, os.sep) + provided = base.replace(os.sep, '.') + packages[provided] = os.path.basename(pkgdatafile) + return packages + @classmethod def run_command(cls, cmd, **popenargs): if 'stderr' not in popenargs: |