diff options
Diffstat (limited to 'scripts/lib/recipetool')
| -rw-r--r-- | scripts/lib/recipetool/append.py | 50 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create.py | 317 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_buildsys.py | 21 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_buildsys_python.py | 78 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_kernel.py | 4 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_npm.py | 204 | ||||
| -rw-r--r-- | scripts/lib/recipetool/newappend.py | 27 | ||||
| -rw-r--r-- | scripts/lib/recipetool/setvar.py | 2 |
8 files changed, 500 insertions, 203 deletions
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py index 5d73d307e0..69c8bb77a0 100644 --- a/scripts/lib/recipetool/append.py +++ b/scripts/lib/recipetool/append.py @@ -48,7 +48,7 @@ def find_target_file(targetpath, d, pkglist=None): """Find the recipe installing the specified target path, optionally limited to a select list of packages""" import json - pkgdata_dir = d.getVar('PKGDATA_DIR', True) + pkgdata_dir = d.getVar('PKGDATA_DIR') # The mix between /etc and ${sysconfdir} here may look odd, but it is just # being consistent with usage elsewhere @@ -97,26 +97,12 @@ def find_target_file(targetpath, d, pkglist=None): recipes[targetpath].append('!%s' % pn) return recipes -def _get_recipe_file(cooker, pn): - import oe.recipeutils - recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) - if not recipefile: - skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) - if skipreasons: - logger.error('\n'.join(skipreasons)) - else: - logger.error("Unable to find any recipe file matching %s" % pn) - return recipefile - def _parse_recipe(pn, tinfoil): - import oe.recipeutils - recipefile = _get_recipe_file(tinfoil.cooker, pn) - if not recipefile: - # Error already logged + try: + rd = tinfoil.parse_recipe(pn) + except bb.providers.NoProvider as e: + logger.error(str(e)) return None - append_files = tinfoil.cooker.collection.get_file_appends(recipefile) - rd = oe.recipeutils.parse_recipe(recipefile, append_files, - tinfoil.config_data) return rd def determine_file_source(targetpath, rd): @@ -124,8 +110,8 @@ def determine_file_source(targetpath, rd): import oe.recipeutils # See if it's in do_install for the recipe - workdir = rd.getVar('WORKDIR', True) - src_uri = rd.getVar('SRC_URI', True) + workdir = rd.getVar('WORKDIR') + src_uri = rd.getVar('SRC_URI') srcfile = '' modpatches = [] elements = check_do_install(rd, targetpath) @@ -135,7 +121,7 @@ def determine_file_source(targetpath, rd): logger.debug('source path: %s' % srcpath) if not srcpath.startswith('/'): # Handle non-absolute path - srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath)) + srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath)) if srcpath.startswith(workdir): # OK, now we have the source file name, look for it in SRC_URI workdirfile = os.path.relpath(srcpath, workdir) @@ -204,22 +190,22 @@ def get_source_path(cmdelements): def get_func_deps(func, d): """Find the function dependencies of a shell function""" - deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True)) - deps |= set((d.getVarFlag(func, "vardeps", True) or "").split()) + deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func)) + deps |= set((d.getVarFlag(func, "vardeps") or "").split()) funcdeps = [] for dep in deps: - if d.getVarFlag(dep, 'func', True): + if d.getVarFlag(dep, 'func'): funcdeps.append(dep) return funcdeps def check_do_install(rd, targetpath): """Look at do_install for a command that installs/copies the specified target path""" - instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/'))) - do_install = rd.getVar('do_install', True) + instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/'))) + do_install = rd.getVar('do_install') # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose) deps = get_func_deps('do_install', rd) for dep in deps: - do_install = do_install.replace(dep, rd.getVar(dep, True)) + do_install = do_install.replace(dep, rd.getVar(dep)) # Look backwards through do_install as we want to catch where a later line (perhaps # from a bbappend) is writing over the top @@ -336,12 +322,12 @@ def appendfile(args): def appendsrc(args, files, rd, extralines=None): import oe.recipeutils - srcdir = rd.getVar('S', True) - workdir = rd.getVar('WORKDIR', True) + srcdir = rd.getVar('S') + workdir = rd.getVar('WORKDIR') import bb.fetch simplified = {} - src_uri = rd.getVar('SRC_URI', True).split() + src_uri = rd.getVar('SRC_URI').split() for uri in src_uri: if uri.endswith(';'): uri = uri[:-1] @@ -354,7 +340,7 @@ def appendsrc(args, files, rd, extralines=None): for newfile, srcfile in files.items(): src_destdir = os.path.dirname(srcfile) if not args.use_workdir: - if rd.getVar('S', True) == rd.getVar('STAGING_KERNEL_DIR', True): + if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'): srcdir = os.path.join(workdir, 'git') if not bb.data.inherits_class('kernel-yocto', rd): logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git') diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index f246028a02..5af58a12f7 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -32,6 +32,18 @@ logger = logging.getLogger('recipetool') tinfoil = None plugins = None +def log_error_cond(message, debugonly): + if debugonly: + logger.debug(message) + else: + logger.error(message) + +def log_info_cond(message, debugonly): + if debugonly: + logger.debug(message) + else: + logger.info(message) + def plugin_init(pluginlist): # Take a reference to the list so we can use it later global plugins @@ -47,6 +59,9 @@ class RecipeHandler(object): recipecmakefilemap = {} recipebinmap = {} + def __init__(self): + self._devtool = False + @staticmethod def load_libmap(d): '''Load library->recipe mapping''' @@ -56,8 +71,8 @@ class RecipeHandler(object): return # First build up library->package mapping shlib_providers = oe.package.read_shlib_providers(d) - libdir = d.getVar('libdir', True) - base_libdir = d.getVar('base_libdir', True) + libdir = d.getVar('libdir') + base_libdir = d.getVar('base_libdir') libpaths = list(set([base_libdir, libdir])) libname_re = re.compile('^lib(.+)\.so.*$') pkglibmap = {} @@ -73,7 +88,7 @@ class RecipeHandler(object): logger.debug('unable to extract library name from %s' % lib) # Now turn it into a library->recipe mapping - pkgdata_dir = d.getVar('PKGDATA_DIR', True) + pkgdata_dir = d.getVar('PKGDATA_DIR') for libname, pkg in pkglibmap.items(): try: with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: @@ -97,9 +112,9 @@ class RecipeHandler(object): '''Build up development file->recipe mapping''' if RecipeHandler.recipeheadermap: return - pkgdata_dir = d.getVar('PKGDATA_DIR', True) - includedir = d.getVar('includedir', True) - cmakedir = os.path.join(d.getVar('libdir', True), 'cmake') + pkgdata_dir = d.getVar('PKGDATA_DIR') + includedir = d.getVar('includedir') + cmakedir = os.path.join(d.getVar('libdir'), 'cmake') for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')): with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: pn = None @@ -128,9 +143,9 @@ class RecipeHandler(object): '''Build up native binary->recipe mapping''' if RecipeHandler.recipebinmap: return - sstate_manifests = d.getVar('SSTATE_MANIFESTS', True) - staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True) - build_arch = d.getVar('BUILD_ARCH', True) + sstate_manifests = d.getVar('SSTATE_MANIFESTS') + staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE') + build_arch = d.getVar('BUILD_ARCH') fileprefix = 'manifest-%s-' % build_arch for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)): with open(fn, 'r') as f: @@ -222,7 +237,8 @@ class RecipeHandler(object): if deps: values['DEPENDS'] = ' '.join(deps) - def genfunction(self, outlines, funcname, content, python=False, forcespace=False): + @staticmethod + def genfunction(outlines, funcname, content, python=False, forcespace=False): if python: prefix = 'python ' else: @@ -256,27 +272,49 @@ def validate_pv(pv): def determine_from_filename(srcfile): """Determine name and version from a filename""" - part = '' - if '.tar.' in srcfile: - namepart = srcfile.split('.tar.')[0].lower() - else: - namepart = os.path.splitext(srcfile)[0].lower() if is_package(srcfile): # Force getting the value from the package metadata return None, None + + if '.tar.' in srcfile: + namepart = srcfile.split('.tar.')[0] else: - splitval = namepart.rsplit('_', 1) + namepart = os.path.splitext(srcfile)[0] + namepart = namepart.lower().replace('_', '-') + if namepart.endswith('.src'): + namepart = namepart[:-4] + if namepart.endswith('.orig'): + namepart = namepart[:-5] + splitval = namepart.split('-') + logger.debug('determine_from_filename: split name %s into: %s' % (srcfile, splitval)) + + ver_re = re.compile('^v?[0-9]') + + pv = None + pn = None if len(splitval) == 1: - splitval = namepart.rsplit('-', 1) - pn = splitval[0].replace('_', '-') - if len(splitval) > 1: - if splitval[1][0] in '0123456789': - pv = splitval[1] + # Try to split the version out if there is no separator (or a .) + res = re.match('^([^0-9]+)([0-9.]+.*)$', namepart) + if res: + if len(res.group(1)) > 1 and len(res.group(2)) > 1: + pn = res.group(1).rstrip('.') + pv = res.group(2) else: - pn = '-'.join(splitval).replace('_', '-') - pv = None + pn = namepart else: - pv = None + if splitval[-1] in ['source', 'src']: + splitval.pop() + if len(splitval) > 2 and re.match('^(alpha|beta|stable|release|rc[0-9]|pre[0-9]|p[0-9]|[0-9]{8})', splitval[-1]) and ver_re.match(splitval[-2]): + pv = '-'.join(splitval[-2:]) + if pv.endswith('-release'): + pv = pv[:-8] + splitval = splitval[:-2] + elif ver_re.match(splitval[-1]): + pv = splitval.pop() + pn = '-'.join(splitval) + if pv and pv.startswith('v'): + pv = pv[1:] + logger.debug('determine_from_filename: name = "%s" version = "%s"' % (pn, pv)) return (pn, pv) def determine_from_url(srcuri): @@ -301,7 +339,7 @@ def determine_from_url(srcuri): pn = res.group(1).strip().replace('_', '-') pv = res.group(2).strip().replace('_', '.') - if not pn and not pv: + if not pn and not pv and parseres.scheme not in ['git', 'gitsm', 'svn', 'hg']: srcfile = os.path.basename(parseres.path.rstrip('/')) pn, pv = determine_from_filename(srcfile) @@ -313,22 +351,30 @@ def supports_srcrev(uri): # This is a bit sad, but if you don't have this set there can be some # odd interactions with the urldata cache which lead to errors localdata.setVar('SRCREV', '${AUTOREV}') - bb.data.update_data(localdata) - fetcher = bb.fetch2.Fetch([uri], localdata) - urldata = fetcher.ud - for u in urldata: - if urldata[u].method.supports_srcrev(): + try: + fetcher = bb.fetch2.Fetch([uri], localdata) + urldata = fetcher.ud + for u in urldata: + if urldata[u].method.supports_srcrev(): + return True + except bb.fetch2.FetchError as e: + logger.debug('FetchError in supports_srcrev: %s' % str(e)) + # Fall back to basic check + if uri.startswith(('git://', 'gitsm://')): return True return False def reformat_git_uri(uri): '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' checkuri = uri.split(';', 1)[0] - if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?', checkuri): - res = re.match('(https?)://([^;]+(\.git)?)(;.*)?$', uri) + if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): + res = re.match('(http|https|ssh)://([^;]+(\.git)?)(;.*)?$', uri) if res: # Need to switch the URI around so that the git fetcher is used return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '') + elif '@' in checkuri: + # Catch e.g. git@git.example.com:repo.git + return 'git://%s;protocol=ssh' % checkuri.replace(':', '/', 1) return uri def is_package(url): @@ -358,7 +404,7 @@ def create_recipe(args): if os.path.isfile(source): source = 'file://%s' % os.path.abspath(source) - if '://' in source: + if scriptutils.is_src_url(source): # Fetch a URL fetchuri = reformat_git_uri(urldefrag(source)[0]) if args.binary: @@ -373,15 +419,14 @@ def create_recipe(args): srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc + d = bb.data.createCopy(tinfoil.config_data) if fetchuri.startswith('npm://'): # Check if npm is available - npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm') - if not npm: - logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager') - sys.exit(1) + npm_bindir = check_npm(tinfoil, args.devtool) + d.prependVar('PATH', '%s:' % npm_bindir) logger.info('Fetching %s...' % srcuri) try: - checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) + checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev) except bb.fetch2.BBFetchException as e: logger.error(str(e).rstrip()) sys.exit(1) @@ -399,6 +444,10 @@ def create_recipe(args): if '<html' in f.read(100).lower(): logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) + if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'): + srcuri = 'gitsm://' + srcuri[6:] + logger.info('Fetching submodules...') + bb.process.run('git submodule update --init --recursive', cwd=srctree) if is_package(fetchuri): tmpfdir = tempfile.mkdtemp(prefix='recipetool-') @@ -416,8 +465,8 @@ def create_recipe(args): if pkgfile: if pkgfile.endswith(('.deb', '.ipk')): - stdout, _ = bb.process.run('ar x %s control.tar.gz' % pkgfile, cwd=tmpfdir) - stdout, _ = bb.process.run('tar xf control.tar.gz ./control', cwd=tmpfdir) + stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir) + stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir) values = convert_debian(tmpfdir) extravalues.update(values) elif pkgfile.endswith(('.rpm', '.srpm')): @@ -446,7 +495,7 @@ def create_recipe(args): for line in stdout.splitlines(): splitline = line.split() if len(splitline) > 1: - if splitline[0] == 'origin' and '://' in splitline[1]: + if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): srcuri = reformat_git_uri(splitline[1]) srcsubdir = 'git' break @@ -477,41 +526,8 @@ def create_recipe(args): # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments lines_before.append('') - licvalues = guess_license(srctree_use) - lic_files_chksum = [] - if licvalues: - licenses = [] - for licvalue in licvalues: - if not licvalue[0] in licenses: - licenses.append(licvalue[0]) - lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) - lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') - lines_before.append('# your responsibility to verify that the values are complete and correct.') - if len(licvalues) > 1: - lines_before.append('#') - lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') - lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') - lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') - lines_before.append('# documentation to determine which situation is applicable.') - else: - lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') - lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') - lines_before.append('#') - lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') - lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') - lines_before.append('# will not be in most cases) you must specify the correct value before using this') - lines_before.append('# recipe for anything other than initial testing/development!') - licenses = ['CLOSED'] - pkg_license = extravalues.pop('LICENSE', None) - if pkg_license: - if licenses == ['Unknown']: - lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata') - licenses = [pkg_license] - else: - lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license) - lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) - lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) - lines_before.append('') + handled = [] + licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data) classes = [] @@ -555,7 +571,6 @@ def create_recipe(args): if name_pv and not realpv: realpv = name_pv - if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) @@ -589,6 +604,11 @@ def create_recipe(args): lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') + if args.fetch_dev: + extravalues['fetchdev'] = True + else: + extravalues['fetchdev'] = None + # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] @@ -605,12 +625,10 @@ def create_recipe(args): handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) + setattr(handler, '_devtool', args.devtool) handlers = [item[0] for item in handlers] # Apply the handlers - handled = [] - handled.append(('license', licvalues)) - if args.binary: classes.append('bin_package') handled.append('buildsystem') @@ -644,12 +662,15 @@ def create_recipe(args): if not outfile: if not pn: - logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile') + log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool) # devtool looks for this specific exit code, so don't change it sys.exit(15) else: - if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')): - outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0]) + if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')): + suffix = srcuri.split(':', 1)[0] + if suffix == 'gitsm': + suffix = 'git' + outfile = '%s_%s.bb' % (pn, suffix) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: @@ -711,6 +732,15 @@ def create_recipe(args): if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') + postinst = ("postinst", extravalues.pop('postinst', None)) + postrm = ("postrm", extravalues.pop('postrm', None)) + preinst = ("preinst", extravalues.pop('preinst', None)) + prerm = ("prerm", extravalues.pop('prerm', None)) + funcs = [postinst, postrm, preinst, prerm] + for func in funcs: + if func[1]: + RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1]) + outlines = [] outlines.extend(lines_before) if classes: @@ -737,26 +767,86 @@ def create_recipe(args): shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None - logger.info('Source extracted to %s' % args.extract_to) + log_info_cond('Source extracted to %s' % args.extract_to, args.devtool) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: - f.write('\n'.join(outlines) + '\n') - logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) + lastline = None + for line in outlines: + if not lastline and not line: + # Skip extra blank lines + continue + f.write('%s\n' % line) + lastline = line + log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) if tempsrc: - shutil.rmtree(tempsrc) + if args.keep_temp: + logger.info('Preserving temporary directory %s' % tempsrc) + else: + shutil.rmtree(tempsrc) return 0 +def handle_license_vars(srctree, lines_before, handled, extravalues, d): + licvalues = guess_license(srctree, d) + lic_files_chksum = [] + lic_unknown = [] + if licvalues: + licenses = [] + for licvalue in licvalues: + if not licvalue[0] in licenses: + licenses.append(licvalue[0]) + lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) + if licvalue[0] == 'Unknown': + lic_unknown.append(licvalue[1]) + lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') + lines_before.append('# your responsibility to verify that the values are complete and correct.') + if len(licvalues) > 1: + lines_before.append('#') + lines_before.append('# NOTE: multiple licenses have been detected; they have been separated with &') + lines_before.append('# in the LICENSE value for now since it is a reasonable assumption that all') + lines_before.append('# of the licenses apply. If instead there is a choice between the multiple') + lines_before.append('# licenses then you should change the value to separate the licenses with |') + lines_before.append('# instead of &. If there is any doubt, check the accompanying documentation') + lines_before.append('# to determine which situation is applicable.') + if lic_unknown: + lines_before.append('#') + lines_before.append('# The following license files were not able to be identified and are') + lines_before.append('# represented as "Unknown" below, you will need to check them yourself:') + for licfile in lic_unknown: + lines_before.append('# %s' % licfile) + lines_before.append('#') + else: + lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') + lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') + lines_before.append('#') + lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') + lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') + lines_before.append('# will not be in most cases) you must specify the correct value before using this') + lines_before.append('# recipe for anything other than initial testing/development!') + licenses = ['CLOSED'] + pkg_license = extravalues.pop('LICENSE', None) + if pkg_license: + if licenses == ['Unknown']: + lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata') + licenses = [pkg_license] + else: + lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license) + lines_before.append('LICENSE = "%s"' % ' & '.join(licenses)) + lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) + lines_before.append('') + handled.append(('license', licvalues)) + return licvalues + def get_license_md5sums(d, static_only=False): import bb.utils md5sums = {} if not static_only: # Gather md5sums of license files in common license dir - commonlicdir = d.getVar('COMMON_LICENSE_DIR', True) + commonlicdir = d.getVar('COMMON_LICENSE_DIR') for fn in os.listdir(commonlicdir): md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn)) md5sums[md5value] = fn @@ -815,7 +905,7 @@ def crunch_license(licfile): # Note: these are carefully constructed! license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') - license_statement_re = re.compile('^This (project|software) is( free software)? released under the .{1,10} [Ll]icen[sc]e:?$') + license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') copyright_re = re.compile('^(#+)? *Copyright .*$') crunched_md5sums = {} @@ -871,9 +961,9 @@ def crunch_license(licfile): license = crunched_md5sums.get(md5val, None) return license, md5val, lictext -def guess_license(srctree): +def guess_license(srctree, d): import bb - md5sums = get_license_md5sums(tinfoil.config_data) + md5sums = get_license_md5sums(d) licenses = [] licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*'] @@ -926,7 +1016,7 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn return outlicenses def read_pkgconfig_provides(d): - pkgdatadir = d.getVar('PKGDATA_DIR', True) + pkgdatadir = d.getVar('PKGDATA_DIR') pkgmap = {} for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')): with open(fn, 'r') as f: @@ -987,6 +1077,25 @@ def convert_debian(debpath): varname = value_map.get(key, None) if varname: values[varname] = value + postinst = os.path.join(debpath, 'postinst') + postrm = os.path.join(debpath, 'postrm') + preinst = os.path.join(debpath, 'preinst') + prerm = os.path.join(debpath, 'prerm') + sfiles = [postinst, postrm, preinst, prerm] + for sfile in sfiles: + if os.path.isfile(sfile): + logger.info("Converting %s file to recipe function..." % + os.path.basename(sfile).upper()) + content = [] + with open(sfile) as f: + for line in f: + if "#!/" in line: + continue + line = line.rstrip("\n") + if line.strip(): + content.append(line) + if content: + values[os.path.basename(f.name)] = content #if depends: # values['DEPENDS'] = ' '.join(depends) @@ -1016,6 +1125,22 @@ def convert_rpm_xml(xmlfile): return values +def check_npm(tinfoil, debugonly=False): + try: + rd = tinfoil.parse_recipe('nodejs-native') + except bb.providers.NoProvider: + # We still conditionally show the message and exit with the special + # return code, otherwise we can't show the proper message for eSDK + # users + log_error_cond('nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs', debugonly) + sys.exit(14) + bindir = rd.getVar('STAGING_BINDIR_NATIVE') + npmpath = os.path.join(bindir, 'npm') + if not os.path.exists(npmpath): + log_error_cond('npm required to process specified source, but npm is not available - you need to run bitbake -c addto_recipe_sysroot nodejs-native first', debugonly) + sys.exit(14) + return bindir + def register_commands(subparsers): parser_create = subparsers.add_parser('create', help='Create a new recipe', @@ -1030,5 +1155,11 @@ def register_commands(subparsers): parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") - parser_create.set_defaults(func=create_recipe) + parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies') + parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) + # FIXME I really hate having to set parserecipes for this, but given we may need + # to call into npm (and we don't know in advance if we will or not) and in order + # to do so we need to know npm's recipe sysroot path, there's not much alternative + parser_create.set_defaults(func=create_recipe, parserecipes=True) diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py index f784f9468a..e914e53aab 100644 --- a/scripts/lib/recipetool/create_buildsys.py +++ b/scripts/lib/recipetool/create_buildsys.py @@ -442,6 +442,7 @@ class AutotoolsRecipeHandler(RecipeHandler): ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') + version_re = re.compile('([0-9.]+)') defines = {} def subst_defines(value): @@ -488,6 +489,7 @@ class AutotoolsRecipeHandler(RecipeHandler): for handler in handlers: if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values): return + logger.debug('Found keyword %s with value "%s"' % (keyword, value)) if keyword == 'PKG_CHECK_MODULES': res = pkg_re.search(value) if res: @@ -569,10 +571,21 @@ class AutotoolsRecipeHandler(RecipeHandler): deps.append('sqlite3') elif keyword == 'AX_LIB_TAGLIB': deps.append('taglib') - elif keyword == 'AX_PKG_SWIG': - deps.append('swig') + elif keyword in ['AX_PKG_SWIG', 'AC_PROG_SWIG']: + deps.append('swig-native') elif keyword == 'AX_PROG_XSLTPROC': deps.append('libxslt-native') + elif keyword in ['AC_PYTHON_DEVEL', 'AX_PYTHON_DEVEL', 'AM_PATH_PYTHON']: + pythonclass = 'pythonnative' + res = version_re.search(value) + if res: + if res.group(1).startswith('3'): + pythonclass = 'python3native' + # Avoid replacing python3native with pythonnative + if not pythonclass in inherits and not 'python3native' in inherits: + if 'pythonnative' in inherits: + inherits.remove('pythonnative') + inherits.append(pythonclass) elif keyword == 'AX_WITH_CURSES': deps.append('ncurses') elif keyword == 'AX_PATH_BDB': @@ -638,7 +651,11 @@ class AutotoolsRecipeHandler(RecipeHandler): 'AX_LIB_SQLITE3', 'AX_LIB_TAGLIB', 'AX_PKG_SWIG', + 'AC_PROG_SWIG', 'AX_PROG_XSLTPROC', + 'AC_PYTHON_DEVEL', + 'AX_PYTHON_DEVEL', + 'AM_PATH_PYTHON', 'AX_WITH_CURSES', 'AX_PATH_BDB', 'AX_PATH_LIB_PCRE', diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index aff13cf1d2..ec5449bee9 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -61,8 +61,6 @@ class PythonRecipeHandler(RecipeHandler): } # PN/PV are already set by recipetool core & desc can be extremely long excluded_fields = [ - 'Name', - 'Version', 'Description', ] setup_parse_map = { @@ -88,8 +86,11 @@ class PythonRecipeHandler(RecipeHandler): ] setuparg_multi_line_values = ['Description'] replacements = [ + ('License', r' +$', ''), + ('License', r'^ +', ''), ('License', r' ', '-'), - ('License', r'-License$', ''), + ('License', r'^GNU-', ''), + ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), ('License', r'^UNKNOWN$', ''), # Remove currently unhandled version numbers from these variables @@ -218,6 +219,9 @@ class PythonRecipeHandler(RecipeHandler): else: info = self.get_setup_args_info(setupscript) + # Grab the license value before applying replacements + license_str = info.get('License', '').strip() + self.apply_info_replacements(info) if uses_setuptools: @@ -225,19 +229,38 @@ class PythonRecipeHandler(RecipeHandler): else: classes.append('distutils') + if license_str: + for i, line in enumerate(lines_before): + if line.startswith('LICENSE = '): + lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) + break + if 'Classifier' in info: + existing_licenses = info.get('License', '') licenses = [] for classifier in info['Classifier']: if classifier in self.classifier_license_map: license = self.classifier_license_map[classifier] + if license == 'Apache' and 'Apache-2.0' in existing_licenses: + license = 'Apache-2.0' + elif license == 'GPL': + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: + license = 'GPL-2.0' + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: + license = 'GPL-3.0' + elif license == 'LGPL': + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: + license = 'LGPL-2.1' + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: + license = 'LGPL-2.0' + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: + license = 'LGPL-3.0' licenses.append(license) if licenses: info['License'] = ' & '.join(licenses) - # Map PKG-INFO & setup.py fields to bitbake variables - bbinfo = {} for field, values in info.items(): if field in self.excluded_fields: continue @@ -251,37 +274,8 @@ class PythonRecipeHandler(RecipeHandler): value = ' '.join(str(v) for v in values if v) bbvar = self.bbvar_map[field] - if bbvar not in bbinfo and value: - bbinfo[bbvar] = value - - comment_lic_line = None - for pos, line in enumerate(list(lines_before)): - if line.startswith('#') and 'LICENSE' in line: - comment_lic_line = pos - elif line.startswith('LICENSE =') and 'LICENSE' in bbinfo: - if line in ('LICENSE = "Unknown"', 'LICENSE = "CLOSED"'): - lines_before[pos] = 'LICENSE = "{}"'.format(bbinfo['LICENSE']) - if line == 'LICENSE = "CLOSED"' and comment_lic_line: - lines_before[comment_lic_line:pos] = [ - '# WARNING: the following LICENSE value is a best guess - it is your', - '# responsibility to verify that the value is complete and correct.' - ] - del bbinfo['LICENSE'] - - src_uri_line = None - for pos, line in enumerate(lines_before): - if line.startswith('SRC_URI ='): - src_uri_line = pos - - if bbinfo: - mdinfo = [''] - for k in sorted(bbinfo): - v = bbinfo[k] - mdinfo.append('{} = "{}"'.format(k, v)) - if src_uri_line: - lines_before[src_uri_line-1:src_uri_line-1] = mdinfo - else: - lines_before.extend(mdinfo) + if bbvar not in extravalues and value: + extravalues[bbvar] = value mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) @@ -504,8 +498,10 @@ class PythonRecipeHandler(RecipeHandler): for dep in scanned_deps: mapped = provided_packages.get(dep) if mapped: + logger.debug('Mapped %s to %s' % (dep, mapped)) mapped_deps.add(mapped) else: + logger.debug('Could not map %s' % dep) unmapped_deps.add(dep) return mapped_deps, unmapped_deps @@ -516,7 +512,7 @@ class PythonRecipeHandler(RecipeHandler): except (OSError, subprocess.CalledProcessError): pass else: - for line in dep_output.decode('utf-8').splitlines(): + for line in dep_output.splitlines(): line = line.rstrip() dep, filename = line.split('\t', 1) if filename.endswith('/setup.py'): @@ -536,11 +532,11 @@ class PythonRecipeHandler(RecipeHandler): def parse_pkgdata_for_python_packages(self): suffixes = [t[0] for t in imp.get_suffixes()] - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') ldata = tinfoil.config_data.createCopy() bb.parse.handle('classes/python-dir.bbclass', ldata, True) - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') python_dirs = [python_sitedir + os.sep, @@ -566,6 +562,8 @@ class PythonRecipeHandler(RecipeHandler): continue if fn.startswith(dynload_dir + os.sep): + if '/.debug/' in fn: + continue base = os.path.basename(fn) provided = base.split('.', 1)[0] packages[provided] = os.path.basename(pkgdatafile) @@ -593,7 +591,7 @@ class PythonRecipeHandler(RecipeHandler): if 'stderr' not in popenargs: popenargs['stderr'] = subprocess.STDOUT try: - return subprocess.check_output(cmd, **popenargs) + return subprocess.check_output(cmd, **popenargs).decode('utf-8') except OSError as exc: logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) raise diff --git a/scripts/lib/recipetool/create_kernel.py b/scripts/lib/recipetool/create_kernel.py index 7dac59fd03..ca4996c7ac 100644 --- a/scripts/lib/recipetool/create_kernel.py +++ b/scripts/lib/recipetool/create_kernel.py @@ -41,7 +41,7 @@ class KernelRecipeHandler(RecipeHandler): handled.append('buildsystem') del lines_after[:] del classes[:] - template = os.path.join(tinfoil.config_data.getVar('COREBASE', True), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb') + template = os.path.join(tinfoil.config_data.getVar('COREBASE'), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb') def handle_var(varname, origvalue, op, newlines): if varname in ['SRCREV', 'SRCREV_machine']: while newlines[-1].startswith('#'): @@ -85,7 +85,7 @@ class KernelRecipeHandler(RecipeHandler): elif varname == 'COMPATIBLE_MACHINE': while newlines[-1].startswith('#'): del newlines[-1] - machine = tinfoil.config_data.getVar('MACHINE', True) + machine = tinfoil.config_data.getVar('MACHINE') return machine, op, 0, True return origvalue, op, 0, True with open(template, 'r') as f: diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index e5aaa60bf8..cb8f338b8b 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -21,7 +21,7 @@ import subprocess import tempfile import shutil import json -from recipetool.create import RecipeHandler, split_pkg_licenses +from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm logger = logging.getLogger('recipetool') @@ -45,11 +45,29 @@ class NpmRecipeHandler(RecipeHandler): license = data['license'] if isinstance(license, dict): license = license.get('type', None) + if license: + if 'OR' in license: + license = license.replace('OR', '|') + license = license.replace('AND', '&') + license = license.replace(' ', '_') + if not license[0] == '(': + license = '(' + license + ')' + print('LICENSE: {}'.format(license)) + else: + license = license.replace('AND', '&') + if license[0] == '(': + license = license[1:] + if license[-1] == ')': + license = license[:-1] + license = license.replace('MIT/X11', 'MIT') + license = license.replace('Public Domain', 'PD') + license = license.replace('SEE LICENSE IN EULA', + 'SEE-LICENSE-IN-EULA') return license - def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before): + def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d): try: - runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) + runenv = dict(os.environ, PATH=d.getVar('PATH')) bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) except bb.process.ExecutionError as e: logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) @@ -61,8 +79,8 @@ class NpmRecipeHandler(RecipeHandler): extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') - def _lockdown(self, srctree, localfilesdir, extravalues, lines_before): - runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) + def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d): + runenv = dict(os.environ, PATH=d.getVar('PATH')) if not NpmRecipeHandler.lockdownpath: NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, @@ -83,6 +101,79 @@ class NpmRecipeHandler(RecipeHandler): extravalues['extrafiles']['lockdown.json'] = tmpfile lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') + def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree): + import scriptutils + # If this isn't a single module we need to get the dependencies + # and add them to SRC_URI + def varfunc(varname, origvalue, op, newlines): + if varname == 'SRC_URI': + if not origvalue.startswith('npm://'): + src_uri = origvalue.split() + changed = False + deplist = {} + for dep, depver in optdeps.items(): + depdata = self.get_npm_data(dep, depver, d) + if self.check_npm_optional_dependency(depdata): + deplist[dep] = depdata + for dep, depver in devdeps.items(): + depdata = self.get_npm_data(dep, depver, d) + if self.check_npm_optional_dependency(depdata): + deplist[dep] = depdata + for dep, depver in deps.items(): + depdata = self.get_npm_data(dep, depver, d) + deplist[dep] = depdata + + for dep, depdata in deplist.items(): + version = depdata.get('version', None) + if version: + url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) + scriptutils.fetch_uri(d, url, srctree) + src_uri.append(url) + changed = True + if changed: + return src_uri, None, -1, True + return origvalue, None, 0, True + updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) + if updated: + del lines_before[:] + for line in newlines: + # Hack to avoid newlines that edit_metadata inserts + if line.endswith('\n'): + line = line[:-1] + lines_before.append(line) + return updated + + def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d): + for item in handled: + if isinstance(item, tuple): + if item[0] == 'license': + del item + break + + calledvars = [] + def varfunc(varname, origvalue, op, newlines): + if varname in ['LICENSE', 'LIC_FILES_CHKSUM']: + for i, e in enumerate(reversed(newlines)): + if not e.startswith('#'): + stop = i + while stop > 0: + newlines.pop() + stop -= 1 + break + calledvars.append(varname) + if len(calledvars) > 1: + # The second time around, put the new license text in + insertpos = len(newlines) + handle_license_vars(srctree, newlines, handled, extravalues, d) + return None, None, 0, True + return origvalue, None, 0, True + updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc) + if updated: + del lines_before[:] + lines_before.extend(newlines) + else: + raise Exception('Did not find license variables') + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): import bb.utils import oe @@ -97,6 +188,10 @@ class NpmRecipeHandler(RecipeHandler): files = RecipeHandler.checkfiles(srctree, ['package.json']) if files: + d = bb.data.createCopy(tinfoil.config_data) + npm_bindir = check_npm(tinfoil, self._devtool) + d.prependVar('PATH', '%s:' % npm_bindir) + data = read_package_json(files[0]) if 'name' in data and 'version' in data: extravalues['PN'] = data['name'] @@ -108,12 +203,19 @@ class NpmRecipeHandler(RecipeHandler): if 'homepage' in data: extravalues['HOMEPAGE'] = data['homepage'] + fetchdev = extravalues['fetchdev'] or None + deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev) + updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree) + if updated: + # We need to redo the license stuff + self._replace_license_vars(srctree, lines_before, handled, extravalues, d) + # Shrinkwrap localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') - self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before) + self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d) # Lockdown - self._lockdown(srctree, localfilesdir, extravalues, lines_before) + self._lockdown(srctree, localfilesdir, extravalues, lines_before, d) # Split each npm module out to is own package npmpackages = oe.package.npm_split_package_dirs(srctree) @@ -139,7 +241,9 @@ class NpmRecipeHandler(RecipeHandler): packages = OrderedDict((x,y[0]) for x,y in npmpackages.items()) packages['${PN}'] = '' pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses) - all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense])) + all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense])) + if '&' in all_licenses: + all_licenses.remove('&') # Go back and update the LICENSE value since we have a bit more # information than when that was written out (and we know all apply # vs. there being a choice, so we can join them with &) @@ -148,9 +252,93 @@ class NpmRecipeHandler(RecipeHandler): lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses) break + # Need to move S setting after inherit npm + for i, line in enumerate(lines_before): + if line.startswith('S ='): + lines_before.pop(i) + lines_after.insert(0, '# Must be set after inherit npm since that itself sets S') + lines_after.insert(1, line) + break + return True return False + # FIXME this is duplicated from lib/bb/fetch2/npm.py + def _parse_view(self, output): + ''' + Parse the output of npm view --json; the last JSON result + is assumed to be the one that we're interested in. + ''' + pdata = None + outdeps = {} + datalines = [] + bracelevel = 0 + for line in output.splitlines(): + if bracelevel: + datalines.append(line) + elif '{' in line: + datalines = [] + datalines.append(line) + bracelevel = bracelevel + line.count('{') - line.count('}') + if datalines: + pdata = json.loads('\n'.join(datalines)) + return pdata + + # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py + # (split out from _getdependencies()) + def get_npm_data(self, pkg, version, d): + import bb.fetch2 + pkgfullname = pkg + if version != '*' and not '/' in version: + pkgfullname += "@'%s'" % version + logger.debug(2, "Calling getdeps on %s" % pkg) + runenv = dict(os.environ, PATH=d.getVar('PATH')) + fetchcmd = "npm view %s --json" % pkgfullname + output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True) + data = self._parse_view(output) + return data + + # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py + # (split out from _getdependencies()) + def get_npm_package_dependencies(self, pdata, fetchdev): + dependencies = pdata.get('dependencies', {}) + optionalDependencies = pdata.get('optionalDependencies', {}) + dependencies.update(optionalDependencies) + if fetchdev: + devDependencies = pdata.get('devDependencies', {}) + dependencies.update(devDependencies) + else: + devDependencies = {} + depsfound = {} + optdepsfound = {} + devdepsfound = {} + for dep in dependencies: + if dep in optionalDependencies: + optdepsfound[dep] = dependencies[dep] + elif dep in devDependencies: + devdepsfound[dep] = dependencies[dep] + else: + depsfound[dep] = dependencies[dep] + return depsfound, optdepsfound, devdepsfound + + # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py + # (split out from _getdependencies()) + def check_npm_optional_dependency(self, pdata): + pkg_os = pdata.get('os', None) + if pkg_os: + if not isinstance(pkg_os, list): + pkg_os = [pkg_os] + blacklist = False + for item in pkg_os: + if item.startswith('!'): + blacklist = True + break + if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: + logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) + return False + return True + + def register_recipe_handlers(handlers): handlers.append((NpmRecipeHandler(), 60)) diff --git a/scripts/lib/recipetool/newappend.py b/scripts/lib/recipetool/newappend.py index 4fbb40a358..0b63759d8c 100644 --- a/scripts/lib/recipetool/newappend.py +++ b/scripts/lib/recipetool/newappend.py @@ -39,28 +39,6 @@ def tinfoil_init(instance): tinfoil = instance -def _provide_to_pn(cooker, provide): - """Get the name of the preferred recipe for the specified provide.""" - import bb.providers - filenames = cooker.recipecache.providers[provide] - eligible, foundUnique = bb.providers.filterProviders(filenames, provide, cooker.expanded_data, cooker.recipecache) - filename = eligible[0] - pn = cooker.recipecache.pkg_fn[filename] - return pn - - -def _get_recipe_file(cooker, pn): - import oe.recipeutils - recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) - if not recipefile: - skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) - if skipreasons: - logger.error('\n'.join(skipreasons)) - else: - logger.error("Unable to find any recipe file matching %s" % pn) - return recipefile - - def layer(layerpath): if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')): raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath)) @@ -70,8 +48,7 @@ def layer(layerpath): def newappend(args): import oe.recipeutils - pn = _provide_to_pn(tinfoil.cooker, args.target) - recipe_path = _get_recipe_file(tinfoil.cooker, pn) + recipe_path = tinfoil.get_recipe_file(args.target) rd = tinfoil.config_data.createCopy() rd.setVar('FILE', recipe_path) @@ -83,7 +60,7 @@ def newappend(args): if not path_ok: logger.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.', os.path.join(args.destlayer, 'conf', 'layer.conf'), os.path.dirname(append_path)) - layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()] + layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] if not os.path.abspath(args.destlayer) in layerdirs: logger.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py index 657d2b6a7b..9de315a0ef 100644 --- a/scripts/lib/recipetool/setvar.py +++ b/scripts/lib/recipetool/setvar.py @@ -51,7 +51,7 @@ def setvar(args): if args.recipe_only: patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)] else: - rd = oe.recipeutils.parse_recipe(args.recipefile, None, tinfoil.config_data) + rd = tinfoil.parse_recipe_file(args.recipefile, False) if not rd: return 1 patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch) |