summaryrefslogtreecommitdiff
path: root/scripts/lib/recipetool
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib/recipetool')
-rw-r--r--scripts/lib/recipetool/append.py50
-rw-r--r--scripts/lib/recipetool/create.py365
-rw-r--r--scripts/lib/recipetool/create_buildsys.py37
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py78
-rw-r--r--scripts/lib/recipetool/create_kernel.py6
-rw-r--r--scripts/lib/recipetool/create_kmod.py4
-rw-r--r--scripts/lib/recipetool/create_npm.py206
-rw-r--r--scripts/lib/recipetool/newappend.py27
-rw-r--r--scripts/lib/recipetool/setvar.py2
9 files changed, 524 insertions, 251 deletions
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index 5d73d307e0..69c8bb77a0 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -48,7 +48,7 @@ def find_target_file(targetpath, d, pkglist=None):
"""Find the recipe installing the specified target path, optionally limited to a select list of packages"""
import json
- pkgdata_dir = d.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
# The mix between /etc and ${sysconfdir} here may look odd, but it is just
# being consistent with usage elsewhere
@@ -97,26 +97,12 @@ def find_target_file(targetpath, d, pkglist=None):
recipes[targetpath].append('!%s' % pn)
return recipes
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
def _parse_recipe(pn, tinfoil):
- import oe.recipeutils
- recipefile = _get_recipe_file(tinfoil.cooker, pn)
- if not recipefile:
- # Error already logged
+ try:
+ rd = tinfoil.parse_recipe(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
return None
- append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
- rd = oe.recipeutils.parse_recipe(recipefile, append_files,
- tinfoil.config_data)
return rd
def determine_file_source(targetpath, rd):
@@ -124,8 +110,8 @@ def determine_file_source(targetpath, rd):
import oe.recipeutils
# See if it's in do_install for the recipe
- workdir = rd.getVar('WORKDIR', True)
- src_uri = rd.getVar('SRC_URI', True)
+ workdir = rd.getVar('WORKDIR')
+ src_uri = rd.getVar('SRC_URI')
srcfile = ''
modpatches = []
elements = check_do_install(rd, targetpath)
@@ -135,7 +121,7 @@ def determine_file_source(targetpath, rd):
logger.debug('source path: %s' % srcpath)
if not srcpath.startswith('/'):
# Handle non-absolute path
- srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath))
+ srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
if srcpath.startswith(workdir):
# OK, now we have the source file name, look for it in SRC_URI
workdirfile = os.path.relpath(srcpath, workdir)
@@ -204,22 +190,22 @@ def get_source_path(cmdelements):
def get_func_deps(func, d):
"""Find the function dependencies of a shell function"""
- deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
- deps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+ deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
+ deps |= set((d.getVarFlag(func, "vardeps") or "").split())
funcdeps = []
for dep in deps:
- if d.getVarFlag(dep, 'func', True):
+ if d.getVarFlag(dep, 'func'):
funcdeps.append(dep)
return funcdeps
def check_do_install(rd, targetpath):
"""Look at do_install for a command that installs/copies the specified target path"""
- instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/')))
- do_install = rd.getVar('do_install', True)
+ instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/')))
+ do_install = rd.getVar('do_install')
# Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
deps = get_func_deps('do_install', rd)
for dep in deps:
- do_install = do_install.replace(dep, rd.getVar(dep, True))
+ do_install = do_install.replace(dep, rd.getVar(dep))
# Look backwards through do_install as we want to catch where a later line (perhaps
# from a bbappend) is writing over the top
@@ -336,12 +322,12 @@ def appendfile(args):
def appendsrc(args, files, rd, extralines=None):
import oe.recipeutils
- srcdir = rd.getVar('S', True)
- workdir = rd.getVar('WORKDIR', True)
+ srcdir = rd.getVar('S')
+ workdir = rd.getVar('WORKDIR')
import bb.fetch
simplified = {}
- src_uri = rd.getVar('SRC_URI', True).split()
+ src_uri = rd.getVar('SRC_URI').split()
for uri in src_uri:
if uri.endswith(';'):
uri = uri[:-1]
@@ -354,7 +340,7 @@ def appendsrc(args, files, rd, extralines=None):
for newfile, srcfile in files.items():
src_destdir = os.path.dirname(srcfile)
if not args.use_workdir:
- if rd.getVar('S', True) == rd.getVar('STAGING_KERNEL_DIR', True):
+ if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
srcdir = os.path.join(workdir, 'git')
if not bb.data.inherits_class('kernel-yocto', rd):
logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 129742807f..5af58a12f7 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -32,6 +32,18 @@ logger = logging.getLogger('recipetool')
tinfoil = None
plugins = None
+def log_error_cond(message, debugonly):
+ if debugonly:
+ logger.debug(message)
+ else:
+ logger.error(message)
+
+def log_info_cond(message, debugonly):
+ if debugonly:
+ logger.debug(message)
+ else:
+ logger.info(message)
+
def plugin_init(pluginlist):
# Take a reference to the list so we can use it later
global plugins
@@ -47,6 +59,9 @@ class RecipeHandler(object):
recipecmakefilemap = {}
recipebinmap = {}
+ def __init__(self):
+ self._devtool = False
+
@staticmethod
def load_libmap(d):
'''Load library->recipe mapping'''
@@ -56,8 +71,8 @@ class RecipeHandler(object):
return
# First build up library->package mapping
shlib_providers = oe.package.read_shlib_providers(d)
- libdir = d.getVar('libdir', True)
- base_libdir = d.getVar('base_libdir', True)
+ libdir = d.getVar('libdir')
+ base_libdir = d.getVar('base_libdir')
libpaths = list(set([base_libdir, libdir]))
libname_re = re.compile('^lib(.+)\.so.*$')
pkglibmap = {}
@@ -73,7 +88,7 @@ class RecipeHandler(object):
logger.debug('unable to extract library name from %s' % lib)
# Now turn it into a library->recipe mapping
- pkgdata_dir = d.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
for libname, pkg in pkglibmap.items():
try:
with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
@@ -97,9 +112,9 @@ class RecipeHandler(object):
'''Build up development file->recipe mapping'''
if RecipeHandler.recipeheadermap:
return
- pkgdata_dir = d.getVar('PKGDATA_DIR', True)
- includedir = d.getVar('includedir', True)
- cmakedir = os.path.join(d.getVar('libdir', True), 'cmake')
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
+ includedir = d.getVar('includedir')
+ cmakedir = os.path.join(d.getVar('libdir'), 'cmake')
for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
pn = None
@@ -128,9 +143,9 @@ class RecipeHandler(object):
'''Build up native binary->recipe mapping'''
if RecipeHandler.recipebinmap:
return
- sstate_manifests = d.getVar('SSTATE_MANIFESTS', True)
- staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True)
- build_arch = d.getVar('BUILD_ARCH', True)
+ sstate_manifests = d.getVar('SSTATE_MANIFESTS')
+ staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE')
+ build_arch = d.getVar('BUILD_ARCH')
fileprefix = 'manifest-%s-' % build_arch
for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)):
with open(fn, 'r') as f:
@@ -222,7 +237,8 @@ class RecipeHandler(object):
if deps:
values['DEPENDS'] = ' '.join(deps)
- def genfunction(self, outlines, funcname, content, python=False, forcespace=False):
+ @staticmethod
+ def genfunction(outlines, funcname, content, python=False, forcespace=False):
if python:
prefix = 'python '
else:
@@ -256,27 +272,49 @@ def validate_pv(pv):
def determine_from_filename(srcfile):
"""Determine name and version from a filename"""
- part = ''
- if '.tar.' in srcfile:
- namepart = srcfile.split('.tar.')[0].lower()
- else:
- namepart = os.path.splitext(srcfile)[0].lower()
if is_package(srcfile):
# Force getting the value from the package metadata
return None, None
+
+ if '.tar.' in srcfile:
+ namepart = srcfile.split('.tar.')[0]
else:
- splitval = namepart.rsplit('_', 1)
+ namepart = os.path.splitext(srcfile)[0]
+ namepart = namepart.lower().replace('_', '-')
+ if namepart.endswith('.src'):
+ namepart = namepart[:-4]
+ if namepart.endswith('.orig'):
+ namepart = namepart[:-5]
+ splitval = namepart.split('-')
+ logger.debug('determine_from_filename: split name %s into: %s' % (srcfile, splitval))
+
+ ver_re = re.compile('^v?[0-9]')
+
+ pv = None
+ pn = None
if len(splitval) == 1:
- splitval = namepart.rsplit('-', 1)
- pn = splitval[0].replace('_', '-')
- if len(splitval) > 1:
- if splitval[1][0] in '0123456789':
- pv = splitval[1]
+ # Try to split the version out if there is no separator (or a .)
+ res = re.match('^([^0-9]+)([0-9.]+.*)$', namepart)
+ if res:
+ if len(res.group(1)) > 1 and len(res.group(2)) > 1:
+ pn = res.group(1).rstrip('.')
+ pv = res.group(2)
else:
- pn = '-'.join(splitval).replace('_', '-')
- pv = None
+ pn = namepart
else:
- pv = None
+ if splitval[-1] in ['source', 'src']:
+ splitval.pop()
+ if len(splitval) > 2 and re.match('^(alpha|beta|stable|release|rc[0-9]|pre[0-9]|p[0-9]|[0-9]{8})', splitval[-1]) and ver_re.match(splitval[-2]):
+ pv = '-'.join(splitval[-2:])
+ if pv.endswith('-release'):
+ pv = pv[:-8]
+ splitval = splitval[:-2]
+ elif ver_re.match(splitval[-1]):
+ pv = splitval.pop()
+ pn = '-'.join(splitval)
+ if pv and pv.startswith('v'):
+ pv = pv[1:]
+ logger.debug('determine_from_filename: name = "%s" version = "%s"' % (pn, pv))
return (pn, pv)
def determine_from_url(srcuri):
@@ -301,7 +339,7 @@ def determine_from_url(srcuri):
pn = res.group(1).strip().replace('_', '-')
pv = res.group(2).strip().replace('_', '.')
- if not pn and not pv:
+ if not pn and not pv and parseres.scheme not in ['git', 'gitsm', 'svn', 'hg']:
srcfile = os.path.basename(parseres.path.rstrip('/'))
pn, pv = determine_from_filename(srcfile)
@@ -313,22 +351,30 @@ def supports_srcrev(uri):
# This is a bit sad, but if you don't have this set there can be some
# odd interactions with the urldata cache which lead to errors
localdata.setVar('SRCREV', '${AUTOREV}')
- bb.data.update_data(localdata)
- fetcher = bb.fetch2.Fetch([uri], localdata)
- urldata = fetcher.ud
- for u in urldata:
- if urldata[u].method.supports_srcrev():
+ try:
+ fetcher = bb.fetch2.Fetch([uri], localdata)
+ urldata = fetcher.ud
+ for u in urldata:
+ if urldata[u].method.supports_srcrev():
+ return True
+ except bb.fetch2.FetchError as e:
+ logger.debug('FetchError in supports_srcrev: %s' % str(e))
+ # Fall back to basic check
+ if uri.startswith(('git://', 'gitsm://')):
return True
return False
def reformat_git_uri(uri):
'''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
checkuri = uri.split(';', 1)[0]
- if checkuri.endswith('.git') or '/git/' in checkuri:
- res = re.match('(https?)://([^;]+(\.git)?)(;.*)?$', uri)
+ if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri):
+ res = re.match('(http|https|ssh)://([^;]+(\.git)?)(;.*)?$', uri)
if res:
# Need to switch the URI around so that the git fetcher is used
return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '')
+ elif '@' in checkuri:
+ # Catch e.g. git@git.example.com:repo.git
+ return 'git://%s;protocol=ssh' % checkuri.replace(':', '/', 1)
return uri
def is_package(url):
@@ -351,11 +397,16 @@ def create_recipe(args):
extravalues = {}
checksums = (None, None)
tempsrc = ''
+ source = args.source
srcsubdir = ''
srcrev = '${AUTOREV}'
- if '://' in args.source:
+
+ if os.path.isfile(source):
+ source = 'file://%s' % os.path.abspath(source)
+
+ if scriptutils.is_src_url(source):
# Fetch a URL
- fetchuri = reformat_git_uri(urldefrag(args.source)[0])
+ fetchuri = reformat_git_uri(urldefrag(source)[0])
if args.binary:
# Assume the archive contains the directory structure verbatim
# so we need to extract to a subdirectory
@@ -368,15 +419,14 @@ def create_recipe(args):
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
+ d = bb.data.createCopy(tinfoil.config_data)
if fetchuri.startswith('npm://'):
# Check if npm is available
- npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm')
- if not npm:
- logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager')
- sys.exit(1)
+ npm_bindir = check_npm(tinfoil, args.devtool)
+ d.prependVar('PATH', '%s:' % npm_bindir)
logger.info('Fetching %s...' % srcuri)
try:
- checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
+ checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev)
except bb.fetch2.BBFetchException as e:
logger.error(str(e).rstrip())
sys.exit(1)
@@ -390,10 +440,14 @@ def create_recipe(args):
srcsubdir = dirlist[0]
srctree = os.path.join(srctree, srcsubdir)
else:
- with open(singleitem, 'r') as f:
+ with open(singleitem, 'r', errors='surrogateescape') as f:
if '<html' in f.read(100).lower():
logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
sys.exit(1)
+ if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'):
+ srcuri = 'gitsm://' + srcuri[6:]
+ logger.info('Fetching submodules...')
+ bb.process.run('git submodule update --init --recursive', cwd=srctree)
if is_package(fetchuri):
tmpfdir = tempfile.mkdtemp(prefix='recipetool-')
@@ -411,8 +465,8 @@ def create_recipe(args):
if pkgfile:
if pkgfile.endswith(('.deb', '.ipk')):
- stdout, _ = bb.process.run('ar x %s control.tar.gz' % pkgfile, cwd=tmpfdir)
- stdout, _ = bb.process.run('tar xf control.tar.gz ./control', cwd=tmpfdir)
+ stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir)
+ stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir)
values = convert_debian(tmpfdir)
extravalues.update(values)
elif pkgfile.endswith(('.rpm', '.srpm')):
@@ -426,10 +480,10 @@ def create_recipe(args):
if args.extract_to:
logger.error('--extract-to cannot be specified if source is a directory')
sys.exit(1)
- if not os.path.isdir(args.source):
- logger.error('Invalid source directory %s' % args.source)
+ if not os.path.isdir(source):
+ logger.error('Invalid source directory %s' % source)
sys.exit(1)
- srctree = args.source
+ srctree = source
srcuri = ''
if os.path.exists(os.path.join(srctree, '.git')):
# Try to get upstream repo location from origin remote
@@ -441,7 +495,7 @@ def create_recipe(args):
for line in stdout.splitlines():
splitline = line.split()
if len(splitline) > 1:
- if splitline[0] == 'origin' and '://' in splitline[1]:
+ if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
srcuri = reformat_git_uri(splitline[1])
srcsubdir = 'git'
break
@@ -472,41 +526,8 @@ def create_recipe(args):
# We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
lines_before.append('')
- licvalues = guess_license(srctree_use)
- lic_files_chksum = []
- if licvalues:
- licenses = []
- for licvalue in licvalues:
- if not licvalue[0] in licenses:
- licenses.append(licvalue[0])
- lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
- lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
- lines_before.append('# your responsibility to verify that the values are complete and correct.')
- if len(licvalues) > 1:
- lines_before.append('#')
- lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
- lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
- lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
- lines_before.append('# documentation to determine which situation is applicable.')
- else:
- lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
- lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
- lines_before.append('#')
- lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
- lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
- lines_before.append('# will not be in most cases) you must specify the correct value before using this')
- lines_before.append('# recipe for anything other than initial testing/development!')
- licenses = ['CLOSED']
- pkg_license = extravalues.pop('LICENSE', None)
- if pkg_license:
- if licenses == ['Unknown']:
- lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
- licenses = [pkg_license]
- else:
- lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
- lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
- lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
- lines_before.append('')
+ handled = []
+ licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data)
classes = []
@@ -550,7 +571,6 @@ def create_recipe(args):
if name_pv and not realpv:
realpv = name_pv
-
if not srcuri:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
@@ -584,6 +604,11 @@ def create_recipe(args):
lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
lines_after.append('')
+ if args.fetch_dev:
+ extravalues['fetchdev'] = True
+ else:
+ extravalues['fetchdev'] = None
+
# Find all plugins that want to register handlers
logger.debug('Loading recipe handlers')
raw_handlers = []
@@ -600,12 +625,10 @@ def create_recipe(args):
handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
for handler, priority, _ in handlers:
logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+ setattr(handler, '_devtool', args.devtool)
handlers = [item[0] for item in handlers]
# Apply the handlers
- handled = []
- handled.append(('license', licvalues))
-
if args.binary:
classes.append('bin_package')
handled.append('buildsystem')
@@ -639,12 +662,15 @@ def create_recipe(args):
if not outfile:
if not pn:
- logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile')
+ log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool)
# devtool looks for this specific exit code, so don't change it
sys.exit(15)
else:
- if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')):
- outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0])
+ if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')):
+ suffix = srcuri.split(':', 1)[0]
+ if suffix == 'gitsm':
+ suffix = 'git'
+ outfile = '%s_%s.bb' % (pn, suffix)
elif realpv:
outfile = '%s_%s.bb' % (pn, realpv)
else:
@@ -706,6 +732,15 @@ def create_recipe(args):
if not bbclassextend:
lines_after.append('BBCLASSEXTEND = "native"')
+ postinst = ("postinst", extravalues.pop('postinst', None))
+ postrm = ("postrm", extravalues.pop('postrm', None))
+ preinst = ("preinst", extravalues.pop('preinst', None))
+ prerm = ("prerm", extravalues.pop('prerm', None))
+ funcs = [postinst, postrm, preinst, prerm]
+ for func in funcs:
+ if func[1]:
+ RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1])
+
outlines = []
outlines.extend(lines_before)
if classes:
@@ -732,26 +767,86 @@ def create_recipe(args):
shutil.move(srctree, args.extract_to)
if tempsrc == srctree:
tempsrc = None
- logger.info('Source extracted to %s' % args.extract_to)
+ log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
- f.write('\n'.join(outlines) + '\n')
- logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
+ lastline = None
+ for line in outlines:
+ if not lastline and not line:
+ # Skip extra blank lines
+ continue
+ f.write('%s\n' % line)
+ lastline = line
+ log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
if tempsrc:
- shutil.rmtree(tempsrc)
+ if args.keep_temp:
+ logger.info('Preserving temporary directory %s' % tempsrc)
+ else:
+ shutil.rmtree(tempsrc)
return 0
+def handle_license_vars(srctree, lines_before, handled, extravalues, d):
+ licvalues = guess_license(srctree, d)
+ lic_files_chksum = []
+ lic_unknown = []
+ if licvalues:
+ licenses = []
+ for licvalue in licvalues:
+ if not licvalue[0] in licenses:
+ licenses.append(licvalue[0])
+ lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
+ if licvalue[0] == 'Unknown':
+ lic_unknown.append(licvalue[1])
+ lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
+ lines_before.append('# your responsibility to verify that the values are complete and correct.')
+ if len(licvalues) > 1:
+ lines_before.append('#')
+ lines_before.append('# NOTE: multiple licenses have been detected; they have been separated with &')
+ lines_before.append('# in the LICENSE value for now since it is a reasonable assumption that all')
+ lines_before.append('# of the licenses apply. If instead there is a choice between the multiple')
+ lines_before.append('# licenses then you should change the value to separate the licenses with |')
+ lines_before.append('# instead of &. If there is any doubt, check the accompanying documentation')
+ lines_before.append('# to determine which situation is applicable.')
+ if lic_unknown:
+ lines_before.append('#')
+ lines_before.append('# The following license files were not able to be identified and are')
+ lines_before.append('# represented as "Unknown" below, you will need to check them yourself:')
+ for licfile in lic_unknown:
+ lines_before.append('# %s' % licfile)
+ lines_before.append('#')
+ else:
+ lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
+ lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
+ lines_before.append('#')
+ lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
+ lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
+ lines_before.append('# will not be in most cases) you must specify the correct value before using this')
+ lines_before.append('# recipe for anything other than initial testing/development!')
+ licenses = ['CLOSED']
+ pkg_license = extravalues.pop('LICENSE', None)
+ if pkg_license:
+ if licenses == ['Unknown']:
+ lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
+ licenses = [pkg_license]
+ else:
+ lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
+ lines_before.append('LICENSE = "%s"' % ' & '.join(licenses))
+ lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
+ lines_before.append('')
+ handled.append(('license', licvalues))
+ return licvalues
+
def get_license_md5sums(d, static_only=False):
import bb.utils
md5sums = {}
if not static_only:
# Gather md5sums of license files in common license dir
- commonlicdir = d.getVar('COMMON_LICENSE_DIR', True)
+ commonlicdir = d.getVar('COMMON_LICENSE_DIR')
for fn in os.listdir(commonlicdir):
md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
md5sums[md5value] = fn
@@ -810,7 +905,7 @@ def crunch_license(licfile):
# Note: these are carefully constructed!
license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
- license_statement_re = re.compile('^This (project|software) is( free software)? released under the .{1,10} [Ll]icen[sc]e:?$')
+ license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
copyright_re = re.compile('^(#+)? *Copyright .*$')
crunched_md5sums = {}
@@ -840,7 +935,7 @@ def crunch_license(licfile):
# https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
lictext = []
- with open(licfile, 'r') as f:
+ with open(licfile, 'r', errors='surrogateescape') as f:
for line in f:
# Drop opening statements
if copyright_re.match(line):
@@ -866,9 +961,9 @@ def crunch_license(licfile):
license = crunched_md5sums.get(md5val, None)
return license, md5val, lictext
-def guess_license(srctree):
+def guess_license(srctree, d):
import bb
- md5sums = get_license_md5sums(tinfoil.config_data)
+ md5sums = get_license_md5sums(d)
licenses = []
licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
@@ -921,7 +1016,7 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn
return outlicenses
def read_pkgconfig_provides(d):
- pkgdatadir = d.getVar('PKGDATA_DIR', True)
+ pkgdatadir = d.getVar('PKGDATA_DIR')
pkgmap = {}
for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')):
with open(fn, 'r') as f:
@@ -937,35 +1032,6 @@ def read_pkgconfig_provides(d):
recipemap[pc] = line.split(':', 1)[1].strip()
return recipemap
-def convert_pkginfo(pkginfofile):
- values = {}
- with open(pkginfofile, 'r') as f:
- indesc = False
- for line in f:
- if indesc:
- if line.strip():
- values['DESCRIPTION'] += ' ' + line.strip()
- else:
- indesc = False
- else:
- splitline = line.split(': ', 1)
- key = line[0]
- value = line[1]
- if key == 'LICENSE':
- for dep in value.split(','):
- dep = dep.split()[0]
- mapped = depmap.get(dep, '')
- if mapped:
- depends.append(mapped)
- elif key == 'License':
- values['LICENSE'] = value
- elif key == 'Summary':
- values['SUMMARY'] = value
- elif key == 'Description':
- values['DESCRIPTION'] = value
- indesc = True
- return values
-
def convert_debian(debpath):
value_map = {'Package': 'PN',
'Version': 'PV',
@@ -978,7 +1044,7 @@ def convert_debian(debpath):
values = {}
depends = []
- with open(os.path.join(debpath, 'control')) as f:
+ with open(os.path.join(debpath, 'control'), 'r', errors='surrogateescape') as f:
indesc = False
for line in f:
if indesc:
@@ -1011,6 +1077,25 @@ def convert_debian(debpath):
varname = value_map.get(key, None)
if varname:
values[varname] = value
+ postinst = os.path.join(debpath, 'postinst')
+ postrm = os.path.join(debpath, 'postrm')
+ preinst = os.path.join(debpath, 'preinst')
+ prerm = os.path.join(debpath, 'prerm')
+ sfiles = [postinst, postrm, preinst, prerm]
+ for sfile in sfiles:
+ if os.path.isfile(sfile):
+ logger.info("Converting %s file to recipe function..." %
+ os.path.basename(sfile).upper())
+ content = []
+ with open(sfile) as f:
+ for line in f:
+ if "#!/" in line:
+ continue
+ line = line.rstrip("\n")
+ if line.strip():
+ content.append(line)
+ if content:
+ values[os.path.basename(f.name)] = content
#if depends:
# values['DEPENDS'] = ' '.join(depends)
@@ -1040,6 +1125,22 @@ def convert_rpm_xml(xmlfile):
return values
+def check_npm(tinfoil, debugonly=False):
+ try:
+ rd = tinfoil.parse_recipe('nodejs-native')
+ except bb.providers.NoProvider:
+ # We still conditionally show the message and exit with the special
+ # return code, otherwise we can't show the proper message for eSDK
+ # users
+ log_error_cond('nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs', debugonly)
+ sys.exit(14)
+ bindir = rd.getVar('STAGING_BINDIR_NATIVE')
+ npmpath = os.path.join(bindir, 'npm')
+ if not os.path.exists(npmpath):
+ log_error_cond('npm required to process specified source, but npm is not available - you need to run bitbake -c addto_recipe_sysroot nodejs-native first', debugonly)
+ sys.exit(14)
+ return bindir
+
def register_commands(subparsers):
parser_create = subparsers.add_parser('create',
help='Create a new recipe',
@@ -1054,5 +1155,11 @@ def register_commands(subparsers):
parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
- parser_create.set_defaults(func=create_recipe)
+ parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
+ parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies')
+ parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
+ # FIXME I really hate having to set parserecipes for this, but given we may need
+ # to call into npm (and we don't know in advance if we will or not) and in order
+ # to do so we need to know npm's recipe sysroot path, there's not much alternative
+ parser_create.set_defaults(func=create_recipe, parserecipes=True)
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 78ae4bcdec..e914e53aab 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -173,7 +173,7 @@ class CmakeRecipeHandler(RecipeHandler):
def parse_cmake_file(fn, paths=None):
searchpaths = (paths or []) + [os.path.dirname(fn)]
logger.debug('Parsing file %s' % fn)
- with open(fn, 'r') as f:
+ with open(fn, 'r', errors='surrogateescape') as f:
for line in f:
line = line.strip()
for handler in handlers:
@@ -354,7 +354,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
conffile = RecipeHandler.checkfiles(srctree, ['configure'])
if conffile:
# Check if this is just a pre-generated autoconf configure script
- with open(conffile[0], 'r') as f:
+ with open(conffile[0], 'r', errors='surrogateescape') as f:
for i in range(1, 10):
if 'Generated by GNU Autoconf' in f.readline():
autoconf = True
@@ -364,7 +364,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
# Last resort
conffile = RecipeHandler.checkfiles(srctree, ['configure'])
if conffile:
- with open(conffile[0], 'r') as f:
+ with open(conffile[0], 'r', errors='surrogateescape') as f:
for line in f:
line = line.strip()
if line.startswith('VERSION=') or line.startswith('PACKAGE_VERSION='):
@@ -442,6 +442,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
+ version_re = re.compile('([0-9.]+)')
defines = {}
def subst_defines(value):
@@ -488,6 +489,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
for handler in handlers:
if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
return
+ logger.debug('Found keyword %s with value "%s"' % (keyword, value))
if keyword == 'PKG_CHECK_MODULES':
res = pkg_re.search(value)
if res:
@@ -569,10 +571,21 @@ class AutotoolsRecipeHandler(RecipeHandler):
deps.append('sqlite3')
elif keyword == 'AX_LIB_TAGLIB':
deps.append('taglib')
- elif keyword == 'AX_PKG_SWIG':
- deps.append('swig')
+ elif keyword in ['AX_PKG_SWIG', 'AC_PROG_SWIG']:
+ deps.append('swig-native')
elif keyword == 'AX_PROG_XSLTPROC':
deps.append('libxslt-native')
+ elif keyword in ['AC_PYTHON_DEVEL', 'AX_PYTHON_DEVEL', 'AM_PATH_PYTHON']:
+ pythonclass = 'pythonnative'
+ res = version_re.search(value)
+ if res:
+ if res.group(1).startswith('3'):
+ pythonclass = 'python3native'
+ # Avoid replacing python3native with pythonnative
+ if not pythonclass in inherits and not 'python3native' in inherits:
+ if 'pythonnative' in inherits:
+ inherits.remove('pythonnative')
+ inherits.append(pythonclass)
elif keyword == 'AX_WITH_CURSES':
deps.append('ncurses')
elif keyword == 'AX_PATH_BDB':
@@ -638,7 +651,11 @@ class AutotoolsRecipeHandler(RecipeHandler):
'AX_LIB_SQLITE3',
'AX_LIB_TAGLIB',
'AX_PKG_SWIG',
+ 'AC_PROG_SWIG',
'AX_PROG_XSLTPROC',
+ 'AC_PYTHON_DEVEL',
+ 'AX_PYTHON_DEVEL',
+ 'AM_PATH_PYTHON',
'AX_WITH_CURSES',
'AX_PATH_BDB',
'AX_PATH_LIB_PCRE',
@@ -654,7 +671,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
nesting = 0
in_keyword = ''
partial = ''
- with open(srcfile, 'r') as f:
+ with open(srcfile, 'r', errors='surrogateescape') as f:
for line in f:
if in_keyword:
partial += ' ' + line.strip()
@@ -737,7 +754,7 @@ class MakefileRecipeHandler(RecipeHandler):
if 'buildsystem' in handled:
return False
- makefile = RecipeHandler.checkfiles(srctree, ['Makefile'])
+ makefile = RecipeHandler.checkfiles(srctree, ['Makefile', 'makefile', 'GNUmakefile'])
if makefile:
lines_after.append('# NOTE: this is a Makefile-only piece of software, so we cannot generate much of the')
lines_after.append('# recipe automatically - you will need to examine the Makefile yourself and ensure')
@@ -780,7 +797,7 @@ class MakefileRecipeHandler(RecipeHandler):
if installtarget:
func.append('# This is a guess; additional arguments may be required')
makeargs = ''
- with open(makefile[0], 'r') as f:
+ with open(makefile[0], 'r', errors='surrogateescape') as f:
for i in range(1, 100):
if 'DESTDIR' in f.readline():
makeargs += " 'DESTDIR=${D}'"
@@ -809,7 +826,7 @@ class VersionFileRecipeHandler(RecipeHandler):
version = None
for fileitem in filelist:
linecount = 0
- with open(fileitem, 'r') as f:
+ with open(fileitem, 'r', errors='surrogateescape') as f:
for line in f:
line = line.rstrip().strip('"\'')
linecount += 1
@@ -838,7 +855,7 @@ class SpecFileRecipeHandler(RecipeHandler):
foundvalues = {}
for fileitem in filelist:
linecount = 0
- with open(fileitem, 'r') as f:
+ with open(fileitem, 'r', errors='surrogateescape') as f:
for line in f:
for value, varname in valuemap.items():
if line.startswith(value + ':') and not varname in foundvalues:
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index aff13cf1d2..ec5449bee9 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -61,8 +61,6 @@ class PythonRecipeHandler(RecipeHandler):
}
# PN/PV are already set by recipetool core & desc can be extremely long
excluded_fields = [
- 'Name',
- 'Version',
'Description',
]
setup_parse_map = {
@@ -88,8 +86,11 @@ class PythonRecipeHandler(RecipeHandler):
]
setuparg_multi_line_values = ['Description']
replacements = [
+ ('License', r' +$', ''),
+ ('License', r'^ +', ''),
('License', r' ', '-'),
- ('License', r'-License$', ''),
+ ('License', r'^GNU-', ''),
+ ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
('License', r'^UNKNOWN$', ''),
# Remove currently unhandled version numbers from these variables
@@ -218,6 +219,9 @@ class PythonRecipeHandler(RecipeHandler):
else:
info = self.get_setup_args_info(setupscript)
+ # Grab the license value before applying replacements
+ license_str = info.get('License', '').strip()
+
self.apply_info_replacements(info)
if uses_setuptools:
@@ -225,19 +229,38 @@ class PythonRecipeHandler(RecipeHandler):
else:
classes.append('distutils')
+ if license_str:
+ for i, line in enumerate(lines_before):
+ if line.startswith('LICENSE = '):
+ lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
+ break
+
if 'Classifier' in info:
+ existing_licenses = info.get('License', '')
licenses = []
for classifier in info['Classifier']:
if classifier in self.classifier_license_map:
license = self.classifier_license_map[classifier]
+ if license == 'Apache' and 'Apache-2.0' in existing_licenses:
+ license = 'Apache-2.0'
+ elif license == 'GPL':
+ if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
+ license = 'GPL-2.0'
+ elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
+ license = 'GPL-3.0'
+ elif license == 'LGPL':
+ if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
+ license = 'LGPL-2.1'
+ elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
+ license = 'LGPL-2.0'
+ elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
+ license = 'LGPL-3.0'
licenses.append(license)
if licenses:
info['License'] = ' & '.join(licenses)
-
# Map PKG-INFO & setup.py fields to bitbake variables
- bbinfo = {}
for field, values in info.items():
if field in self.excluded_fields:
continue
@@ -251,37 +274,8 @@ class PythonRecipeHandler(RecipeHandler):
value = ' '.join(str(v) for v in values if v)
bbvar = self.bbvar_map[field]
- if bbvar not in bbinfo and value:
- bbinfo[bbvar] = value
-
- comment_lic_line = None
- for pos, line in enumerate(list(lines_before)):
- if line.startswith('#') and 'LICENSE' in line:
- comment_lic_line = pos
- elif line.startswith('LICENSE =') and 'LICENSE' in bbinfo:
- if line in ('LICENSE = "Unknown"', 'LICENSE = "CLOSED"'):
- lines_before[pos] = 'LICENSE = "{}"'.format(bbinfo['LICENSE'])
- if line == 'LICENSE = "CLOSED"' and comment_lic_line:
- lines_before[comment_lic_line:pos] = [
- '# WARNING: the following LICENSE value is a best guess - it is your',
- '# responsibility to verify that the value is complete and correct.'
- ]
- del bbinfo['LICENSE']
-
- src_uri_line = None
- for pos, line in enumerate(lines_before):
- if line.startswith('SRC_URI ='):
- src_uri_line = pos
-
- if bbinfo:
- mdinfo = ['']
- for k in sorted(bbinfo):
- v = bbinfo[k]
- mdinfo.append('{} = "{}"'.format(k, v))
- if src_uri_line:
- lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
- else:
- lines_before.extend(mdinfo)
+ if bbvar not in extravalues and value:
+ extravalues[bbvar] = value
mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
@@ -504,8 +498,10 @@ class PythonRecipeHandler(RecipeHandler):
for dep in scanned_deps:
mapped = provided_packages.get(dep)
if mapped:
+ logger.debug('Mapped %s to %s' % (dep, mapped))
mapped_deps.add(mapped)
else:
+ logger.debug('Could not map %s' % dep)
unmapped_deps.add(dep)
return mapped_deps, unmapped_deps
@@ -516,7 +512,7 @@ class PythonRecipeHandler(RecipeHandler):
except (OSError, subprocess.CalledProcessError):
pass
else:
- for line in dep_output.decode('utf-8').splitlines():
+ for line in dep_output.splitlines():
line = line.rstrip()
dep, filename = line.split('\t', 1)
if filename.endswith('/setup.py'):
@@ -536,11 +532,11 @@ class PythonRecipeHandler(RecipeHandler):
def parse_pkgdata_for_python_packages(self):
suffixes = [t[0] for t in imp.get_suffixes()]
- pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
ldata = tinfoil.config_data.createCopy()
bb.parse.handle('classes/python-dir.bbclass', ldata, True)
- python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True)
+ python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
python_dirs = [python_sitedir + os.sep,
@@ -566,6 +562,8 @@ class PythonRecipeHandler(RecipeHandler):
continue
if fn.startswith(dynload_dir + os.sep):
+ if '/.debug/' in fn:
+ continue
base = os.path.basename(fn)
provided = base.split('.', 1)[0]
packages[provided] = os.path.basename(pkgdatafile)
@@ -593,7 +591,7 @@ class PythonRecipeHandler(RecipeHandler):
if 'stderr' not in popenargs:
popenargs['stderr'] = subprocess.STDOUT
try:
- return subprocess.check_output(cmd, **popenargs)
+ return subprocess.check_output(cmd, **popenargs).decode('utf-8')
except OSError as exc:
logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
raise
diff --git a/scripts/lib/recipetool/create_kernel.py b/scripts/lib/recipetool/create_kernel.py
index c6e86bd2b9..ca4996c7ac 100644
--- a/scripts/lib/recipetool/create_kernel.py
+++ b/scripts/lib/recipetool/create_kernel.py
@@ -41,7 +41,7 @@ class KernelRecipeHandler(RecipeHandler):
handled.append('buildsystem')
del lines_after[:]
del classes[:]
- template = os.path.join(tinfoil.config_data.getVar('COREBASE', True), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
+ template = os.path.join(tinfoil.config_data.getVar('COREBASE'), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
def handle_var(varname, origvalue, op, newlines):
if varname in ['SRCREV', 'SRCREV_machine']:
while newlines[-1].startswith('#'):
@@ -59,7 +59,7 @@ class KernelRecipeHandler(RecipeHandler):
kpatchlevel = -1
ksublevel = -1
kextraversion = ''
- with open(makefile, 'r') as f:
+ with open(makefile, 'r', errors='surrogateescape') as f:
for i, line in enumerate(f):
if i > 10:
break
@@ -85,7 +85,7 @@ class KernelRecipeHandler(RecipeHandler):
elif varname == 'COMPATIBLE_MACHINE':
while newlines[-1].startswith('#'):
del newlines[-1]
- machine = tinfoil.config_data.getVar('MACHINE', True)
+ machine = tinfoil.config_data.getVar('MACHINE')
return machine, op, 0, True
return origvalue, op, 0, True
with open(template, 'r') as f:
diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py
index fe39edb288..7cf188db21 100644
--- a/scripts/lib/recipetool/create_kmod.py
+++ b/scripts/lib/recipetool/create_kmod.py
@@ -53,7 +53,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
break
else:
continue
- with open(cfile, 'r') as f:
+ with open(cfile, 'r', errors='surrogateescape') as f:
for line in f:
if module_inc_re.match(line.strip()):
is_module = True
@@ -73,7 +73,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
in_install = False
in_compile = False
install_target = None
- with open(makefile, 'r') as f:
+ with open(makefile, 'r', errors='surrogateescape') as f:
for line in f:
if line.startswith('install:'):
if not install_lines:
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index fcc0172af8..cb8f338b8b 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -21,7 +21,7 @@ import subprocess
import tempfile
import shutil
import json
-from recipetool.create import RecipeHandler, split_pkg_licenses
+from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm
logger = logging.getLogger('recipetool')
@@ -45,11 +45,29 @@ class NpmRecipeHandler(RecipeHandler):
license = data['license']
if isinstance(license, dict):
license = license.get('type', None)
+ if license:
+ if 'OR' in license:
+ license = license.replace('OR', '|')
+ license = license.replace('AND', '&')
+ license = license.replace(' ', '_')
+ if not license[0] == '(':
+ license = '(' + license + ')'
+ print('LICENSE: {}'.format(license))
+ else:
+ license = license.replace('AND', '&')
+ if license[0] == '(':
+ license = license[1:]
+ if license[-1] == ')':
+ license = license[:-1]
+ license = license.replace('MIT/X11', 'MIT')
+ license = license.replace('Public Domain', 'PD')
+ license = license.replace('SEE LICENSE IN EULA',
+ 'SEE-LICENSE-IN-EULA')
return license
- def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before):
+ def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
try:
- runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
except bb.process.ExecutionError as e:
logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
@@ -61,8 +79,8 @@ class NpmRecipeHandler(RecipeHandler):
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
- def _lockdown(self, srctree, localfilesdir, extravalues, lines_before):
- runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+ def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
if not NpmRecipeHandler.lockdownpath:
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
@@ -83,6 +101,79 @@ class NpmRecipeHandler(RecipeHandler):
extravalues['extrafiles']['lockdown.json'] = tmpfile
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
+ def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
+ import scriptutils
+ # If this isn't a single module we need to get the dependencies
+ # and add them to SRC_URI
+ def varfunc(varname, origvalue, op, newlines):
+ if varname == 'SRC_URI':
+ if not origvalue.startswith('npm://'):
+ src_uri = origvalue.split()
+ changed = False
+ deplist = {}
+ for dep, depver in optdeps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ if self.check_npm_optional_dependency(depdata):
+ deplist[dep] = depdata
+ for dep, depver in devdeps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ if self.check_npm_optional_dependency(depdata):
+ deplist[dep] = depdata
+ for dep, depver in deps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ deplist[dep] = depdata
+
+ for dep, depdata in deplist.items():
+ version = depdata.get('version', None)
+ if version:
+ url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
+ scriptutils.fetch_uri(d, url, srctree)
+ src_uri.append(url)
+ changed = True
+ if changed:
+ return src_uri, None, -1, True
+ return origvalue, None, 0, True
+ updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
+ if updated:
+ del lines_before[:]
+ for line in newlines:
+ # Hack to avoid newlines that edit_metadata inserts
+ if line.endswith('\n'):
+ line = line[:-1]
+ lines_before.append(line)
+ return updated
+
+ def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d):
+ for item in handled:
+ if isinstance(item, tuple):
+ if item[0] == 'license':
+ del item
+ break
+
+ calledvars = []
+ def varfunc(varname, origvalue, op, newlines):
+ if varname in ['LICENSE', 'LIC_FILES_CHKSUM']:
+ for i, e in enumerate(reversed(newlines)):
+ if not e.startswith('#'):
+ stop = i
+ while stop > 0:
+ newlines.pop()
+ stop -= 1
+ break
+ calledvars.append(varname)
+ if len(calledvars) > 1:
+ # The second time around, put the new license text in
+ insertpos = len(newlines)
+ handle_license_vars(srctree, newlines, handled, extravalues, d)
+ return None, None, 0, True
+ return origvalue, None, 0, True
+ updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
+ if updated:
+ del lines_before[:]
+ lines_before.extend(newlines)
+ else:
+ raise Exception('Did not find license variables')
+
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
import bb.utils
import oe
@@ -92,11 +183,15 @@ class NpmRecipeHandler(RecipeHandler):
return False
def read_package_json(fn):
- with open(fn, 'r') as f:
+ with open(fn, 'r', errors='surrogateescape') as f:
return json.loads(f.read())
files = RecipeHandler.checkfiles(srctree, ['package.json'])
if files:
+ d = bb.data.createCopy(tinfoil.config_data)
+ npm_bindir = check_npm(tinfoil, self._devtool)
+ d.prependVar('PATH', '%s:' % npm_bindir)
+
data = read_package_json(files[0])
if 'name' in data and 'version' in data:
extravalues['PN'] = data['name']
@@ -108,12 +203,19 @@ class NpmRecipeHandler(RecipeHandler):
if 'homepage' in data:
extravalues['HOMEPAGE'] = data['homepage']
+ fetchdev = extravalues['fetchdev'] or None
+ deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
+ updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
+ if updated:
+ # We need to redo the license stuff
+ self._replace_license_vars(srctree, lines_before, handled, extravalues, d)
+
# Shrinkwrap
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
- self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before)
+ self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
# Lockdown
- self._lockdown(srctree, localfilesdir, extravalues, lines_before)
+ self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
# Split each npm module out to is own package
npmpackages = oe.package.npm_split_package_dirs(srctree)
@@ -139,7 +241,9 @@ class NpmRecipeHandler(RecipeHandler):
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
packages['${PN}'] = ''
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
- all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense]))
+ all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
+ if '&' in all_licenses:
+ all_licenses.remove('&')
# Go back and update the LICENSE value since we have a bit more
# information than when that was written out (and we know all apply
# vs. there being a choice, so we can join them with &)
@@ -148,9 +252,93 @@ class NpmRecipeHandler(RecipeHandler):
lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses)
break
+ # Need to move S setting after inherit npm
+ for i, line in enumerate(lines_before):
+ if line.startswith('S ='):
+ lines_before.pop(i)
+ lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
+ lines_after.insert(1, line)
+ break
+
return True
return False
+ # FIXME this is duplicated from lib/bb/fetch2/npm.py
+ def _parse_view(self, output):
+ '''
+ Parse the output of npm view --json; the last JSON result
+ is assumed to be the one that we're interested in.
+ '''
+ pdata = None
+ outdeps = {}
+ datalines = []
+ bracelevel = 0
+ for line in output.splitlines():
+ if bracelevel:
+ datalines.append(line)
+ elif '{' in line:
+ datalines = []
+ datalines.append(line)
+ bracelevel = bracelevel + line.count('{') - line.count('}')
+ if datalines:
+ pdata = json.loads('\n'.join(datalines))
+ return pdata
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def get_npm_data(self, pkg, version, d):
+ import bb.fetch2
+ pkgfullname = pkg
+ if version != '*' and not '/' in version:
+ pkgfullname += "@'%s'" % version
+ logger.debug(2, "Calling getdeps on %s" % pkg)
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
+ fetchcmd = "npm view %s --json" % pkgfullname
+ output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
+ data = self._parse_view(output)
+ return data
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def get_npm_package_dependencies(self, pdata, fetchdev):
+ dependencies = pdata.get('dependencies', {})
+ optionalDependencies = pdata.get('optionalDependencies', {})
+ dependencies.update(optionalDependencies)
+ if fetchdev:
+ devDependencies = pdata.get('devDependencies', {})
+ dependencies.update(devDependencies)
+ else:
+ devDependencies = {}
+ depsfound = {}
+ optdepsfound = {}
+ devdepsfound = {}
+ for dep in dependencies:
+ if dep in optionalDependencies:
+ optdepsfound[dep] = dependencies[dep]
+ elif dep in devDependencies:
+ devdepsfound[dep] = dependencies[dep]
+ else:
+ depsfound[dep] = dependencies[dep]
+ return depsfound, optdepsfound, devdepsfound
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def check_npm_optional_dependency(self, pdata):
+ pkg_os = pdata.get('os', None)
+ if pkg_os:
+ if not isinstance(pkg_os, list):
+ pkg_os = [pkg_os]
+ blacklist = False
+ for item in pkg_os:
+ if item.startswith('!'):
+ blacklist = True
+ break
+ if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
+ logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
+ return False
+ return True
+
+
def register_recipe_handlers(handlers):
handlers.append((NpmRecipeHandler(), 60))
diff --git a/scripts/lib/recipetool/newappend.py b/scripts/lib/recipetool/newappend.py
index 4fbb40a358..0b63759d8c 100644
--- a/scripts/lib/recipetool/newappend.py
+++ b/scripts/lib/recipetool/newappend.py
@@ -39,28 +39,6 @@ def tinfoil_init(instance):
tinfoil = instance
-def _provide_to_pn(cooker, provide):
- """Get the name of the preferred recipe for the specified provide."""
- import bb.providers
- filenames = cooker.recipecache.providers[provide]
- eligible, foundUnique = bb.providers.filterProviders(filenames, provide, cooker.expanded_data, cooker.recipecache)
- filename = eligible[0]
- pn = cooker.recipecache.pkg_fn[filename]
- return pn
-
-
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
-
def layer(layerpath):
if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
@@ -70,8 +48,7 @@ def layer(layerpath):
def newappend(args):
import oe.recipeutils
- pn = _provide_to_pn(tinfoil.cooker, args.target)
- recipe_path = _get_recipe_file(tinfoil.cooker, pn)
+ recipe_path = tinfoil.get_recipe_file(args.target)
rd = tinfoil.config_data.createCopy()
rd.setVar('FILE', recipe_path)
@@ -83,7 +60,7 @@ def newappend(args):
if not path_ok:
logger.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.', os.path.join(args.destlayer, 'conf', 'layer.conf'), os.path.dirname(append_path))
- layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()]
+ layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
if not os.path.abspath(args.destlayer) in layerdirs:
logger.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
index 657d2b6a7b..9de315a0ef 100644
--- a/scripts/lib/recipetool/setvar.py
+++ b/scripts/lib/recipetool/setvar.py
@@ -51,7 +51,7 @@ def setvar(args):
if args.recipe_only:
patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)]
else:
- rd = oe.recipeutils.parse_recipe(args.recipefile, None, tinfoil.config_data)
+ rd = tinfoil.parse_recipe_file(args.recipefile, False)
if not rd:
return 1
patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch)
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-02 22:43:37 +0000