diff options
| -rw-r--r-- | scripts/lib/devtool/upgrade.py | 10 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create.py | 40 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_npm.py | 10 | ||||
| -rw-r--r-- | scripts/lib/scriptutils.py | 158 |
4 files changed, 153 insertions, 65 deletions
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 05fb9e5ed0..f077f37726 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py @@ -207,10 +207,16 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin tmpdir = tempfile.mkdtemp(prefix='devtool') try: - md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) - except bb.fetch2.FetchError as e: + checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp) + except scriptutils.FetchUrlFailure as e: raise DevtoolError(e) + if ftmpdir and keep_temp: + logger.info('Fetch temp directory is %s' % ftmpdir) + + md5 = checksums['md5sum'] + sha256 = checksums['sha256sum'] + tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 2b7cc76ccb..2a6a28ba91 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -417,7 +417,7 @@ def create_recipe(args): pkgarch = "${MACHINE_ARCH}" extravalues = {} - checksums = (None, None) + checksums = {} tempsrc = '' source = args.source srcsubdir = '' @@ -439,22 +439,25 @@ def create_recipe(args): if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) - tempsrc = tempfile.mkdtemp(prefix='recipetool-') - srctree = tempsrc - d = bb.data.createCopy(tinfoil.config_data) - if fetchuri.startswith('npm://'): - # Check if npm is available - npm_bindir = check_npm(tinfoil, args.devtool) - d.prependVar('PATH', '%s:' % npm_bindir) - logger.info('Fetching %s...' % srcuri) + + tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') + bb.utils.mkdirhier(tmpparent) + tempsrc = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent) + srctree = os.path.join(tempsrc, 'source') + try: - checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev) - except bb.fetch2.BBFetchException as e: - logger.error(str(e).rstrip()) + checksums, ftmpdir = scriptutils.fetch_url(tinfoil, srcuri, srcrev, srctree, logger, preserve_tmp=args.keep_temp) + except scriptutils.FetchUrlFailure as e: + logger.error(str(e)) sys.exit(1) + + if ftmpdir and args.keep_temp: + logger.info('Fetch temp directory is %s' % ftmpdir) + dirlist = os.listdir(srctree) - if 'git.indirectionsymlink' in dirlist: - dirlist.remove('git.indirectionsymlink') + filterout = ['git.indirectionsymlink'] + dirlist = [x for x in dirlist if x not in filterout] + logger.debug('Directory listing (excluding filtered out):\n %s' % '\n '.join(dirlist)) if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): @@ -465,7 +468,7 @@ def create_recipe(args): check_single_file(dirlist[0], fetchuri) elif len(dirlist) == 0: if '/' in fetchuri: - fn = os.path.join(d.getVar('DL_DIR'), fetchuri.split('/')[-1]) + fn = os.path.join(tinfoil.config_data.getVar('DL_DIR'), fetchuri.split('/')[-1]) if os.path.isfile(fn): check_single_file(fn, fetchuri) # If we've got to here then there's no source so we might as well give up @@ -593,11 +596,8 @@ def create_recipe(args): if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) - (md5value, sha256value) = checksums - if md5value: - lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) - if sha256value: - lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) + for key, value in sorted(checksums.items()): + lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index cb8f338b8b..ba7e39a406 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -109,7 +109,6 @@ class NpmRecipeHandler(RecipeHandler): if varname == 'SRC_URI': if not origvalue.startswith('npm://'): src_uri = origvalue.split() - changed = False deplist = {} for dep, depver in optdeps.items(): depdata = self.get_npm_data(dep, depver, d) @@ -123,14 +122,15 @@ class NpmRecipeHandler(RecipeHandler): depdata = self.get_npm_data(dep, depver, d) deplist[dep] = depdata + extra_urls = [] for dep, depdata in deplist.items(): version = depdata.get('version', None) if version: url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) - scriptutils.fetch_uri(d, url, srctree) - src_uri.append(url) - changed = True - if changed: + extra_urls.append(url) + if extra_urls: + scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger) + src_uri.extend(extra_urls) return src_uri, None, -1, True return origvalue, None, 0, True updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py index 92b601c7e8..1005dd495a 100644 --- a/scripts/lib/scriptutils.py +++ b/scripts/lib/scriptutils.py @@ -23,6 +23,8 @@ import argparse import subprocess import tempfile import shutil +import random +import string def logger_create(name, stream=None): logger = logging.getLogger(name) @@ -78,50 +80,130 @@ def git_convert_standalone_clone(repodir): bb.process.run('git repack -a', cwd=repodir) os.remove(alternatesfile) -def fetch_uri(d, uri, destdir, srcrev=None): - """Fetch a URI to a local directory""" +def _get_temp_recipe_dir(d): + # This is a little bit hacky but we need to find a place where we can put + # the recipe so that bitbake can find it. We're going to delete it at the + # end so it doesn't really matter where we put it. + bbfiles = d.getVar('BBFILES').split() + fetchrecipedir = None + for pth in bbfiles: + if pth.endswith('.bb'): + pthdir = os.path.dirname(pth) + if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK): + fetchrecipedir = pthdir.replace('*', 'recipetool') + if pthdir.endswith('workspace/recipes/*'): + # Prefer the workspace + break + return fetchrecipedir + +class FetchUrlFailure(Exception): + def __init__(self, url): + self.url = url + def __str__(self): + return "Failed to fetch URL %s" % self.url + +def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False): + """ + Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e. + any dependencies that need to be satisfied in order to support the fetch + operation will be taken care of + """ + import bb - tmpparent = d.getVar('BASE_WORKDIR') + + checksums = {} + fetchrecipepn = None + + # We need to put our temp directory under ${BASE_WORKDIR} otherwise + # we may have problems with the recipe-specific sysroot population + tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') bb.utils.mkdirhier(tmpparent) - tmpworkdir = tempfile.mkdtemp(dir=tmpparent) + tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent) try: - bb.utils.mkdirhier(destdir) - localdata = bb.data.createCopy(d) - - # Set some values to allow extend_recipe_sysroot to work here we're we are not running from a task - localdata.setVar('WORKDIR', tmpworkdir) - localdata.setVar('BB_RUNTASK', 'do_fetch') - localdata.setVar('PN', 'dummy') - localdata.setVar('BB_LIMITEDDEPS', '1') - bb.build.exec_func("extend_recipe_sysroot", localdata) - - # Set some values for the benefit of the fetcher code - localdata.setVar('BB_STRICT_CHECKSUM', '') - localdata.setVar('SRCREV', srcrev) - ret = (None, None) - olddir = os.getcwd() + tmpworkdir = os.path.join(tmpdir, 'work') + logger.debug('fetch_url: temp dir is %s' % tmpdir) + + fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data) + if not fetchrecipedir: + logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe') + sys.exit(1) + fetchrecipe = None + bb.utils.mkdirhier(fetchrecipedir) try: - fetcher = bb.fetch2.Fetch([uri], localdata) - for u in fetcher.ud: - ud = fetcher.ud[u] - ud.ignore_checksums = True - fetcher.download() - for u in fetcher.ud: - ud = fetcher.ud[u] - if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR').rstrip(os.sep): - raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri) - fetcher.unpack(destdir) - for u in fetcher.ud: - ud = fetcher.ud[u] - if ud.method.recommends_checksum(ud): - md5value = bb.utils.md5_file(ud.localpath) - sha256value = bb.utils.sha256_file(ud.localpath) - ret = (md5value, sha256value) + # Generate a dummy recipe so we can follow more or less normal paths + # for do_fetch and do_unpack + # I'd use tempfile functions here but underscores can be produced by that and those + # aren't allowed in recipe file names except to separate the version + rndstring = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) + fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring) + fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0] + logger.debug('Generating initial recipe %s for fetching' % fetchrecipe) + with open(fetchrecipe, 'w') as f: + # We don't want to have to specify LIC_FILES_CHKSUM + f.write('LICENSE = "CLOSED"\n') + # We don't need the cross-compiler + f.write('INHIBIT_DEFAULT_DEPS = "1"\n') + # We don't have the checksums yet so we can't require them + f.write('BB_STRICT_CHECKSUM = "ignore"\n') + f.write('SRC_URI = "%s"\n' % srcuri) + f.write('SRCREV = "%s"\n' % srcrev) + f.write('WORKDIR = "%s"\n' % tmpworkdir) + # Set S out of the way so it doesn't get created under the workdir + f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc')) + + logger.info('Fetching %s...' % srcuri) + + # FIXME this is too noisy at the moment + + # Parse recipes so our new recipe gets picked up + tinfoil.parse_recipes() + + def eventhandler(event): + if isinstance(event, bb.fetch2.MissingChecksumEvent): + checksums.update(event.checksums) + return True + return False + + # Run the fetch + unpack tasks + res = tinfoil.build_targets(fetchrecipepn, + 'do_unpack', + handle_events=True, + extra_events=['bb.fetch2.MissingChecksumEvent'], + event_callback=eventhandler) + if not res: + raise FetchUrlFailure(srcuri) + + # Remove unneeded directories + rd = tinfoil.parse_recipe(fetchrecipepn) + if rd: + pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE'] + for pathvar in pathvars: + path = rd.getVar(pathvar) + shutil.rmtree(path) finally: - os.chdir(olddir) + if fetchrecipe: + try: + os.remove(fetchrecipe) + except FileNotFoundError: + pass + try: + os.rmdir(fetchrecipedir) + except OSError as e: + import errno + if e.errno != errno.ENOTEMPTY: + raise + + bb.utils.mkdirhier(destdir) + for fn in os.listdir(tmpworkdir): + shutil.move(os.path.join(tmpworkdir, fn), destdir) + finally: - shutil.rmtree(tmpworkdir) - return ret + if not preserve_tmp: + shutil.rmtree(tmpdir) + tmpdir = None + + return checksums, tmpdir + def run_editor(fn): if isinstance(fn, str): |