path: root/meta/lib/oe/recipeutils.py

# Utility functions for reading and modifying recipes
#
# Some code borrowed from the OE layer index
#
# Copyright (C) 2013-2015 Intel Corporation
#

import sys
import os
import os.path
import tempfile
import textwrap
import difflib
from . import utils
import shutil
import re
import fnmatch
from collections import OrderedDict, defaultdict


# Help us to find places to insert values
recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()']
# Variables that sometimes are a bit long but shouldn't be wrapped
nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]']
list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']


def pn_to_recipe(cooker, pn):
    """Convert a recipe name (PN) to the path to the recipe file"""
    import bb.providers

    if pn in cooker.recipecache.pkg_pn:
        best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn)
        return best[3]
    elif pn in cooker.recipecache.providers:
        filenames = cooker.recipecache.providers[pn]
        eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecache)
        filename = eligible[0]
        return filename
    else:
        return None


def get_unavailable_reasons(cooker, pn):
    """If a recipe could not be found, find out why if possible"""
    import bb.taskdata
    taskdata = bb.taskdata.TaskData(None, skiplist=cooker.skiplist)
    return taskdata.get_reasons(pn)


def parse_recipe(fn, appendfiles, d):
    """
    Parse an individual recipe file, optionally with a list of
    bbappend files.
    """
    import bb.cache
    envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d)
    return envdata


def parse_recipe_simple(cooker, pn, d, appends=True):
    """
    Parse a recipe and optionally all bbappends that apply to it
    in the current configuration.
    """
    import bb.providers

    recipefile = pn_to_recipe(cooker, pn)
    if not recipefile:
        skipreasons = get_unavailable_reasons(cooker, pn)
        # We may as well re-use bb.providers.NoProvider here
        if skipreasons:
            raise bb.providers.NoProvider(skipreasons)
        else:
            raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
    if appends:
        appendfiles = cooker.collection.get_file_appends(recipefile)
    else:
        appendfiles = None
    return parse_recipe(recipefile, appendfiles, d)


def get_var_files(fn, varlist, d):
    """Find the file in which each of a list of variables is set.
    Note: requires variable history to be enabled when parsing.
    """
    varfiles = {}
    for v in varlist:
        history = d.varhistory.variable(v)
        files = []
        for event in history:
            if 'file' in event and not 'flag' in event:
                files.append(event['file'])
        if files:
            actualfile = files[-1]
        else:
            actualfile = None
        varfiles[v] = actualfile

    return varfiles


def split_var_value(value, assignment=True):
    """
    Split a space-separated variable's value into a list of items,
    taking into account that some of the items might be made up of
    expressions containing spaces that should not be split.
    Parameters:
        value:
            The string value to split
        assignment:
            True to assume that the value represents an assignment
            statement, False otherwise. If True, and an assignment
            statement is passed in the first item in
            the returned list will be the part of the assignment
            statement up to and including the opening quote character,
            and the last item will be the closing quote.
    """
    inexpr = 0
    lastchar = None
    out = []
    buf = ''
    for char in value:
        if char == '{':
            if lastchar == '$':
                inexpr += 1
        elif char == '}':
            inexpr -= 1
        elif assignment and char in '"\'' and inexpr == 0:
            if buf:
                out.append(buf)
            out.append(char)
            char = ''
            buf = ''
        elif char.isspace() and inexpr == 0:
            char = ''
            if buf:
                out.append(buf)
            buf = ''
        buf += char
        lastchar = char
    if buf:
        out.append(buf)

    # Join together assignment statement and opening quote
    outlist = out
    if assignment:
        assigfound = False
        for idx, item in enumerate(out):
            if '=' in item:
                assigfound = True
            if assigfound:
                if '"' in item or "'" in item:
                    outlist = [' '.join(out[:idx+1])]
                    outlist.extend(out[idx+1:])
                    break
    return outlist


def patch_recipe_lines(fromlines, values, trailing_newline=True):
    """Update or insert variable values into lines from a recipe.
       Note that some manual inspection/intervention may be required
       since this cannot handle all situations.
    """

    import bb.utils

    if trailing_newline:
        newline = '\n'
    else:
        newline = ''

    recipe_progression_res = []
    recipe_progression_restrs = []
    for item in recipe_progression:
        if item.endswith('()'):
            key = item[:-2]
        else:
            key = item
        restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
        if item.endswith('()'):
            recipe_progression_restrs.append(restr + '()')
        else:
            recipe_progression_restrs.append(restr)
        recipe_progression_res.append(re.compile('^%s$' % restr))

    def get_recipe_pos(variable):
        for i, p in enumerate(recipe_progression_res):
            if p.match(variable):
                return i
        return -1

    remainingnames = {}
    for k in values.keys():
        remainingnames[k] = get_recipe_pos(k)
    remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1]))

    modifying = False

    def outputvalue(name, lines, rewindcomments=False):
        if values[name] is None:
            return
        rawtext = '%s = "%s"%s' % (name, values[name], newline)
        addlines = []
        if name in nowrap_vars:
            addlines.append(rawtext)
        elif name in list_vars:
            splitvalue = split_var_value(values[name], assignment=False)
            if len(splitvalue) > 1:
                linesplit = ' \\\n' + (' ' * (len(name) + 4))
                addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline))
            else:
                addlines.append(rawtext)
        else:
            wrapped = textwrap.wrap(rawtext)
            for wrapline in wrapped[:-1]:
                addlines.append('%s \\%s' % (wrapline, newline))
            addlines.append('%s%s' % (wrapped[-1], newline))
        if rewindcomments:
            # Ensure we insert the lines before any leading comments
            # (that we'd want to ensure remain leading the next value)
            for i, ln in reversed(list(enumerate(lines))):
                if not ln.startswith('#'):
                    lines[i+1:i+1] = addlines
                    break
            else:
                lines.extend(addlines)
        else:
            lines.extend(addlines)

    existingnames = []
    def patch_recipe_varfunc(varname, origvalue, op, newlines):
        if modifying:
            # Insert anything that should come before this variable
            pos = get_recipe_pos(varname)
            for k in list(remainingnames):
                if remainingnames[k] > -1 and pos >= remainingnames[k]