path: root/meta/classes/populate_sdk_ext.bbclass

# Extensible SDK

inherit populate_sdk_base

# NOTE: normally you cannot use task overrides for this kind of thing - this
# only works because of get_sdk_ext_rdepends()

TOOLCHAIN_HOST_TASK_task-populate-sdk-ext = " \
    meta-environment-extsdk-${MACHINE} \
    "

TOOLCHAIN_TARGET_TASK_task-populate-sdk-ext = ""

SDK_RDEPENDS_append_task-populate-sdk-ext = " ${SDK_TARGETS}"

SDK_RELOCATE_AFTER_INSTALL_task-populate-sdk-ext = "0"

SDK_EXT = ""
SDK_EXT_task-populate-sdk-ext = "-ext"

# Options are full or minimal
SDK_EXT_TYPE ?= "full"

SDK_RECRDEP_TASKS ?= ""

SDK_LOCAL_CONF_WHITELIST ?= ""
SDK_LOCAL_CONF_BLACKLIST ?= "CONF_VERSION \
                             BB_NUMBER_THREADS \
                             BB_NUMBER_PARSE_THREADS \
                             PARALLEL_MAKE \
                             PRSERV_HOST \
                             SSTATE_MIRRORS \
                             DL_DIR \
                             SSTATE_DIR \
                             TMPDIR \
                            "
SDK_INHERIT_BLACKLIST ?= "buildhistory icecc"
SDK_UPDATE_URL ?= ""

SDK_TARGETS ?= "${PN}"

def get_sdk_install_targets(d):
    sdk_install_targets = ''
    if d.getVar('SDK_EXT_TYPE', True) != 'minimal':
        sdk_install_targets = d.getVar('SDK_TARGETS', True)

        depd = d.getVar('BB_TASKDEPDATA', False)
        for v in depd.itervalues():
            if v[1] == 'do_image_complete':
                if v[0] not in sdk_install_targets:
                    sdk_install_targets += ' {}'.format(v[0])

    if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
        sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'

    return sdk_install_targets

get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA"

OE_INIT_ENV_SCRIPT ?= "oe-init-build-env"

# The files from COREBASE that you want preserved in the COREBASE copied
# into the sdk. This allows someone to have their own setup scripts in
# COREBASE be preserved as well as untracked files.
COREBASE_FILES ?= " \
    oe-init-build-env \
    oe-init-build-env-memres \
    scripts \
    LICENSE \
    .templateconf \
"

SDK_DIR_task-populate-sdk-ext = "${WORKDIR}/sdk-ext"
B_task-populate-sdk-ext = "${SDK_DIR}"
TOOLCHAINEXT_OUTPUTNAME = "${SDK_NAME}-toolchain-ext-${SDK_VERSION}"
TOOLCHAIN_OUTPUTNAME_task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"

SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"

SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} Extensible SDK"

python copy_buildsystem () {
    import re
    import shutil
    import glob
    import oe.copy_buildsystem

    oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT', True)

    conf_bbpath = ''
    conf_initpath = ''
    core_meta_subdir = ''

    # Copy in all metadata layers + bitbake (as repositories)
    buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
    baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True)

    # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
    derivative = (d.getVar('SDK_DERIVATIVE', True) or '') == '1'
    if derivative:
        workspace_name = 'orig-workspace'
    else:
        workspace_name = None
    layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)

    sdkbblayers = []
    corebase = os.path.basename(d.getVar('COREBASE', True))
    for layer in layers_copied:
        if corebase == os.path.basename(layer):
            conf_bbpath = os.path.join('layers', layer, 'bitbake')
        else:
            sdkbblayers.append(layer)

    for path in os.listdir(baseoutpath + '/layers'):
        relpath = os.path.join('layers', path, oe_init_env_script)
        if os.path.exists(os.path.join(baseoutpath, relpath)):
            conf_initpath = relpath

        relpath = os.path.join('layers', path, 'scripts', 'devtool')
        if os.path.exists(os.path.join(baseoutpath, relpath)):
            scriptrelpath = os.path.dirname(relpath)

        relpath = os.path.join('layers', path, 'meta')
        if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')):
            core_meta_subdir = relpath

    d.setVar('oe_init_build_env_path', conf_initpath)
    d.setVar('scriptrelpath', scriptrelpath)

    # Write out config file for devtool
    import ConfigParser
    config = ConfigParser.SafeConfigParser()
    config.add_section('General')
    config.set('General', 'bitbake_subdir', conf_bbpath)
    config.set('General', 'init_path', conf_initpath)
    config.set('General', 'core_meta_subdir', core_meta_subdir)
    config.add_section('SDK')
    config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS', True))
    updateurl = d.getVar('SDK_UPDATE_URL', True)
    if updateurl:
        config.set('SDK', 'updateserver', updateurl)
    bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf'))
    with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
        config.write(f)

    unlockedsigs =  os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc')
    with open(unlockedsigs, 'w') as f:
        pass

    # Create a layer for new recipes / appends
    bbpath = d.getVar('BBPATH', True)
    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])

    # Create bblayers.conf
    bb.utils.mkdirhier(baseoutpath + '/conf')
    with open(baseoutpath + '/conf/bblayers.conf', 'w') as f:
        f.write('# WARNING: this configuration has been automatically generated and in\n')
        f.write('# most cases should not be edited. If you need more flexibility than\n')
        f.write('# this configuration provides, it is strongly suggested that you set\n')
        f.write('# up a proper instance of the full build system and use that instead.\n\n')

        # LCONF_VERSION may not be set, for example when using meta-poky
        # so don't error if it isn't found
        lconf_version = d.getVar('LCONF_VERSION', False)
        if lconf_version is not None:
            f.write('LCONF_VERSION = "%s"\n\n' % lconf_version)

        f.write('BBPATH = "$' + '{TOPDIR}"\n')
        f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n')
        f.write('BBLAYERS := " \\\n')
        for layerrelpath in sdkbblayers:
            f.write('    $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath)
        f.write('    $' + '{SDKBASEMETAPATH}/workspace \\\n')
        f.write('    "\n')

    env_whitelist = (d.getVar('BB_ENV_EXTRAWHITE', True) or '').split()
    env_whitelist_values = {}

    # Create local.conf
    builddir = d.getVar('TOPDIR', True)
    if derivative:
        shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
    else:
        local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split()
        local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split()
        def handle_var(varname, origvalue, op, newlines):
            if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist):
                newlines.append('# Removed original setting of %s\n' % varname)
                return None, op, 0, True
            else:
                if varname in env_whitelist:
                    env_whitelist_values[varname] = origvalue
                return origvalue, op, 0, True
        varlist = ['[^#=+ ]*']
        with open(builddir + '/conf/local.conf', 'r') as f:
            oldlines = f.readlines()
        (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)

        with open(baseoutpath + '/conf/local.conf', 'w') as f:
            f.write('# WARNING: this configuration has been automatically generated and in\n')
            f.write('# most cases should not be edited. If you need more flexibility than\n')
            f.write('# this configuration provides, it is strongly suggested that you set\n')
            f.write('# up a proper instance of the full build system and use that instead.\n\n')
            for line in newlines:
                if line.strip() and not line.startswith('#'):
                    f.write(line)
            # Write a newline just in case there's none at the end of the original
            f.write('\n')

            f.write('INHERIT += "%s"\n\n' % 'uninative')
            f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))

            # Some classes are not suitable for SDK, remove them from INHERIT
            f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False))

            # Bypass the default connectivity check if any
            f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')

            # This warning will come out if reverse dependencies for a task
            # don't have sstate as well as the task itself. We already know
            # this will be the case for the extensible sdk, so turn off the
            # warning.
            f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')

            # Error if the sigs in the locked-signature file don't match
            # the sig computed from the metadata.
            f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')

            # Hide the config information from bitbake output (since it's fixed within the SDK)
            f.write('BUILDCFG_HEADER = ""\n')

            # Allow additional config through sdk-extra.conf
            fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
            if fn:
                with open(fn, 'r') as xf:
                    for line in xf:
                        f.write(line)

            # If you define a sdk_extraconf() function then it can contain additional config
            # (Though this is awkward; sdk-extra.conf should probably be used instead)
            extraconf = (d.getVar('sdk_extraconf', True) or '').strip()
            if extraconf:
                # Strip off any leading / trailing spaces
                for line in extraconf.splitlines():
                    f.write(line.strip() + '\n')

            f.write('require conf/locked-sigs.inc\n')
            f.write('require conf/unlocked-sigs.inc\n')

    if os.path.exists(builddir + '/conf/auto.conf'):
        if derivative:
            shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
        else:
            with open(builddir + '/conf/auto.conf', 'r') as f:
                oldlines = f.readlines()
            (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
            with open(baseoutpath + '/conf/auto.conf', 'w') as f:
                f.write('# WARNING: this configuration has been automatically generated and in\n')
                f.write('# most cases should not be edited. If you need more flexibility than\n')
                f.write('# this configuration provides, it is strongly suggested that you set\n')
                f.write('# up a proper instance of the full build system and use that instead.\n\n')
                for line in newlines:
                    if line.strip() and not line.startswith('#'):
                        f.write(line)

    # Ensure any variables set from the external environment (by way of
    # BB_ENV_EXTRAWHITE) are set in the SDK's configuration
    extralines = []
    for name, value in env_whitelist_values.iteritems():
        actualvalue = d.getVar(name, True) or ''
        if value != actualvalue:
            extralines.append('%s = "%s"\n' % (name, actualvalue))
    if extralines:
        with open(baseoutpath + '/conf/local.conf', 'a') as f:
            f.write('\n')
            f.write('# Extra settings from environment:\n')
            for line in extralines:
                f.write(line)
            f.write('\n')

    # Filter the locked signatures file to just the sstate tasks we are interested in
    excluded_targets = d.getVar('SDK_TARGETS', True)
    sigfile = d.getVar('WORKDIR', True) + '/locked-sigs.inc'
    lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc'
    oe.copy_buildsystem.prune_lockedsigs([],
                                         excluded_targets.split(),
                                         sigfile,
                                         lockedsigs_pruned)

    sstate_out = baseoutpath + '/sstate-cache'
    bb.utils.remove(sstate_out, True)
    # uninative.bbclass sets NATIVELSBSTRING to 'universal'
    fixedlsbstring = 'universal'

    # Add packagedata if enabled
    if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
        lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base.inc'
        lockedsigs_copy = d.getVar('WORKDIR', True) + '/locked-sigs-copy.inc'
        shutil.move(lockedsigs_pruned, lockedsigs_base)
        oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'],
                                             lockedsigs_base,
                                             d.getVar('STAGING_DIR_HOST', True) + '/world-pkgdata/locked-sigs-pkgdata.inc',
                                             lockedsigs_pruned,
                                             lockedsigs_copy)

    if d.getVar('SDK_EXT_TYPE', True) == 'minimal':
        if derivative:
            # Assume the user is not going to set up an additional sstate
            # mirror, thus we need to copy the additional artifacts (from
            # workspace recipes) into the derivative SDK
            lockedsigs_orig = d.getVar('TOPDIR', True) + '/conf/locked-sigs.inc'
            if os.path.exists(lockedsigs_orig):
                lockedsigs_extra = d.getVar('WORKDIR', True) + '/locked-sigs-extra.inc'
                oe.copy_buildsystem.merge_lockedsigs(None,
                                                     lockedsigs_orig,
                                                     lockedsigs_pruned,
                                                     None,
                                                     lockedsigs_extra)
                oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
                                                               d.getVar('SSTATE_DIR', True),
                                                               sstate_out, d,
                                                               fixedlsbstring)
    else:
        oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
                                                       d.getVar('SSTATE_DIR', True),
                                                       sstate_out, d,
                                                       fixedlsbstring)

    # We don't need sstate do_package files
    for root, dirs, files in os.walk(sstate_out):
        for name in files:
            if name.endswith("_package.tgz"):
                f = os.path.join(root, name)
                os.remove(f)

    # Write manifest file
    # Note: at the moment we cannot include the env setup script here to keep
    # it updated, since it gets modified during SDK installation (see
    # sdk_ext_postinst() below) thus the checksum we take here would always
    # be different.
    manifest_file_list = ['conf/*']
    manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest')
    with open(manifest_file, 'w') as f:
        for item in manifest_file_list:
            for fn in glob.glob(os.path.join(baseoutpath, item)):
                if fn == manifest_file:
                    continue
                chksum = bb.utils.sha256_file(fn)
                f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath)))
}

def extsdk_get_buildtools_filename(d):
    return '*-buildtools-nativesdk-standalone-*.sh'

install_tools() {
	install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}
	lnr ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath}/devtool ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/devtool
	lnr ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath}/recipetool ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/recipetool
	touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase

	localconf=${SDK_OUTPUT}/${SDKPATH}/conf/local.conf

	# find latest buildtools-tarball and install it
	buildtools_path=`ls -t1 ${SDK_DEPLOY}/${@extsdk_get_buildtools_filename(d)} | head -n1`
	install $buildtools_path ${SDK_OUTPUT}/${SDKPATH}

	# For now this is where uninative.bbclass expects the tarball
	chksum=`sha256sum ${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2 | cut -f 1 -d ' '`
	install -d ${SDK_OUTPUT}/${SDKPATH}/downloads/uninative/$chksum/
	install ${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2 ${SDK_OUTPUT}/${SDKPATH}/downloads/uninative/$chksum/
	echo "UNINATIVE_CHECKSUM[${BUILD_ARCH}] = '$chksum'" >> ${SDK_OUTPUT}/${SDKPATH}/conf/local.conf

	install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
}
do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"

# Since bitbake won't run as root it doesn't make sense to try and install
# the extensible sdk as root.
sdk_ext_preinst() {
	if [ "`id -u`" = "0" ]; then
		echo "ERROR: The extensible sdk cannot be installed as root."
		exit 1
	fi
	SDK_EXTENSIBLE="1"
	if [ "$publish" = "1" ] ; then
		EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=ext-sdk-prepare.py"
		if [ "${SDK_EXT_TYPE}" = "minimal" ] ; then
			EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache"
		fi
	fi
}
SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_preinst}"

# FIXME this preparation should be done as part of the SDK construction
sdk_ext_postinst() {
	printf "\nExtracting buildtools...\n"
	cd $target_sdk_dir
	printf "buildtools\ny" | ./*buildtools-nativesdk-standalone* > /dev/null || ( printf 'ERROR: buildtools installation failed\n' ; exit 1 )

	# Delete the buildtools tar file since it won't be used again
	rm ./*buildtools-nativesdk-standalone*.sh -f

	# Make sure when the user sets up the environment, they also get
	# the buildtools-tarball tools in their path.
	env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
	echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script

	# Allow bitbake environment setup to be ran as part of this sdk.
	echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script

	# A bit of another hack, but we need this in the path only for devtool
	# so put it at the end of $PATH.
	echo "export PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $env_setup_script

	echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script

	# Warn if trying to use external bitbake and the ext SDK together
	echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script

	if [ "$prepare_buildsystem" != "no" ]; then
		printf "Preparing build system...\n"
		# dash which is /bin/sh on Ubuntu will not preserve the
		# current working directory when first ran, nor will it set $1 when
		# sourcing a script. That is why this has to look so ugly.
		LOGFILE="$target_sdk_dir/preparing_build_system.log"
		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py '${SDK_INSTALL_TARGETS}' >> $LOGFILE 2>&1" || { echo "ERROR: SDK preparation failed: see $LOGFILE"; cat "$LOGFILE"; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
		rm $target_sdk_dir/ext-sdk-prepare.py
	fi
	echo done
}

SDK_POST_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_postinst}"

SDK_POSTPROCESS_COMMAND_prepend_task-populate-sdk-ext = "copy_buildsystem; install_tools; "

SDK_INSTALL_TARGETS = ""
fakeroot python do_populate_sdk_ext() {
    # FIXME hopefully we can remove this restriction at some point, but uninative
    # currently forces this upon us
    if d.getVar('SDK_ARCH', True) != d.getVar('BUILD_ARCH', True):
        bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH', True), d.getVar('BUILD_ARCH', True)))

    d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))

    bb.build.exec_func("do_populate_sdk", d)
}

def get_ext_sdk_depends(d):
    return d.getVarFlag('do_rootfs', 'depends', True) + ' ' + d.getVarFlag('do_build', 'depends', True)

python do_sdk_depends() {
    # We have to do this separately in its own task so we avoid recursing into
    # dependencies we don't need to (e.g. buildtools-tarball) and bringing those
    # into the SDK's sstate-cache
    import oe.copy_buildsystem
    sigfile = d.getVar('WORKDIR', True) + '/locked-sigs.inc'
    oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
}
addtask sdk_depends

do_sdk_depends[dirs] = "${WORKDIR}"
do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)}"
do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}"
do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}"
do_sdk_depends[rdepends] = "${@get_sdk_ext_rdepends(d)}"

def get_sdk_ext_rdepends(d):
    localdata = d.createCopy()
    localdata.appendVar('OVERRIDES', ':task-populate-sdk-ext')
    bb.data.update_data(localdata)
    return localdata.getVarFlag('do_populate_sdk', 'rdepends', True)

do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}"

do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \
                                buildtools-tarball:do_populate_sdk uninative-tarball:do_populate_sdk \
                                ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1' else ''}"

do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':do_build' for x in d.getVar('SDK_TARGETS', True).split()])}"

# Make sure code changes can result in rebuild
do_populate_sdk_ext[vardeps] += "copy_buildsystem \
                                 sdk_ext_postinst"

# Since any change in the metadata of any layer should cause a rebuild of the
# sdk(since the layers are put in the sdk) set the task to nostamp so it
# always runs.
do_populate_sdk_ext[nostamp] = "1"

addtask populate_sdk_ext after do_sdk_depends