Start sstate

Add pre clean hook
Add sstate_installpkg
sstate: Tie all pieces together with a setscene function
Use pythonic functions rather than os.system()
Implement sstate_clean function
package_ipk: clean shared state in prestate
packaged-staging2: implement fetching packages from mirror
Copy add staging_fetch() from packaged-staging.bbclass
packaged-staging2: make the output slightly friendlier
packaged-staging2: handle directories in the shared state
Add directories to the manifest when installing and remove them if empty when
cleaning.
Move most of the sstate code into the class itself removing the need for heavy boilderplate

packaged-staging2: Programatically ensure directories are last in the manifest
packaged-staging2: add cleanall
packaged-staging2.bbclass: Connect in cleanall function to do_clean

We want to ensure that directories appear at the end of the manifest so that
when we test to see if they should be deleted any contents added by the task
will have been removed first.

pstage2: Use oe.path.copytree
packaged-staging2: support lockfiles and plaindirs

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Signed-off-by: Joshua Lock <josh@linux.intel.com>

2 files changed, 298 insertions, 1 deletions

diff --git a/meta/classes/packaged-staging2.bbclass b/meta/classes/packaged-staging2.bbclass
new file mode 100644
index 0000000000..6476257fc8
--- /dev/null
+++ b/meta/classes/packaged-staging2.bbclass
@@ -0,0 +1,297 @@
+PSTAGE2_MANIFESTS = "${TMPDIR}/pstagelogs"
+PSTAGE2_MANFILEPREFIX = "${PSTAGE2_MANIFESTS}/manifest-${PSTAGE2_PKGARCH}-${PN}"
+
+
+PSTAGE2_PKGARCH    = "${TARGET_ARCH}"
+PSTAGE2_PKGVERSION = "${PV}-${PR}"
+PSTAGE2_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
+
+PSTAGE2_PKGNAME    = "${PSTAGE2_PKGPN}_${PSTAGE2_PKGVERSION}_${PSTAGE2_PKGARCH}"
+PSTAGE2_EXTRAPATH  ?= ""
+PSTAGE2_PKGPATH    = "${DISTRO}/${OELAYOUT_ABI}${PSTAGE2_EXTRAPATH}"
+PSTAGE2_PKG        = "${PSTAGE_DIR}2/${PSTAGE2_PKGPATH}/${PSTAGE2_PKGNAME}"
+
+PSTAGE2_SCAN_CMD ?= "find ${PSTAGE2_BUILDDIR} \( -name "*.la" -o -name "*-config" \) -type f"
+
+python () {
+    if bb.data.inherits_class('native', d):
+        bb.data.setVar('PSTAGE2_PKGARCH', bb.data.getVar('BUILD_ARCH', d), d)
+    elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
+        bb.data.setVar('PSTAGE2_PKGARCH', bb.data.expand("${BUILD_ARCH}_${TARGET_ARCH}", d), d)
+    elif bb.data.inherits_class('nativesdk', d):
+        bb.data.setVar('PSTAGE2_PKGARCH', bb.data.expand("${SDK_ARCH}", d), d)
+    elif bb.data.inherits_class('cross-canadian', d):
+        bb.data.setVar('PSTAGE2_PKGARCH', bb.data.expand("${SDK_ARCH}_${TARGET_ARCH}", d), d)
+
+    # These classes encode staging paths into their scripts data so can only be
+    # reused if we manipulate the paths
+    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d):
+        scan_cmd = "grep -Irl ${STAGING_DIR} ${PSTAGE2_BUILDDIR}"
+        bb.data.setVar('PSTAGE2_SCAN_CMD', scan_cmd, d)
+
+    for task in (bb.data.getVar('SSTATETASKS', d, True) or "").split():
+        funcs = bb.data.getVarFlag(task, 'prefuncs', d) or ""
+        funcs = "sstate_task_prefunc " + funcs
+        bb.data.setVarFlag(task, 'prefuncs', funcs, d)
+        funcs = bb.data.getVarFlag(task, 'postfuncs', d) or ""
+        funcs = "sstate_task_postfunc " + funcs
+        bb.data.setVarFlag(task, 'postfuncs', funcs, d)
+}
+
+def sstate_init(name, d):
+    ss = {}
+    ss['name'] = name
+    ss['dirs'] = []
+    ss['plaindirs'] = []
+    ss['lockfiles'] = []
+    return ss
+
+def sstate_state_fromvars(d):
+    task = bb.data.getVar('BB_CURRENTTASK', d, True)
+    if not task:
+        bb.fatal("sstate code running without task context?!")
+    task = task.replace("_setscene", "")
+
+    name = bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-name', d), d)
+    inputs = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-inputdirs', d) or "", d)).split()
+    outputs = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-outputdirs', d) or "", d)).split()
+    plaindirs = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-plaindirs', d) or "", d)).split()
+    lockfiles = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-lockfile', d) or "", d)).split()
+    if not name or len(inputs) != len(outputs):
+        bb.fatal("sstate variables not setup correctly?!")
+
+    ss = sstate_init(name, d)
+    for i in range(len(inputs)):
+        sstate_add(ss, inputs[i], outputs[i], d)
+    ss['lockfiles'] = lockfiles
+    ss['plaindirs'] = plaindirs
+    return ss
+
+def sstate_add(ss, source, dest, d):
+    srcbase = os.path.basename(source)
+    ss['dirs'].append([srcbase, source, dest])
+    return ss
+
+def sstate_install(ss, d):
+    import oe.path
+
+    sharedfiles = []
+    shareddirs = []
+    bb.mkdirhier(bb.data.expand("${PSTAGE2_MANIFESTS}", d))
+    manifest = bb.data.expand("${PSTAGE2_MANFILEPREFIX}.%s" % ss['name'], d)
+
+    if os.access(manifest, os.R_OK):
+        bb.fatal("Package already staged (%s)?!" % manifest)
+
+    locks = []
+    for lock in ss['lockfiles']:
+        locks.append(bb.utils.lockfile(lock))
+
+    for state in ss['dirs']:
+        oe.path.copytree(state[1], state[2])
+        for walkroot, dirs, files in os.walk(state[1]):
+            for file in files:
+                srcpath = os.path.join(walkroot, file)
+                dstpath = srcpath.replace(state[1], state[2])
+                bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
+                sharedfiles.append(dstpath)
+            for dir in dirs:
+                dir = os.path.join(state[2], dir)
+                if not dir.endswith("/"):
+                    dir = dir + "/"
+                shareddirs.append(dir)
+    f = open(manifest, "w")
+    for file in sharedfiles:
+        f.write(file + "\n")
+    # We want to ensure that directories appear at the end of the manifest
+    # so that when we test to see if they should be deleted any contents
+    # added by the task will have been removed first.
+    for dir in shareddirs:
+        f.write(dir + "\n")
+    f.close()
+
+    for lock in locks:
+        bb.utils.unlockfile(lock)
+
+def sstate_installpkg(ss, d):
+    import oe.path
+
+    pstageinst = bb.data.expand("${WORKDIR}/pstage-install-%s/" % ss['name'], d)
+    pstagepkg = bb.data.getVar('PSTAGE2_PKG', d, True) + '_' + ss['name'] + ".tgz"
+
+    if not os.path.exists(pstagepkg):
+       pstaging_fetch(pstagepkg, d)
+
+    if not os.path.isfile(pstagepkg):
+        bb.note("Staging package %s does not exist" % pstagepkg)
+        return False
+
+    sstate_clean(ss, d)
+
+    bb.data.setVar('PSTAGE2_INSTDIR', pstageinst, d)
+    bb.data.setVar('PSTAGE2_PKG', pstagepkg, d)
+    bb.build.exec_func('sstate_unpack_package', d)
+
+    # Fixup hardcoded paths
+    fixmefn =  pstageinst + "fixmepath"
+    if os.path.isfile(fixmefn):
+        staging = bb.data.getVar('STAGING_DIR', d, True)
+        fixmefd = open(fixmefn, "r")
+        fixmefiles = fixmefd.readlines()
+        fixmefd.close()
+        for file in fixmefiles:
+            os.system("sed -i -e s:FIXMESTAGINGDIR:%s:g %s" % (staging, pstageinst + file))
+
+    for state in ss['dirs']:
+        if os.path.exists(state[1]):
+            oe.path.remove(state[1])
+        oe.path.copytree(pstageinst + state[0], state[1])
+    sstate_install(ss, d)
+
+    for plain in ss['plaindirs']:
+        bb.mkdirhier(pstageinst + plain)
+        oe.path.copytree(pstageinst + plain, bb.data.getVar('WORKDIR', d, True) + plain)
+
+    return True
+
+def sstate_clean_manifest(manifest, d):
+    import oe.path
+
+    if not os.path.exists(manifest):
+       return
+
+    mfile = open(manifest)
+    entries = mfile.readlines()
+    mfile.close()
+
+    for entry in entries:
+        entry = entry.strip()
+        if entry.endswith("/"):
+           if os.path.exists(entry) and len(os.listdir(entry)) == 0:
+              os.rmdir(entry)
+        else:
+           oe.path.remove(entry)
+
+    oe.path.remove(manifest)
+
+def sstate_clean(ss, d):
+
+    manifest = bb.data.expand("${PSTAGE2_MANFILEPREFIX}.%s" % ss['name'], d)
+
+    locks = []
+    for lock in ss['lockfiles']:
+        locks.append(bb.utils.lockfile(lock))
+
+    sstate_clean_manifest(manifest, d)
+
+    for lock in locks:
+        bb.utils.unlockfile(lock)
+
+python sstate_cleanall() {
+    import fnmatch
+
+    bb.note("Removing %s from staging" % bb.data.getVar('PN', d, True))
+
+    manifest_dir = bb.data.getVar('PSTAGE2_MANIFESTS', d, True)
+    manifest_pattern = bb.data.expand("manifest-${PN}.*", d)
+
+    for manifest in (os.listdir(manifest_dir)):
+        if fnmatch.fnmatch(manifest, manifest_pattern):
+             sstate_clean_manifest(manifest_dir + "/" + manifest, d)
+}
+
+do_clean[postfuncs] += "sstate_cleanall"
+do_clean[dirs] += "${PSTAGE2_MANIFESTS}"
+
+def sstate_package(ss, d):
+    import oe.path
+
+    pstagebuild = bb.data.expand("${WORKDIR}/pstage-build-%s/" % ss['name'], d)
+    pstagepkg = bb.data.getVar('PSTAGE2_PKG', d, True) + '_'+ ss['name'] + ".tgz"
+    bb.mkdirhier(pstagebuild)
+    bb.mkdirhier(os.path.dirname(pstagepkg))
+    for state in ss['dirs']:
+        srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
+        oe.path.copytree(state[1], pstagebuild + state[0])
+        for walkroot, dirs, files in os.walk(state[1]):
+            for file in files:
+                srcpath = os.path.join(walkroot, file)
+                dstpath = srcpath.replace(state[1], pstagebuild + state[0])
+                bb.debug(2, "Preparing %s for packaging at %s" % (srcpath, dstpath))
+
+    workdir = bb.data.getVar('WORKDIR', d, True)
+    for plain in ss['plaindirs']:
+        pdir = plain.replace(workdir, pstagebuild)
+        bb.mkdirhier(plain)
+        bb.mkdirhier(pdir)
+        oe.path.copytree(plain, pdir)
+
+    bb.data.setVar('PSTAGE2_BUILDDIR', pstagebuild, d)
+    bb.data.setVar('PSTAGE2_PKG', pstagepkg, d)
+    bb.build.exec_func('sstate_create_package', d)
+
+    return
+
+def pstaging_fetch(pstagepkg, d):
+    import bb.fetch
+
+    # only try and fetch if the user has configured a mirror
+    if bb.data.getVar('PSTAGE_MIRROR', d) != "":
+        # Copy the data object and override DL_DIR and SRC_URI
+        pd = d.createCopy()
+        dldir = bb.data.expand("${PSTAGE_DIR}/${PSTAGE_PKGPATH}", pd)
+        mirror = bb.data.expand("${PSTAGE_MIRROR}/${PSTAGE_PKGPATH}/", pd)
+        srcuri = mirror + os.path.basename(pstagepkg)
+        bb.data.setVar('DL_DIR', dldir, pd)
+        bb.data.setVar('SRC_URI', srcuri, pd)
+
+        # Try a fetch from the pstage mirror, if it fails just return and
+        # we will build the package
+        try:
+            bb.fetch.init([srcuri], pd)
+            bb.fetch.go(pd, [srcuri])
+        except:
+            return
+
+def sstate_setscene(d):
+    shared_state = sstate_state_fromvars(d)
+    accelerate = sstate_installpkg(shared_state, d)
+    if not accelerate:
+        raise bb.build.FuncFailed("No suitable staging package found")
+
+python sstate_task_prefunc () {
+    shared_state = sstate_state_fromvars(d)
+    sstate_clean(shared_state, d)
+}
+
+python sstate_task_postfunc () {
+    shared_state = sstate_state_fromvars(d)
+    sstate_install(shared_state, d)
+    sstate_package(shared_state, d)
+}
+  
+
+#
+# Shell function to generate a pstage package from a directory
+# set as PSTAGE2_BUILDDIR
+#
+sstate_create_package () {
+	# Need to remove hardcoded paths and fix these when we install the
+	# staging packages.
+	for i in `${PSTAGE2_SCAN_CMD}` ; do \
+		sed -i -e s:${STAGING_DIR}:FIXMESTAGINGDIR:g $i
+		echo $i | sed -e 's:${PSTAGE2_BUILDDIR}::' >> ${PSTAGE2_BUILDDIR}fixmepath
+	done
+
+	cd ${PSTAGE2_BUILDDIR}
+	tar -cvzf ${PSTAGE2_PKG} *
+}
+
+#
+# Shell function to decompress and prepare a package for installation
+#
+sstate_unpack_package () {
+	mkdir -p ${PSTAGE2_INSTDIR}
+	cd ${PSTAGE2_INSTDIR}
+	tar -xvzf ${PSTAGE2_PKG}
+}
diff --git a/meta/conf/distro/poky.conf b/meta/conf/distro/poky.conf
index df07d04a90..d6d335e5a6 100644
--- a/meta/conf/distro/poky.conf
+++ b/meta/conf/distro/poky.conf
@@ -11,7 +11,7 @@ USER_CLASSES ?= ""
 
 PACKAGE_CLASSES ?= "package_ipk"
 INHERIT_INSANE ?= "insane"
-INHERIT += "${PACKAGE_CLASSES} ${USER_CLASSES} debian poky devshell ${INHERIT_INSANE} packaged-staging"
+INHERIT += "${PACKAGE_CLASSES} ${USER_CLASSES} debian poky devshell ${INHERIT_INSANE} packaged-staging packaged-staging2"
 # For some reason, this doesn't work
 # TARGET_OS ?= "linux"
 # TARGET_VENDOR ?= "-poky"