From 7971cb0aa3e517a53f0ce6d3ee9bc3179041ccb8 Mon Sep 17 00:00:00 2001 From: John Klug Date: Wed, 25 May 2022 17:12:18 -0500 Subject: mLinux 6 --- .gitignore | 7 + .gitmodules | 57 + .templateconf | 2 + LICENSE | 27 + README | 121 ++ bitbake | 1 + conf/.gitignore | 2 + conf/bblayers.conf.mlinux | 50 + conf/bblayers.conf.sample | 22 + conf/local.conf.sample | 51 + install-deps/install-centos-rhel-deps.sh | 4 + install-deps/install-debian-ubuntu-deps.sh | 4 + install-deps/install-fedora-deps.sh | 5 + install-deps/install-opensuse-deps.sh | 4 + layers/meta-java | 1 + layers/meta-mlinux | 1 + layers/meta-mlinux-atmel | 1 + layers/meta-mono | 1 + layers/meta-multitech | 1 + layers/meta-multitech-atmel | 1 + layers/meta-nodejs | 1 + layers/meta-openembedded | 1 + layers/meta-python2 | 1 + layers/meta-security | 1 + layers/meta-swupdate | 1 + layers/meta-virtualization | 1 + layers/openembedded-core | 1 + oe-init-build-env | 77 + oe-init-build-env-memres | 94 + patches/bitbake_checksumwarningsuppress.patch | 18 + patches/dont_use_network_on_unpack_of_tag.patch | 95 + patches/kernel-revision-ext-mod.patch | 25 + patches/meta-openembedded-github-https.patch | 171 ++ patches/no_net_check.patch | 12 + patches/openembedded-core-github-https.patch | 14 + .../rename-mtr-conflicts-with-mtr-product.patch | 58 + scripts/README | 1 + scripts/bitbake-prserv-tool | 112 ++ scripts/bitbake-whatchanged | 339 ++++ scripts/buildhistory-collect-srcrevs | 117 ++ scripts/buildhistory-diff | 100 + scripts/cleanup-workdir | 198 ++ scripts/combo-layer | 924 ++++++++++ scripts/combo-layer-hook-default.sh | 20 + scripts/combo-layer.conf.example | 93 + scripts/contrib/bb-perf/bb-matrix-plot.sh | 137 ++ scripts/contrib/bb-perf/bb-matrix.sh | 79 + scripts/contrib/bb-perf/buildstats.sh | 90 + scripts/contrib/bbvars.py | 186 ++ scripts/contrib/build-perf-test.sh | 400 ++++ scripts/contrib/ddimage | 104 ++ scripts/contrib/devtool-stress.py | 242 +++ scripts/contrib/dialog-power-control | 53 + scripts/contrib/documentation-audit.sh | 94 + scripts/contrib/graph-tool | 92 + scripts/contrib/list-packageconfig-flags.py | 179 ++ scripts/contrib/mkefidisk.sh | 459 +++++ scripts/contrib/python/generate-manifest-2.7.py | 397 ++++ scripts/contrib/python/generate-manifest-3.5.py | 396 ++++ scripts/contrib/serdevtry | 60 + scripts/contrib/test_build_time.sh | 237 +++ scripts/contrib/test_build_time_worker.sh | 37 + scripts/contrib/verify-homepage.py | 62 + scripts/cp-noerror | 52 + scripts/create-pull-request | 276 +++ scripts/crosstap | 148 ++ scripts/devtool | 347 ++++ scripts/gen-lockedsig-cache | 57 + scripts/gen-site-config | 53 + scripts/git-hooks/post-checkout | 3 + scripts/git-hooks/post-merge | 3 + scripts/git-hooks/pre-commit | 23 + scripts/git-hooks/pre-push | 23 + scripts/lib/argparse_oe.py | 129 ++ scripts/lib/bsp/__init__.py | 22 + scripts/lib/bsp/engine.py | 1947 ++++++++++++++++++++ scripts/lib/bsp/help.py | 1046 +++++++++++ scripts/lib/bsp/kernel.py | 1072 +++++++++++ .../lib/bsp/substrate/target/arch/arm/.gitignore | 0 .../target/arch/arm/conf/machine/machine.conf | 102 + .../xserver-xf86-config/machine.noinstall | 1 + .../xserver-xf86-config/machine/xorg.conf | 34 + .../xorg-xserver/xserver-xf86-config_0.1.bbappend | 2 + .../arch/arm/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-non_hardware.cfg | 31 + .../linux/files/machine-preempt-rt.scc | 15 + .../linux/files/machine-standard.scc | 15 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../arm/recipes-kernel/linux/files/machine.cfg | 321 ++++ .../arm/recipes-kernel/linux/files/machine.scc | 8 + .../arm/recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 25 + .../linux/linux-yocto-tiny_4.1.bbappend | 33 + .../linux/linux-yocto-tiny_4.4.bbappend | 33 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 32 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 32 + .../bsp/substrate/target/arch/common/COPYING.MIT | 17 + .../lib/bsp/substrate/target/arch/common/README | 118 ++ .../substrate/target/arch/common/README.sources | 17 + .../substrate/target/arch/common/binary/.gitignore | 0 .../substrate/target/arch/common/conf/layer.conf | 10 + .../formfactor/formfactor/machine.noinstall | 1 + .../formfactor/formfactor/machine/machconfig | 5 + .../recipes-bsp/formfactor/formfactor_0.0.bbappend | 2 + .../recipes-kernel/linux/kernel-list.noinstall | 26 + .../recipes-kernel/linux/linux-yocto-custom.bb | 57 + .../linux/linux-yocto-custom.noinstall | 1 + .../linux/linux-yocto-custom/defconfig | 5 + .../linux-yocto-custom/machine-user-config.cfg | 9 + .../linux-yocto-custom/machine-user-patches.scc | 9 + .../linux/linux-yocto-custom/machine.cfg | 4 + .../linux/linux-yocto-custom/machine.scc | 18 + .../target/arch/i386/conf/machine/machine.conf | 77 + .../xserver-xf86-config/machine.noinstall | 1 + .../xserver-xf86-config/machine/xorg.conf | 1 + .../xorg-xserver/xserver-xf86-config_0.1.bbappend | 2 + .../arch/i386/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-preempt-rt.scc | 17 + .../linux/files/machine-standard.scc | 17 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../i386/recipes-kernel/linux/files/machine.cfg | 55 + .../i386/recipes-kernel/linux/files/machine.scc | 21 + .../recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 25 + .../linux/linux-yocto-tiny_4.1.bbappend | 33 + .../linux/linux-yocto-tiny_4.4.bbappend | 33 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 32 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 32 + .../bsp/substrate/target/arch/layer/COPYING.MIT | 17 + scripts/lib/bsp/substrate/target/arch/layer/README | 64 + .../substrate/target/arch/layer/conf/layer.conf | 10 + .../target/arch/layer/layer-questions.noinstall | 14 + .../arch/layer/recipes-example-bbappend.noinstall | 1 + .../example-bbappend-version.bbappend | 9 + .../example-bbappend-version.noinstall | 1 + .../example-bbappend-version/example.patch | 12 + .../target/arch/layer/recipes-example.noinstall | 1 + .../recipes-example/example/example-recipe-0.1.bb | 23 + .../example/example-recipe-0.1.noinstall | 1 + .../example/example-recipe-0.1/example.patch | 12 + .../example/example-recipe-0.1/helloworld.c | 8 + .../lib/bsp/substrate/target/arch/mips/.gitignore | 0 .../target/arch/mips/conf/machine/machine.conf | 39 + .../arch/mips/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-preempt-rt.scc | 11 + .../linux/files/machine-standard.scc | 11 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../mips/recipes-kernel/linux/files/machine.cfg | 2 + .../mips/recipes-kernel/linux/files/machine.scc | 8 + .../recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 25 + .../linux/linux-yocto-tiny_4.1.bbappend | 33 + .../linux/linux-yocto-tiny_4.4.bbappend | 33 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 32 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 32 + .../bsp/substrate/target/arch/mips64/.gitignore | 0 .../target/arch/mips64/conf/machine/machine.conf | 39 + .../mips64/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-preempt-rt.scc | 11 + .../linux/files/machine-standard.scc | 11 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../mips64/recipes-kernel/linux/files/machine.cfg | 66 + .../mips64/recipes-kernel/linux/files/machine.scc | 8 + .../recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 25 + .../linux/linux-yocto-tiny_4.1.bbappend | 33 + .../linux/linux-yocto-tiny_4.4.bbappend | 33 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 32 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 32 + .../bsp/substrate/target/arch/powerpc/.gitignore | 0 .../target/arch/powerpc/conf/machine/machine.conf | 87 + .../powerpc/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-preempt-rt.scc | 11 + .../linux/files/machine-standard.scc | 11 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../powerpc/recipes-kernel/linux/files/machine.cfg | 164 ++ .../powerpc/recipes-kernel/linux/files/machine.scc | 10 + .../recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 25 + .../linux/linux-yocto-tiny_4.1.bbappend | 33 + .../linux/linux-yocto-tiny_4.4.bbappend | 33 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 32 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 32 + .../target/arch/qemu/conf/machine/machine.conf | 74 + .../init-ifupdown/init-ifupdown/machine.noinstall | 1 + .../init-ifupdown/init-ifupdown/machine/interfaces | 5 + .../init-ifupdown/init-ifupdown_1.0.bbappend | 1 + .../xserver-xf86-config/machine.noinstall | 1 + .../xserver-xf86-config/machine/xorg.conf | 77 + .../xorg-xserver/xserver-xf86-config_0.1.bbappend | 1 + .../arch/qemu/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-preempt-rt.scc | 11 + .../linux/files/machine-standard.scc | 20 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../qemu/recipes-kernel/linux/files/machine.cfg | 1 + .../qemu/recipes-kernel/linux/files/machine.scc | 5 + .../recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 55 + .../linux/linux-yocto-tiny_4.1.bbappend | 62 + .../linux/linux-yocto-tiny_4.4.bbappend | 62 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 61 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 61 + .../bsp/substrate/target/arch/x86_64/.gitignore | 0 .../target/arch/x86_64/conf/machine/machine.conf | 65 + .../xserver-xf86-config/machine.noinstall | 1 + .../xserver-xf86-config/machine/xorg.conf | 1 + .../xorg-xserver/xserver-xf86-config_0.1.bbappend | 2 + .../x86_64/recipes-kernel/linux/files.noinstall | 1 + .../linux/files/machine-preempt-rt.scc | 17 + .../linux/files/machine-standard.scc | 17 + .../recipes-kernel/linux/files/machine-tiny.scc | 11 + .../linux/files/machine-user-config.cfg | 1 + .../linux/files/machine-user-features.scc | 1 + .../linux/files/machine-user-patches.scc | 1 + .../x86_64/recipes-kernel/linux/files/machine.cfg | 48 + .../x86_64/recipes-kernel/linux/files/machine.scc | 14 + .../recipes-kernel/linux/kernel-list.noinstall | 5 + .../recipes-kernel/linux/linux-yocto-dev.bbappend | 25 + .../linux/linux-yocto-tiny_4.1.bbappend | 33 + .../linux/linux-yocto-tiny_4.4.bbappend | 33 + .../recipes-kernel/linux/linux-yocto_4.1.bbappend | 32 + .../recipes-kernel/linux/linux-yocto_4.4.bbappend | 32 + scripts/lib/bsp/tags.py | 49 + scripts/lib/devtool/__init__.py | 257 +++ scripts/lib/devtool/build.py | 86 + scripts/lib/devtool/build_image.py | 168 ++ scripts/lib/devtool/build_sdk.py | 65 + scripts/lib/devtool/deploy.py | 304 +++ scripts/lib/devtool/package.py | 62 + scripts/lib/devtool/runqemu.py | 65 + scripts/lib/devtool/sdk.py | 366 ++++ scripts/lib/devtool/search.py | 88 + scripts/lib/devtool/standard.py | 1452 +++++++++++++++ scripts/lib/devtool/upgrade.py | 382 ++++ scripts/lib/devtool/utilcmds.py | 233 +++ scripts/lib/recipetool/__init__.py | 0 scripts/lib/recipetool/append.py | 471 +++++ scripts/lib/recipetool/create.py | 963 ++++++++++ scripts/lib/recipetool/create_buildsys.py | 859 +++++++++ scripts/lib/recipetool/create_buildsys_python.py | 719 ++++++++ scripts/lib/recipetool/create_kernel.py | 99 + scripts/lib/recipetool/create_kmod.py | 152 ++ scripts/lib/recipetool/create_npm.py | 156 ++ scripts/lib/recipetool/newappend.py | 112 ++ scripts/lib/recipetool/setvar.py | 75 + scripts/lib/scriptpath.py | 42 + scripts/lib/scriptutils.py | 118 ++ scripts/lib/wic/__init__.py | 4 + scripts/lib/wic/__version__.py | 1 + scripts/lib/wic/canned-wks/common.wks.inc | 3 + .../canned-wks/directdisk-bootloader-config.cfg | 11 + .../canned-wks/directdisk-bootloader-config.wks | 8 + scripts/lib/wic/canned-wks/directdisk-gpt.wks | 10 + .../lib/wic/canned-wks/directdisk-multi-rootfs.wks | 23 + scripts/lib/wic/canned-wks/directdisk.wks | 8 + scripts/lib/wic/canned-wks/mkefidisk.wks | 11 + scripts/lib/wic/canned-wks/mkgummidisk.wks | 11 + scripts/lib/wic/canned-wks/mkhybridiso.wks | 7 + scripts/lib/wic/canned-wks/qemux86-directdisk.wks | 8 + scripts/lib/wic/canned-wks/sdimage-bootpart.wks | 6 + scripts/lib/wic/conf.py | 103 ++ scripts/lib/wic/config/wic.conf | 6 + scripts/lib/wic/creator.py | 124 ++ scripts/lib/wic/engine.py | 220 +++ scripts/lib/wic/help.py | 777 ++++++++ scripts/lib/wic/imager/__init__.py | 0 scripts/lib/wic/imager/baseimager.py | 192 ++ scripts/lib/wic/imager/direct.py | 380 ++++ scripts/lib/wic/ksparser.py | 169 ++ scripts/lib/wic/msger.py | 309 ++++ scripts/lib/wic/partition.py | 414 +++++ scripts/lib/wic/plugin.py | 150 ++ scripts/lib/wic/pluginbase.py | 108 ++ scripts/lib/wic/plugins/imager/direct_plugin.py | 102 + scripts/lib/wic/plugins/source/bootimg-efi.py | 237 +++ .../lib/wic/plugins/source/bootimg-partition.py | 140 ++ scripts/lib/wic/plugins/source/bootimg-pcbios.py | 210 +++ scripts/lib/wic/plugins/source/fsimage.py | 73 + .../lib/wic/plugins/source/isoimage-isohybrid.py | 550 ++++++ scripts/lib/wic/plugins/source/rawcopy.py | 88 + scripts/lib/wic/plugins/source/rootfs.py | 83 + .../lib/wic/plugins/source/rootfs_pcbios_ext.py | 177 ++ scripts/lib/wic/test | 1 + scripts/lib/wic/utils/__init__.py | 0 scripts/lib/wic/utils/errors.py | 29 + scripts/lib/wic/utils/fs_related.py | 84 + scripts/lib/wic/utils/misc.py | 95 + scripts/lib/wic/utils/oe/__init__.py | 22 + scripts/lib/wic/utils/oe/misc.py | 250 +++ scripts/lib/wic/utils/partitionedfs.py | 360 ++++ scripts/lib/wic/utils/runner.py | 110 ++ scripts/lib/wic/utils/syslinux.py | 58 + scripts/lnr | 21 + scripts/multilib_header_wrapper.h | 55 + scripts/native-intercept/chown | 2 + scripts/oe-buildenv-internal | 121 ++ scripts/oe-find-native-sysroot | 81 + scripts/oe-git-proxy | 159 ++ scripts/oe-pkgdata-util | 525 ++++++ scripts/oe-publish-sdk | 155 ++ scripts/oe-selftest | 636 +++++++ scripts/oe-setup-builddir | 128 ++ scripts/oe-setup-rpmrepo | 97 + scripts/oe-trim-schemas | 49 + scripts/oepydevshell-internal.py | 92 + scripts/opkg-query-helper.py | 85 + scripts/postinst-intercepts/postinst_intercept | 56 + scripts/postinst-intercepts/update_font_cache | 7 + .../postinst-intercepts/update_gio_module_cache | 7 + scripts/postinst-intercepts/update_icon_cache | 13 + scripts/postinst-intercepts/update_pixbuf_cache | 11 + scripts/pybootchartgui/AUTHORS | 11 + scripts/pybootchartgui/COPYING | 340 ++++ scripts/pybootchartgui/MAINTAINERS | 3 + scripts/pybootchartgui/NEWS | 204 ++ scripts/pybootchartgui/README.pybootchart | 37 + scripts/pybootchartgui/pybootchartgui.py | 23 + scripts/pybootchartgui/pybootchartgui/__init__.py | 0 scripts/pybootchartgui/pybootchartgui/batch.py | 46 + scripts/pybootchartgui/pybootchartgui/draw.py | 894 +++++++++ scripts/pybootchartgui/pybootchartgui/gui.py | 350 ++++ scripts/pybootchartgui/pybootchartgui/main.py | 1 + scripts/pybootchartgui/pybootchartgui/main.py.in | 187 ++ scripts/pybootchartgui/pybootchartgui/parsing.py | 740 ++++++++ .../pybootchartgui/pybootchartgui/process_tree.py | 292 +++ scripts/pybootchartgui/pybootchartgui/samples.py | 151 ++ .../pybootchartgui/tests/parser_test.py | 105 ++ .../pybootchartgui/tests/process_tree_test.py | 92 + scripts/pythondeps | 250 +++ scripts/recipetool | 121 ++ scripts/relocate_sdk.py | 264 +++ scripts/rootfs_rpm-extract-postinst.awk | 11 + scripts/rpm2cpio.sh | 53 + scripts/runqemu | 546 ++++++ scripts/runqemu-addptable2image | 51 + scripts/runqemu-export-rootfs | 163 ++ scripts/runqemu-extract-sdk | 104 ++ scripts/runqemu-gen-tapdevs | 100 + scripts/runqemu-ifdown | 66 + scripts/runqemu-ifup | 121 ++ scripts/runqemu-internal | 717 +++++++ scripts/runqemu.README | 42 + scripts/send-error-report | 200 ++ scripts/send-pull-request | 179 ++ scripts/sstate-cache-management.sh | 469 +++++ scripts/sstate-diff-machines.sh | 172 ++ scripts/sstate-sysroot-cruft.sh | 185 ++ scripts/swabber-strace-attach | 31 + scripts/sysroot-relativelinks.py | 31 + scripts/test-dependencies.sh | 286 +++ scripts/test-reexec | 123 ++ scripts/test-remote-image | 361 ++++ scripts/tiny/dirsize.py | 93 + scripts/tiny/ksize.py | 165 ++ scripts/wic | 323 ++++ scripts/wipe-sysroot | 54 + scripts/yocto-bsp | 156 ++ scripts/yocto-kernel | 399 ++++ scripts/yocto-layer | 151 ++ setup.sh | 111 ++ 378 files changed, 39706 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .templateconf create mode 100644 LICENSE create mode 100644 README create mode 160000 bitbake create mode 100644 conf/.gitignore create mode 100644 conf/bblayers.conf.mlinux create mode 100644 conf/bblayers.conf.sample create mode 100644 conf/local.conf.sample create mode 100755 install-deps/install-centos-rhel-deps.sh create mode 100755 install-deps/install-debian-ubuntu-deps.sh create mode 100755 install-deps/install-fedora-deps.sh create mode 100755 install-deps/install-opensuse-deps.sh create mode 160000 layers/meta-java create mode 160000 layers/meta-mlinux create mode 160000 layers/meta-mlinux-atmel create mode 160000 layers/meta-mono create mode 160000 layers/meta-multitech create mode 160000 layers/meta-multitech-atmel create mode 160000 layers/meta-nodejs create mode 160000 layers/meta-openembedded create mode 160000 layers/meta-python2 create mode 160000 layers/meta-security create mode 160000 layers/meta-swupdate create mode 160000 layers/meta-virtualization create mode 160000 layers/openembedded-core create mode 100755 oe-init-build-env create mode 100755 oe-init-build-env-memres create mode 100644 patches/bitbake_checksumwarningsuppress.patch create mode 100644 patches/dont_use_network_on_unpack_of_tag.patch create mode 100644 patches/kernel-revision-ext-mod.patch create mode 100644 patches/meta-openembedded-github-https.patch create mode 100644 patches/no_net_check.patch create mode 100644 patches/openembedded-core-github-https.patch create mode 100644 patches/rename-mtr-conflicts-with-mtr-product.patch create mode 100644 scripts/README create mode 100755 scripts/bitbake-prserv-tool create mode 100755 scripts/bitbake-whatchanged create mode 100755 scripts/buildhistory-collect-srcrevs create mode 100755 scripts/buildhistory-diff create mode 100755 scripts/cleanup-workdir create mode 100755 scripts/combo-layer create mode 100755 scripts/combo-layer-hook-default.sh create mode 100644 scripts/combo-layer.conf.example create mode 100755 scripts/contrib/bb-perf/bb-matrix-plot.sh create mode 100755 scripts/contrib/bb-perf/bb-matrix.sh create mode 100755 scripts/contrib/bb-perf/buildstats.sh create mode 100755 scripts/contrib/bbvars.py create mode 100755 scripts/contrib/build-perf-test.sh create mode 100755 scripts/contrib/ddimage create mode 100755 scripts/contrib/devtool-stress.py create mode 100755 scripts/contrib/dialog-power-control create mode 100755 scripts/contrib/documentation-audit.sh create mode 100755 scripts/contrib/graph-tool create mode 100755 scripts/contrib/list-packageconfig-flags.py create mode 100755 scripts/contrib/mkefidisk.sh create mode 100755 scripts/contrib/python/generate-manifest-2.7.py create mode 100755 scripts/contrib/python/generate-manifest-3.5.py create mode 100755 scripts/contrib/serdevtry create mode 100755 scripts/contrib/test_build_time.sh create mode 100755 scripts/contrib/test_build_time_worker.sh create mode 100755 scripts/contrib/verify-homepage.py create mode 100755 scripts/cp-noerror create mode 100755 scripts/create-pull-request create mode 100755 scripts/crosstap create mode 100755 scripts/devtool create mode 100755 scripts/gen-lockedsig-cache create mode 100755 scripts/gen-site-config create mode 100755 scripts/git-hooks/post-checkout create mode 100755 scripts/git-hooks/post-merge create mode 100755 scripts/git-hooks/pre-commit create mode 100755 scripts/git-hooks/pre-push create mode 100644 scripts/lib/argparse_oe.py create mode 100644 scripts/lib/bsp/__init__.py create mode 100644 scripts/lib/bsp/engine.py create mode 100644 scripts/lib/bsp/help.py create mode 100644 scripts/lib/bsp/kernel.py create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/.gitignore create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-non_hardware.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/arm/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/common/COPYING.MIT create mode 100644 scripts/lib/bsp/substrate/target/arch/common/README create mode 100644 scripts/lib/bsp/substrate/target/arch/common/README.sources create mode 100644 scripts/lib/bsp/substrate/target/arch/common/binary/.gitignore create mode 100644 scripts/lib/bsp/substrate/target/arch/common/conf/layer.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor/machine/machconfig create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-bsp/formfactor/formfactor_0.0.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.bb create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/defconfig create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/common/recipes-kernel/linux/linux-yocto-custom/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/i386/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/COPYING.MIT create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/README create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/conf/layer.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/layer-questions.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example-bbappend/example-bbappend/example-bbappend-version/example.patch create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.bb create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/example.patch create mode 100644 scripts/lib/bsp/substrate/target/arch/layer/recipes-example/example/example-recipe-0.1/helloworld.c create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/.gitignore create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/.gitignore create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/mips64/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/.gitignore create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/powerpc/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown/machine/interfaces create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-core/init-ifupdown/init-ifupdown_1.0.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/qemu/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/.gitignore create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/conf/machine/machine.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config/machine/xorg.conf create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-preempt-rt.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-standard.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-tiny.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-config.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-features.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine-user-patches.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.cfg create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/files/machine.scc create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/kernel-list.noinstall create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-dev.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto-tiny_4.4.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.1.bbappend create mode 100644 scripts/lib/bsp/substrate/target/arch/x86_64/recipes-kernel/linux/linux-yocto_4.4.bbappend create mode 100644 scripts/lib/bsp/tags.py create mode 100644 scripts/lib/devtool/__init__.py create mode 100644 scripts/lib/devtool/build.py create mode 100644 scripts/lib/devtool/build_image.py create mode 100644 scripts/lib/devtool/build_sdk.py create mode 100644 scripts/lib/devtool/deploy.py create mode 100644 scripts/lib/devtool/package.py create mode 100644 scripts/lib/devtool/runqemu.py create mode 100644 scripts/lib/devtool/sdk.py create mode 100644 scripts/lib/devtool/search.py create mode 100644 scripts/lib/devtool/standard.py create mode 100644 scripts/lib/devtool/upgrade.py create mode 100644 scripts/lib/devtool/utilcmds.py create mode 100644 scripts/lib/recipetool/__init__.py create mode 100644 scripts/lib/recipetool/append.py create mode 100644 scripts/lib/recipetool/create.py create mode 100644 scripts/lib/recipetool/create_buildsys.py create mode 100644 scripts/lib/recipetool/create_buildsys_python.py create mode 100644 scripts/lib/recipetool/create_kernel.py create mode 100644 scripts/lib/recipetool/create_kmod.py create mode 100644 scripts/lib/recipetool/create_npm.py create mode 100644 scripts/lib/recipetool/newappend.py create mode 100644 scripts/lib/recipetool/setvar.py create mode 100644 scripts/lib/scriptpath.py create mode 100644 scripts/lib/scriptutils.py create mode 100644 scripts/lib/wic/__init__.py create mode 100644 scripts/lib/wic/__version__.py create mode 100644 scripts/lib/wic/canned-wks/common.wks.inc create mode 100644 scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg create mode 100644 scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks create mode 100644 scripts/lib/wic/canned-wks/directdisk-gpt.wks create mode 100644 scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks create mode 100644 scripts/lib/wic/canned-wks/directdisk.wks create mode 100644 scripts/lib/wic/canned-wks/mkefidisk.wks create mode 100644 scripts/lib/wic/canned-wks/mkgummidisk.wks create mode 100644 scripts/lib/wic/canned-wks/mkhybridiso.wks create mode 100644 scripts/lib/wic/canned-wks/qemux86-directdisk.wks create mode 100644 scripts/lib/wic/canned-wks/sdimage-bootpart.wks create mode 100644 scripts/lib/wic/conf.py create mode 100644 scripts/lib/wic/config/wic.conf create mode 100644 scripts/lib/wic/creator.py create mode 100644 scripts/lib/wic/engine.py create mode 100644 scripts/lib/wic/help.py create mode 100644 scripts/lib/wic/imager/__init__.py create mode 100644 scripts/lib/wic/imager/baseimager.py create mode 100644 scripts/lib/wic/imager/direct.py create mode 100644 scripts/lib/wic/ksparser.py create mode 100644 scripts/lib/wic/msger.py create mode 100644 scripts/lib/wic/partition.py create mode 100644 scripts/lib/wic/plugin.py create mode 100644 scripts/lib/wic/pluginbase.py create mode 100644 scripts/lib/wic/plugins/imager/direct_plugin.py create mode 100644 scripts/lib/wic/plugins/source/bootimg-efi.py create mode 100644 scripts/lib/wic/plugins/source/bootimg-partition.py create mode 100644 scripts/lib/wic/plugins/source/bootimg-pcbios.py create mode 100644 scripts/lib/wic/plugins/source/fsimage.py create mode 100644 scripts/lib/wic/plugins/source/isoimage-isohybrid.py create mode 100644 scripts/lib/wic/plugins/source/rawcopy.py create mode 100644 scripts/lib/wic/plugins/source/rootfs.py create mode 100644 scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py create mode 100644 scripts/lib/wic/test create mode 100644 scripts/lib/wic/utils/__init__.py create mode 100644 scripts/lib/wic/utils/errors.py create mode 100644 scripts/lib/wic/utils/fs_related.py create mode 100644 scripts/lib/wic/utils/misc.py create mode 100644 scripts/lib/wic/utils/oe/__init__.py create mode 100644 scripts/lib/wic/utils/oe/misc.py create mode 100644 scripts/lib/wic/utils/partitionedfs.py create mode 100644 scripts/lib/wic/utils/runner.py create mode 100644 scripts/lib/wic/utils/syslinux.py create mode 100755 scripts/lnr create mode 100644 scripts/multilib_header_wrapper.h create mode 100755 scripts/native-intercept/chown create mode 100755 scripts/oe-buildenv-internal create mode 100755 scripts/oe-find-native-sysroot create mode 100755 scripts/oe-git-proxy create mode 100755 scripts/oe-pkgdata-util create mode 100755 scripts/oe-publish-sdk create mode 100755 scripts/oe-selftest create mode 100755 scripts/oe-setup-builddir create mode 100755 scripts/oe-setup-rpmrepo create mode 100755 scripts/oe-trim-schemas create mode 100755 scripts/oepydevshell-internal.py create mode 100755 scripts/opkg-query-helper.py create mode 100755 scripts/postinst-intercepts/postinst_intercept create mode 100644 scripts/postinst-intercepts/update_font_cache create mode 100644 scripts/postinst-intercepts/update_gio_module_cache create mode 100644 scripts/postinst-intercepts/update_icon_cache create mode 100644 scripts/postinst-intercepts/update_pixbuf_cache create mode 100644 scripts/pybootchartgui/AUTHORS create mode 100644 scripts/pybootchartgui/COPYING create mode 100644 scripts/pybootchartgui/MAINTAINERS create mode 100644 scripts/pybootchartgui/NEWS create mode 100644 scripts/pybootchartgui/README.pybootchart create mode 100755 scripts/pybootchartgui/pybootchartgui.py create mode 100644 scripts/pybootchartgui/pybootchartgui/__init__.py create mode 100644 scripts/pybootchartgui/pybootchartgui/batch.py create mode 100644 scripts/pybootchartgui/pybootchartgui/draw.py create mode 100644 scripts/pybootchartgui/pybootchartgui/gui.py create mode 120000 scripts/pybootchartgui/pybootchartgui/main.py create mode 100644 scripts/pybootchartgui/pybootchartgui/main.py.in create mode 100644 scripts/pybootchartgui/pybootchartgui/parsing.py create mode 100644 scripts/pybootchartgui/pybootchartgui/process_tree.py create mode 100644 scripts/pybootchartgui/pybootchartgui/samples.py create mode 100644 scripts/pybootchartgui/pybootchartgui/tests/parser_test.py create mode 100644 scripts/pybootchartgui/pybootchartgui/tests/process_tree_test.py create mode 100755 scripts/pythondeps create mode 100755 scripts/recipetool create mode 100755 scripts/relocate_sdk.py create mode 100644 scripts/rootfs_rpm-extract-postinst.awk create mode 100755 scripts/rpm2cpio.sh create mode 100755 scripts/runqemu create mode 100755 scripts/runqemu-addptable2image create mode 100755 scripts/runqemu-export-rootfs create mode 100755 scripts/runqemu-extract-sdk create mode 100755 scripts/runqemu-gen-tapdevs create mode 100755 scripts/runqemu-ifdown create mode 100755 scripts/runqemu-ifup create mode 100755 scripts/runqemu-internal create mode 100644 scripts/runqemu.README create mode 100755 scripts/send-error-report create mode 100755 scripts/send-pull-request create mode 100755 scripts/sstate-cache-management.sh create mode 100755 scripts/sstate-diff-machines.sh create mode 100755 scripts/sstate-sysroot-cruft.sh create mode 100755 scripts/swabber-strace-attach create mode 100755 scripts/sysroot-relativelinks.py create mode 100755 scripts/test-dependencies.sh create mode 100755 scripts/test-reexec create mode 100755 scripts/test-remote-image create mode 100755 scripts/tiny/dirsize.py create mode 100755 scripts/tiny/ksize.py create mode 100755 scripts/wic create mode 100755 scripts/wipe-sysroot create mode 100755 scripts/yocto-bsp create mode 100755 scripts/yocto-kernel create mode 100755 scripts/yocto-layer create mode 100755 setup.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c02cf7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +downloads +build +buildhistory +env-oe.sh +layers/user-layer +bitbake.lock +password.txt diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..734426c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,57 @@ +[submodule "bitbake"] + path = bitbake + url = git://git.openembedded.org/bitbake + branch = 1.46 +[submodule "layers/openembedded-core"] + path = layers/openembedded-core + url = git://git.openembedded.org/openembedded-core + branch = dunfell +[submodule "layers/meta-openembedded"] + path = layers/meta-openembedded + url = git://git.openembedded.org/meta-openembedded + branch = dunfell +[submodule "layers/meta-multitech"] + path = layers/meta-multitech + url = git@gitlab.multitech.net:mirrors/meta-multitech.git + branch = . +[submodule "layers/meta-multitech-atmel"] + path = layers/meta-multitech-atmel + url = git@gitlab.multitech.net:mirrors/meta-multitech-atmel.git + branch = . +[submodule "layers/meta-java"] + path = layers/meta-java + branch = dunfell + url = git://git.yoctoproject.org/meta-java +[submodule "layers/meta-nodejs"] + path = layers/meta-nodejs + url = https://github.com/imyller/meta-nodejs.git + branch = master +[submodule "layers/meta-python2"] + path = layers/meta-python2 + url = git://git.openembedded.org/meta-python2 + branch = dunfell +[submodule "layers/meta-mono"] + path = layers/meta-mono + url = git://git.yoctoproject.org/meta-mono + branch = dunfell +[submodule "layers/meta-mlinux"] + path = layers/meta-mlinux + url = git@gitlab.multitech.net:mirrors/meta-mlinux.git + branch = . +[submodule "layers/meta-mlinux-atmel"] + path = layers/meta-mlinux-atmel + url = git@gitlab.multitech.net:mirrors/meta-mlinux-atmel.git + branch = . +[submodule "layers/meta-virtualization"] + path = layers/meta-virtualization + url = git://git.yoctoproject.org/meta-virtualization + branch = dunfell +[submodule "layers/meta-swupdate"] + path = layers/meta-swupdate + url = https://github.com/sbabic/meta-swupdate + branch = dunfell +[submodule "layers/meta-security"] + path = layers/meta-security + url = https://git.yoctoproject.org/git/meta-security + branch = dunfell + diff --git a/.templateconf b/.templateconf new file mode 100644 index 0000000..2d51eee --- /dev/null +++ b/.templateconf @@ -0,0 +1,2 @@ +# Template settings +TEMPLATECONF=${TEMPLATECONF:-layers/meta-mlinux/contrib/} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7ed0be5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +MultiTech mLinux is a software build system and custom Linux distribution. +It is your responsibility to review each license of the contained products +to understand what is, and what is not, allowed by each such license. + +The OpenEmbedded-Core common-licenses directory 'layers/openembedded-core/meta/files/common-licenses/' +contains licenses represented within this distribution. To the extent a license +is not contained in the 'common-licenses' directory, it is referenced in the +software and remains your responsibility to review. + +Different components of OpenEmbedded-Core are under different licenses (a mix +of MIT and GPLv2). Please see: + +layers/openembedded-core/meta/COPYING.GPLv2 (GPLv2) +layers/openembedded-core/meta/COPYING.MIT (MIT) + +For licensing information for other layers, see the files included with +that particular layer in the layers subdirectory. + +All MultiTech-copyrighted software contained in mLinux is covered by +GPL (GNU General Public License) version 2, unless provided under a different license. + +All bitbake metadata is MIT licensed unless otherwise stated. Source code +included in tree for individual recipes is under the LICENSE stated in +the associated recipe (.bb file) unless otherwise stated. + +License information for any other files is either explicitly stated +or defaults to GPL version 2. diff --git a/README b/README new file mode 100644 index 0000000..21b13b1 --- /dev/null +++ b/README @@ -0,0 +1,121 @@ +This is the branch for mLinux >= 6.0.0 + +mLinux - MultiTech Linux for MultiConnect Conduit gateways +============================================================ + +mLinux is a distribution provided by MultiTech tailored for the +MultiConnect Conduit products (MTCDT models). + +mLinux utilizes OpenEmbedded-Core, Bitbake, which are part of the Yocto Project. + +For more information about mLinux, see http://www.multitech.net + +For more information about the Yocto Project, see http://www.yoctoproject.org +For more information about OpenEmbedded, see http://www.openembedded.org + +This branch is based on Yocto Thud. Atmel devices use kernel Linux-AT91 4.9. + +It is no longer safe to assume MTCDT for the MACHINE type. All builds +now require the external variable MACHINE to be set. + +Building MTCDTIP2 requires: + +MACHINE=mtcap + + +------------------------- + BUILDING +------------------------- + +All version between 5.1 and 5.3.0a inclusive have +cellular issues with modems becoming disabled +due to COPS=2 set more or less permanently +on rare occassions. + +Version 5.3.0a through 5.3.0c have issues with git +trying to access unreachable repositories. The +recommended version after 5.0.x is 5.3.0d. + +Note that mLinux should *not* be built as root and you will need full write +permissions to the build directory structure. It is recommended to build +the software inside your $HOME dir as a normal user. + +1. Build from tarball + + tar xzf mLinux-3.0.0.tar.gz + cd mlinux-3.0.0 + # install needed dependencies, see scripts in install-deps + # Also switch to the build directory. + # example: sudo ./install-deps/install-debian-ubuntu-deps.sh + source oe-init-build-env + + # set your default machine type in conf/local.conf + # MACHINE="mtcdt" + + # build an image + bitbake mlinux-base-image + +2. Build from git repository + + # clone repo to a dir name of your choice + git clone git://git.multitech.net/mlinux.git mlinux-3.0.0 + cd mlinux-3.0.0 + + # checkout desired branch or tag + git checkout 3.0.0 + + # install needed dependencies, see scripts in install-deps + # example: sudo ./install-deps/install-debian-ubuntu-deps.sh + + # initialize git submodules and setup dir structure + ./setup.sh + # setup.sh generates a random mtadm password, and places the + # password in build/conf/local.conf and password.txt + # + # + # The specified password is ignored for commissioning images. + # + # To specify a password use an environmental variable, + # MTADM_PASSWORD + MTADM_PASSWORD="3g_t1zX0" ./setup.sh + # + # To change the password, remove MTADM_PASSWORD and + # MTADM_PASSWORD_HASH from conf/local.conf and re-run: + MTADM_PASSWORD="Y5bG3m_2" ./setup.sh + + # setup environment and switch to build directory + source oe-init-build-env + + # set your default machine type in conf/local.conf + # MACHINE="mtcdt" + + # build an image (current working directory is + # build) + bitbake mlinux-base-image + +------------------------- + IMAGES +------------------------- + +mlinux-base-image: Small image with essential packages -- good starting point for a custom image + +mlinux-commissioning-image: Large image loaded on units by default -- contains full language support (perl, python) +mlinux-mtcap-commissioning-image: MTCAP image + +------------------------- + PRE-BUILT IMAGES +------------------------- + +http://multitech.net/mlinux/images + +------------------------- + MORE INFO +------------------------- + +See the MultiTech Developer website (http://www.multitech.net) for full +installation and build instructions. In particular, see the "Introduction" and +"Getting Started" pages under the mLinux section. + +You can also join the available developer forums to post queries and receive +updates from MultiTech developers. + diff --git a/bitbake b/bitbake new file mode 160000 index 0000000..0e0af15 --- /dev/null +++ b/bitbake @@ -0,0 +1 @@ +Subproject commit 0e0af15b84e07e6763300dcd092b980086b9b9c4 diff --git a/conf/.gitignore b/conf/.gitignore new file mode 100644 index 0000000..143ff36 --- /dev/null +++ b/conf/.gitignore @@ -0,0 +1,2 @@ +local.conf +sanity_info diff --git a/conf/bblayers.conf.mlinux b/conf/bblayers.conf.mlinux new file mode 100644 index 0000000..fb68424 --- /dev/null +++ b/conf/bblayers.conf.mlinux @@ -0,0 +1,50 @@ +LCONF_VERSION = "6" +TOPDIR := "${@os.path.dirname(os.path.dirname(d.getVar('FILE', True)))}" +BBPATH = "__OEROOT__" +BBFILES ?= "" +MACHINE ?= "mtcdt" + +# MACHINE SPECIFIC LAYER CONFIGURATIONS +MTBSP_AT91_LIST = "['mtcdt', 'mtcap', 'mtr', 'mtrv1', 'mtre']" +MTBSP_NXP_LIST = "['mtcap3']" + +MTBSP = "${@'mtbsp-at91' if '${MACHINE}' in ${MTBSP_AT91_LIST} else \ + 'mtbsp-nxp' if '${MACHINE}' in ${MTBSP_NXP_LIST} else ''}" + +OVERRIDES .= ":${MTBSP}" + +# Atmel AT91 +BSPPATH_mtbsp-at91 = "" +BSPLAYER_mtbsp-at91= "__OEROOT__/layers/meta-multitech-atmel \ + __OEROOT__/layers/meta-mlinux-atmel" + +BSPPATH_mtbsp-nxp = "__OEROOT__/layers/meta-multitech-nxp/meta-freescale" +BSPLAYER_mtbsp-nxp= "__OEROOT__/layers/meta-multitech-nxp \ + __OEROOT__/layers/meta-multitech-nxp/meta-freescale \ + __OEROOT__/layers/meta-mlinux-nxp \ + __OEROOT__/layers/meta-security/meta-tpm" + +# Append the BSP configuration +BBPATH =. "${@'${BSPPATH_${MTBSP}}:' if '${BSPPATH_${MTBSP}}' else ''}" + +# No virtaulization and swupdate on at91 (mask bbappends and images) +BBMASK_mtbsp-at91 += "wl18xx-fw tpm2-tss-engine" + +BBLAYERS = " \ + __OEROOT__/layers/user-layer \ + ${BSPLAYER_${MTBSP}} \ + __OEROOT__/layers/meta-mlinux \ + __OEROOT__/layers/meta-multitech \ + __OEROOT__/layers/meta-mono \ + __OEROOT__/layers/meta-java \ + __OEROOT__/layers/meta-python2 \ + __OEROOT__/layers/meta-swupdate \ + __OEROOT__/layers/meta-nodejs \ + __OEROOT__/layers/meta-openembedded/meta-oe \ + __OEROOT__/layers/meta-openembedded/meta-python \ + __OEROOT__/layers/meta-openembedded/meta-networking \ + __OEROOT__/layers/meta-openembedded/meta-webserver \ + __OEROOT__/layers/meta-openembedded/meta-multimedia \ + __OEROOT__/layers/meta-openembedded/meta-filesystems \ + __OEROOT__/layers/openembedded-core/meta \ + " diff --git a/conf/bblayers.conf.sample b/conf/bblayers.conf.sample new file mode 100644 index 0000000..a8e65be --- /dev/null +++ b/conf/bblayers.conf.sample @@ -0,0 +1,22 @@ +LCONF_VERSION = "6" +TOPDIR := "${@os.path.dirname(os.path.dirname(d.getVar('FILE', True)))}" +BBPATH = "${TOPDIR}" +BBFILES ?= "" +BBLAYERS = " \ + ${TOPDIR}/layers/user-layer \ + ${TOPDIR}/layers/meta-mlinux \ + ${TOPDIR}/layers/meta-mlinux-atmel \ + ${TOPDIR}/layers/meta-multitech \ + ${TOPDIR}/layers/meta-multitech-atmel \ + ${TOPDIR}/layers/meta-mono \ + ${TOPDIR}/layers/meta-nodejs \ + ${TOPDIR}/layers/meta-java \ + ${TOPDIR}/layers/meta-openembedded/meta-oe \ + ${TOPDIR}/layers/meta-openembedded/meta-perl \ + ${TOPDIR}/layers/meta-openembedded/meta-networking \ + ${TOPDIR}/layers/meta-openembedded/meta-webserver \ + ${TOPDIR}/layers/meta-openembedded/meta-multimedia \ + ${TOPDIR}/layers/meta-openembedded/meta-filesystems \ + ${TOPDIR}/layers/meta-openembedded/toolchain-layer \ + ${TOPDIR}/layers/openembedded-core/meta \ + " diff --git a/conf/local.conf.sample b/conf/local.conf.sample new file mode 100644 index 0000000..f798315 --- /dev/null +++ b/conf/local.conf.sample @@ -0,0 +1,51 @@ +# CONF_VERSION is increased each time build/conf/ changes incompatibly +CONF_VERSION = "2" + +# Use MultiTech mLinux distribution +DISTRO = "mlinux" +MACHINE ?= "mtcdt" + +# Where to store downloaded sources +DL_DIR = "${TOPDIR}/downloads" + +# Where to save shared state +SSTATE_DIR = "${TOPDIR}/build/sstate-cache" +# bitbake cache location +PERSISTENT_DIR = "${TOPDIR}/build/cache" +# build output +TMPDIR = "${TOPDIR}/build/tmp" + +# Which files do we want to parse: +BBFILES ?= "${TOPDIR}/layers/openembedded-core/meta/recipes-*/*/*.bb" + +# Go through the Firewall +#HTTP_PROXY = "http://:/" + +# Uncomment this to remove unpacked source and intermediate work +# after successfully building a package. +# Saves a LOT of disk space, but leaving work around is useful for debugging. +INHERIT += "rm_work" + +# skip parsing of masked files +BBMASK = "" + +# Make use of SMP: +# PARALLEL_MAKE specifies how many concurrent compiler threads are spawned per bitbake process +# BB_NUMBER_THREADS specifies how many concurrent bitbake tasks will be run +BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()*2}" +PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()*2}" + +# Don't generate the mirror tarball for SCM repos, the snapshot is enough +BB_GENERATE_MIRROR_TARBALLS = "0" + +# Disable build time patch resolution. This would lauch a devshell +# and wait for manual intervention. We disable it. +PATCHRESOLVE = "noop" + +# enable local PR server +PRSERV_HOST = "localhost:0" + +# enable buildhistory for images +INHERIT += "buildhistory" +BUILDHISTORY_COMMIT = "0" +BUILDHISTORY_FEATURES = "image" diff --git a/install-deps/install-centos-rhel-deps.sh b/install-deps/install-centos-rhel-deps.sh new file mode 100755 index 0000000..0f0e6d9 --- /dev/null +++ b/install-deps/install-centos-rhel-deps.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +yum install gawk make wget tar bzip2 gzip python unzip perl patch \ +diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath SDL-devel xterm diff --git a/install-deps/install-debian-ubuntu-deps.sh b/install-deps/install-debian-ubuntu-deps.sh new file mode 100755 index 0000000..6088db4 --- /dev/null +++ b/install-deps/install-debian-ubuntu-deps.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +apt-get install gawk wget git-core diffstat unzip texinfo gcc-multilib \ +build-essential chrpath libsdl1.2-dev xterm diff --git a/install-deps/install-fedora-deps.sh b/install-deps/install-fedora-deps.sh new file mode 100755 index 0000000..f1c34d4 --- /dev/null +++ b/install-deps/install-fedora-deps.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +yum install gawk make wget tar bzip2 gzip python unzip perl patch \ +diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \ +ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue SDL-devel xterm diff --git a/install-deps/install-opensuse-deps.sh b/install-deps/install-opensuse-deps.sh new file mode 100755 index 0000000..7f3c193 --- /dev/null +++ b/install-deps/install-opensuse-deps.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +zypper install python gcc gcc-c++ git chrpath make wget python-xml \ +diffstat texinfo python-curses patch libSDL-devel xterm diff --git a/layers/meta-java b/layers/meta-java new file mode 160000 index 0000000..6e84638 --- /dev/null +++ b/layers/meta-java @@ -0,0 +1 @@ +Subproject commit 6e84638d77ac921aac46649095bca5ddbde94d2a diff --git a/layers/meta-mlinux b/layers/meta-mlinux new file mode 160000 index 0000000..6c88861 --- /dev/null +++ b/layers/meta-mlinux @@ -0,0 +1 @@ +Subproject commit 6c8886126e74b4e3dbd33bd075ec5f446ae95504 diff --git a/layers/meta-mlinux-atmel b/layers/meta-mlinux-atmel new file mode 160000 index 0000000..8c79c93 --- /dev/null +++ b/layers/meta-mlinux-atmel @@ -0,0 +1 @@ +Subproject commit 8c79c938c68f28385b0b47146ac95277f7429744 diff --git a/layers/meta-mono b/layers/meta-mono new file mode 160000 index 0000000..0c5afc7 --- /dev/null +++ b/layers/meta-mono @@ -0,0 +1 @@ +Subproject commit 0c5afc77a4ec61326996f006acac6f025f56a8bc diff --git a/layers/meta-multitech b/layers/meta-multitech new file mode 160000 index 0000000..cec6945 --- /dev/null +++ b/layers/meta-multitech @@ -0,0 +1 @@ +Subproject commit cec6945d9087a9b66c5107c66f1029eca3f62364 diff --git a/layers/meta-multitech-atmel b/layers/meta-multitech-atmel new file mode 160000 index 0000000..58a6156 --- /dev/null +++ b/layers/meta-multitech-atmel @@ -0,0 +1 @@ +Subproject commit 58a6156b61364d8effe0876afc4ab9e0ce62c39b diff --git a/layers/meta-nodejs b/layers/meta-nodejs new file mode 160000 index 0000000..eec531e --- /dev/null +++ b/layers/meta-nodejs @@ -0,0 +1 @@ +Subproject commit eec531e97a17bfd406f3bf76dee4057dcf5286a4 diff --git a/layers/meta-openembedded b/layers/meta-openembedded new file mode 160000 index 0000000..85f8047 --- /dev/null +++ b/layers/meta-openembedded @@ -0,0 +1 @@ +Subproject commit 85f8047c71047d93cf79a954142157f85079968d diff --git a/layers/meta-python2 b/layers/meta-python2 new file mode 160000 index 0000000..b901080 --- /dev/null +++ b/layers/meta-python2 @@ -0,0 +1 @@ +Subproject commit b901080cf57d9a7f5476ab4d96e56c30db8170a8 diff --git a/layers/meta-security b/layers/meta-security new file mode 160000 index 0000000..c74cc97 --- /dev/null +++ b/layers/meta-security @@ -0,0 +1 @@ +Subproject commit c74cc97641fd93e0e7a4383255e9a0ab3deaf9d7 diff --git a/layers/meta-swupdate b/layers/meta-swupdate new file mode 160000 index 0000000..be283cb --- /dev/null +++ b/layers/meta-swupdate @@ -0,0 +1 @@ +Subproject commit be283cb6be560dee5e16451ad5f31ec9db2b1dbd diff --git a/layers/meta-virtualization b/layers/meta-virtualization new file mode 160000 index 0000000..c5f61e5 --- /dev/null +++ b/layers/meta-virtualization @@ -0,0 +1 @@ +Subproject commit c5f61e547b90aa8058cf816f00902afed9c96f72 diff --git a/layers/openembedded-core b/layers/openembedded-core new file mode 160000 index 0000000..9ae339a --- /dev/null +++ b/layers/openembedded-core @@ -0,0 +1 @@ +Subproject commit 9ae339ace9274be71bfd3b5e5da64dceac9fa963 diff --git a/oe-init-build-env b/oe-init-build-env new file mode 100755 index 0000000..525c23f --- /dev/null +++ b/oe-init-build-env @@ -0,0 +1,77 @@ +#!/bin/sh + +# OE Build Environment Setup Script +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# +# Normally this is called as '. ./oe-init-build-env' +# + +if [ -n "$BASH_SOURCE" ]; then + THIS_SCRIPT=$BASH_SOURCE +elif [ -n "$ZSH_NAME" ]; then + THIS_SCRIPT=$0 +else + THIS_SCRIPT="$(pwd)/oe-init-build-env" +fi +if [ -n "$BBSERVER" ]; then + unset BBSERVER +fi + +if [ -z "$ZSH_NAME" ] && [ "$0" = "$THIS_SCRIPT" ]; then + echo "Error: This script needs to be sourced. Please run as '. $THIS_SCRIPT'" + exit 1 +fi + +if [ -z "$OEROOT" ]; then + OEROOT=$(dirname "$THIS_SCRIPT") + OEROOT=$(readlink -f "$OEROOT") +fi +unset THIS_SCRIPT + +export OEROOT +. $OEROOT/scripts/oe-buildenv-internal && + TEMPLATECONF="$TEMPLATECONF" $OEROOT/scripts/oe-setup-builddir || { + unset OEROOT + return 1 +} + +# Shutdown any bitbake server if the BBSERVER variable is not set +if [ -z "$BBSERVER" ] && [ -f $OEROOT/bitbake.lock ]; then + grep ":" $OEROOT/bitbake.lock > /dev/null && BBSERVER=$(cat $OEROOT/bitbake.lock) bitbake --status-only + if [ $? = 0 ]; then + echo "Shutting down bitbake memory resident server with bitbake -m" + BBSERVER=$(cat $OEROOT/bitbake.lock) bitbake -m + fi +fi + +export BBPATH="$OEROOT" + +# define if you wish to create soft links to the image in the tftpboot dir +#export TFTPBOOT_DIR=/tftpboot + +declare -a WlAdditions=("OEROOT") + +for entry in ${WlAdditions[@]}; do + if [ "$(echo "$BB_ENV_EXTRAWHITE" | sed -re 's/[ \t]+/\n/g' | egrep ''^$entry'')" != "$entry" ] ; then + echo $BB_ENV_EXTRAWHITE needs correction + BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE $entry" + export BB_ENV_EXTRAWHITE + fi +done + diff --git a/oe-init-build-env-memres b/oe-init-build-env-memres new file mode 100755 index 0000000..f3db2b6 --- /dev/null +++ b/oe-init-build-env-memres @@ -0,0 +1,94 @@ +#!/bin/sh + +# OE Build Environment Setup Script +# +# Copyright (C) 2006-2011 Linux Foundation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# +# Normally this is called as '. ./oe-init-build-env-memres ' +# +# This works in most shells (not dash), but not all of them pass the arguments +# when being sourced. To workaround the shell limitation use "set " +# prior to sourcing this script. +# +if [ -z "$1" ]; then + echo "No port specified, using dynamically selected port" + port=-1 +else + port=$1 + shift +fi + +if [ -n "$BASH_SOURCE" ]; then + THIS_SCRIPT=$BASH_SOURCE +elif [ -n "$ZSH_NAME" ]; then + THIS_SCRIPT=$0 +else + THIS_SCRIPT="$(pwd)/oe-init-build-env" +fi +if [ -n "$BBSERVER" ]; then + unset BBSERVER +fi + +if [ -z "$ZSH_NAME" ] && [ "$0" = "$THIS_SCRIPT" ]; then + echo "Error: This script needs to be sourced. Please run as '. $THIS_SCRIPT'" + exit 1 +fi + +if [ -z "$OEROOT" ]; then + OEROOT=$(dirname "$THIS_SCRIPT") + OEROOT=$(readlink -f "$OEROOT") +fi +unset THIS_SCRIPT + +export OEROOT +. $OEROOT/scripts/oe-buildenv-internal && + TEMPLATECONF="$TEMPLATECONF" $OEROOT/scripts/oe-setup-builddir || { + unset OEROOT + return 1 +} + +res=1 +if [ -e $OEROOT/bitbake.lock ] && grep : $OEROOT/bitbake.lock > /dev/null; then + BBSERVER=$(cat $OEROOT/bitbake.lock) bitbake --status-only + res=$? +fi + +if [ $res != 0 ]; then + bitbake --server-only -t xmlrpc -B localhost:$port +fi + +if [ $port = -1 ]; then + export BBSERVER=localhost:-1 + echo "Bitbake server started on demand as needed, use bitbake -m to shut it down" +else + export BBSERVER=$(cat $OEROOT/bitbake.lock) + + if [ $res = 0 ]; then + echo "Using existing bitbake server at: $BBSERVER, use bitbake -m to shut it down" + else + echo "Bitbake server started at: $BBSERVER, use bitbake -m to shut it down" + fi +fi +unset port res + +export BBPATH="$OEROOT" + +# define if you wish to create soft links to the image in the tftpboot dir +#export TFTPBOOT_DIR=/tftpboot + +unset OEROOT diff --git a/patches/bitbake_checksumwarningsuppress.patch b/patches/bitbake_checksumwarningsuppress.patch new file mode 100644 index 0000000..438a6f2 --- /dev/null +++ b/patches/bitbake_checksumwarningsuppress.patch @@ -0,0 +1,18 @@ +diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py +index dc99914c..f622fc7b 100644 +--- a/bitbake/lib/bb/fetch2/__init__.py ++++ b/bitbake/lib/bb/fetch2/__init__.py +@@ -1188,11 +1188,12 @@ def get_checksum_file_list(d): + + if ud and isinstance(ud.method, local.Local): + paths = ud.method.localpaths(ud, d) ++ strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0" + for f in paths: + pth = ud.decodedurl + if '*' in pth: + f = os.path.join(os.path.abspath(f), pth) +- if f.startswith(dl_dir): ++ if f.startswith(dl_dir) and (strict == "1"): + # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else + if os.path.exists(f): + bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f))) diff --git a/patches/dont_use_network_on_unpack_of_tag.patch b/patches/dont_use_network_on_unpack_of_tag.patch new file mode 100644 index 0000000..6946535 --- /dev/null +++ b/patches/dont_use_network_on_unpack_of_tag.patch @@ -0,0 +1,95 @@ +diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py +index 112b833f..8c6d04d9 100644 +--- a/bitbake/lib/bb/fetch2/git.py ++++ b/bitbake/lib/bb/fetch2/git.py +@@ -129,6 +129,19 @@ class Git(FetchMethod): + def supports_checksum(self, urldata): + return False + ++ # git HASH verification code (is the REV a hash or not) ++ def bad_git_hash(self, revision): ++ if not revision or len(revision) != 40 or (False in [c in "abcdef0123456789" for c in revision]): ++ if not revision: ++ bb.note("bad_git_hash: SRCREV is not a defined string") ++ elif len(revision) != 40: ++ bb.debug(2,"bad_git_hash: revision len is not 40: " + str(len(revision))) ++ bb.debug(2,"bad_git_hash: revision dump: %s" % ":".join("{:02x}".format(ord(c)) for c in revision)) ++ else: ++ bb.debug(2,"bad_git_hash: revision bad char: %s" % ":".join("{:02x}".format(ord(c)) for c in revision)) ++ return True ++ return False ++ + def urldata_init(self, ud, d): + """ + init git specific variable within url data +@@ -229,16 +242,52 @@ class Git(FetchMethod): + + ud.setup_revisions(d) + ++ # EARLY_GITSRCNAME start ++ gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) ++ if gitsrcname.startswith('.'): ++ gitsrcname = gitsrcname[1:] ++ # EARLY_GITSRCNAME end ++ ++ # EARLY_GITDIR start ++ dl_dir = d.getVar("DL_DIR") ++ gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") ++ ud.clonedir = os.path.join(gitdir, gitsrcname) ++ ud.localfile = ud.clonedir ++ # EARLY_GITDIR end ++ + for name in ud.names: + # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one +- if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): ++ # First try the local repository ++ if self.bad_git_hash(ud.revisions[name]): + if ud.revisions[name]: + ud.unresolvedrev[name] = ud.revisions[name] +- ud.revisions[name] = self.latest_revision(ud, d, name) +- +- gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) +- if gitsrcname.startswith('.'): +- gitsrcname = gitsrcname[1:] ++ if len(ud.revisions[name]) != 0: ++ # Assume this is a tag, and retrieve hash from local source, ++ # if possible. Deviation from openembedded, which goes to ++ # remote source for tags. ++ cmd = "%s rev-parse %s^{}" % (ud.basecmd, ud.revisions[name]) ++ savedrev = ud.revisions[name] ++ if os.path.isdir(ud.localfile): ++ try: ++ # Validate local git repository ++ cmd = "%s --bare fsck --no-full" % (ud.basecmd) ++ runfetchcmd(cmd,d,workdir=ud.localfile) ++ # Valid repository, so see if we can get the hash ++ cmd = "%s rev-parse %s^{}" % (ud.basecmd, ud.revisions[name]) ++ ud.revisions[name] = runfetchcmd(cmd,d,workdir=ud.localfile).rstrip() ++ except bb.fetch.FetchError: ++ bb.note("do_fetch: removing invalid git file system: " + ud.localfile) ++ cmd = "rm -rf " + ud.localfile + ";rm -f " + ud.localfile + ".done" ++ ud.revisions[name] = savedrev ++ ++ if self.bad_git_hash(ud.revisions[name]): ++ ud.revisions[name] = self.latest_revision(ud, d, name) ++ ++ bb.debug(2,"urldata_init: ud.revisions[name] " + ud.revisions[name]) ++ bb.debug(2,"urldata_init: ud.host " + ud.host) ++ bb.debug(2,"urldata_init: ud.path " + ud.path) ++ ++ # gitsrcname code purposely moved to EARLY_GITSRCNAME + + # for rebaseable git repo, it is necessary to keep mirror tar ball + # per revision, so that even the revision disappears from the +@@ -248,10 +297,7 @@ class Git(FetchMethod): + for name in ud.names: + gitsrcname = gitsrcname + '_' + ud.revisions[name] + +- dl_dir = d.getVar("DL_DIR") +- gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") +- ud.clonedir = os.path.join(gitdir, gitsrcname) +- ud.localfile = ud.clonedir ++ # gitdir/clonedir/localfile purposely moved to EARLY_GITDIR + + mirrortarball = 'git2_%s.tar.gz' % gitsrcname + ud.fullmirror = os.path.join(dl_dir, mirrortarball) diff --git a/patches/kernel-revision-ext-mod.patch b/patches/kernel-revision-ext-mod.patch new file mode 100644 index 0000000..c7da73c --- /dev/null +++ b/patches/kernel-revision-ext-mod.patch @@ -0,0 +1,25 @@ +diff -Naru orig/layers/openembedded-core/meta/classes/kernel-module-split.bbclass new/layers/openembedded-core/meta/classes/kernel-module-split.bbclass +--- orig/layers/openembedded-core/meta/classes/kernel-module-split.bbclass 2020-07-13 17:38:32.148762374 -0500 ++++ new/layers/openembedded-core/meta/classes/kernel-module-split.bbclass 2020-07-13 17:36:27.740764540 -0500 +@@ -33,7 +33,7 @@ + KERNEL_MODULES_META_PACKAGE ?= "${@ d.getVar("KERNEL_PACKAGE_NAME") or "kernel" }-modules" + + KERNEL_MODULE_PACKAGE_PREFIX ?= "" +-KERNEL_MODULE_PACKAGE_SUFFIX ?= "-${KERNEL_VERSION}" ++KERNEL_MODULE_PACKAGE_SUFFIX ?= "-${KERNEL_VERSION}${KERNEL_REVISION}" + KERNEL_MODULE_PROVIDE_VIRTUAL ?= "1" + + python split_kernel_module_packages () { +diff -Naru orig/layers/openembedded-core/meta/classes/module-base.bbclass new/layers/openembedded-core/meta/classes/module-base.bbclass +--- orig/layers/openembedded-core/meta/classes/module-base.bbclass 2020-07-13 17:38:20.708762573 -0500 ++++ new/layers/openembedded-core/meta/classes/module-base.bbclass 2020-07-13 17:36:52.756764104 -0500 +@@ -16,6 +16,9 @@ + export KERNEL_VERSION = "${@oe.utils.read_file('${STAGING_KERNEL_BUILDDIR}/kernel-abiversion')}" + KERNEL_OBJECT_SUFFIX = ".ko" + ++# Kernel revision ++export KERNEL_REVISION = "${@oe.utils.read_file('${STAGING_KERNEL_BUILDDIR}/mlinux_pr')}" ++ + # kernel modules are generally machine specific + PACKAGE_ARCH = "${MACHINE_ARCH}" + diff --git a/patches/meta-openembedded-github-https.patch b/patches/meta-openembedded-github-https.patch new file mode 100644 index 0000000..9a49b85 --- /dev/null +++ b/patches/meta-openembedded-github-https.patch @@ -0,0 +1,171 @@ +github no longer allows clear git protocol, so switch to https +=============================================================== +diff --git a/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb b/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb +index 23583650b..ed003ee7b 100644 +--- a/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb ++++ b/layers/meta-openembedded/meta-filesystems/recipes-utils/ufs-utils/ufs-utils_git.bb +@@ -8,7 +8,7 @@ BRANCH ?= "dev" + + SRCREV = "a3cf93b66f4606a46354cf884d24aa966661f848" + +-SRC_URI = "git://github.com/westerndigitalcorporation/ufs-utils.git;protocol=git;branch=${BRANCH} \ ++SRC_URI = "git://github.com/westerndigitalcorporation/ufs-utils.git;protocol=https;branch=${BRANCH} \ + file://0001-Replace-u_intXX_t-with-kernel-typedefs.patch \ + " + +diff --git a/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb b/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb +index d7911681c..c499119c6 100644 +--- a/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb ++++ b/layers/meta-openembedded/meta-multimedia/recipes-multimedia/fdk-aac/fdk-aac_2.0.1.bb +@@ -11,7 +11,7 @@ LICENSE = "Fraunhofer_FDK_AAC_Codec_Library_for_Android" + LICENSE_FLAGS = "commercial" + LIC_FILES_CHKSUM = "file://NOTICE;md5=5985e1e12f4afa710d64ed7bfd291875" + +-SRC_URI = "git://github.com/mstorsjo/fdk-aac.git;protocol=git;branch=master" ++SRC_URI = "git://github.com/mstorsjo/fdk-aac.git;protocol=https;branch=master" + SRCREV = "d387d3b6ed79ff9a82c60440bdd86e6e5e324bec" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb b/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb +index 6dd15ad9f..46b50c26b 100644 +--- a/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb ++++ b/layers/meta-openembedded/meta-networking/recipes-protocols/babeld/babeld_1.9.1.bb +@@ -12,7 +12,7 @@ SECTION = "net" + LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://LICENCE;md5=411a48ac3c2e9e0911b8dd9aed26f754" + +-SRC_URI = "git://github.com/jech/babeld.git;protocol=git" ++SRC_URI = "git://github.com/jech/babeld.git;protocol=https" + SRCREV = "0835d5d894ea016ab7b81562466cade2c51a12d4" + + UPSTREAM_CHECK_GITTAGREGEX = "babeld-(?P\d+(\.\d+)+)" +diff --git a/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb b/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb +index 0b63f79ac..d8a1f6140 100644 +--- a/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb ++++ b/layers/meta-openembedded/meta-networking/recipes-support/smcroute/smcroute_2.4.4.bb +@@ -6,7 +6,7 @@ LICENSE = "GPLv2+" + LIC_FILES_CHKSUM = "file://COPYING;md5=751419260aa954499f7abaabaa882bbe" + + SRCREV = "a8e5847e5f7e411be424f9b52a6cdf9d2ed4aeb5" +-SRC_URI = "git://github.com/troglobit/smcroute.git;branch=master;protocol=git" ++SRC_URI = "git://github.com/troglobit/smcroute.git;branch=master;protocol=https" + + S = "${WORKDIR}/git" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb b/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb +index c8e81a412..f0e928d0d 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-core/sdbus-c++/sdbus-c++-libsystemd_243.bb +@@ -12,7 +12,7 @@ DEPENDS += "gperf-native gettext-native util-linux libcap" + + SRCREV = "efb536d0cbe2e58f80e501d19999928c75e08f6a" + SRCBRANCH = "v243-stable" +-SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=git;branch=${SRCBRANCH}" ++SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=https;branch=${SRCBRANCH}" + + SRC_URI += "file://static-libsystemd-pkgconfig.patch" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb b/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb +index 29f8de8d2..b2f38728d 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-extended/hiredis/hiredis_0.14.0.bb +@@ -6,7 +6,7 @@ DEPENDS = "redis" + + LIC_FILES_CHKSUM = "file://COPYING;md5=d84d659a35c666d23233e54503aaea51" + SRCREV = "685030652cd98c5414ce554ff5b356dfe8437870" +-SRC_URI = "git://github.com/redis/hiredis;protocol=git \ ++SRC_URI = "git://github.com/redis/hiredis;protocol=https \ + file://0001-Makefile-remove-hardcoding-of-CC.patch" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb b/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb +index 519368817..5015ac53b 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-extended/socketcan/can-utils_git.bb +@@ -4,7 +4,7 @@ LIC_FILES_CHKSUM = "file://include/linux/can.h;endline=44;md5=a9e1169c6c9a114a61 + + DEPENDS = "libsocketcan" + +-SRC_URI = "git://github.com/linux-can/${BPN}.git;protocol=git" ++SRC_URI = "git://github.com/linux-can/${BPN}.git;protocol=https" + + SRCREV = "da65fdfe0d1986625ee00af0b56ae17ec132e700" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb +index 3800d147f..37631acae 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/gpm/gpm_git.bb +@@ -13,7 +13,7 @@ SRCREV = "1fd19417b8a4dd9945347e98dfa97e4cfd798d77" + + DEPENDS = "ncurses bison-native" + +-SRC_URI = "git://github.com/telmich/gpm;protocol=git \ ++SRC_URI = "git://github.com/telmich/gpm;protocol=https \ + file://init \ + file://gpm.service.in \ + file://0001-Use-sigemptyset-API-instead-of-__sigemptyset.patch \ +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb +index 3a437659e..0e3e5ff73 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/funyahoo-plusplus_git.bb +@@ -7,7 +7,7 @@ DEPENDS = "pidgin json-glib glib-2.0" + + inherit pkgconfig + +-SRC_URI = "git://github.com/EionRobb/funyahoo-plusplus;branch=master;protocol=git" ++SRC_URI = "git://github.com/EionRobb/funyahoo-plusplus;branch=master;protocol=https" + SRCREV = "fbbd9c591100aa00a0487738ec7b6acd3d924b3f" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb +index 092e6059b..854920d2e 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/pidgin/purple-skypeweb_git.bb +@@ -7,7 +7,7 @@ DEPENDS = "pidgin json-glib glib-2.0 zlib" + + inherit pkgconfig + +-SRC_URI = "git://github.com/EionRobb/skype4pidgin;branch=master;protocol=git" ++SRC_URI = "git://github.com/EionRobb/skype4pidgin;branch=master;protocol=https" + SRCREV = "14f1b69b6292bbdc98cca484b050ec8359394c4e" + + S = "${WORKDIR}/git" +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb +index 33f5dccca..6fe8aa76f 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/rsnapshot/rsnapshot_git.bb +@@ -25,7 +25,7 @@ RDEPENDS_${PN} = "rsync \ + SRCREV = "a9e29850fc33c503c289e245c7bad350eed746d9" + PV = "1.4.3+git${SRCPV}" + +-SRC_URI = "git://github.com/DrHyde/${BPN};branch=master;protocol=git \ ++SRC_URI = "git://github.com/DrHyde/${BPN};branch=master;protocol=https \ + file://configure-fix-cmd_rsync.patch \ + " + +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb b/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb +index 39629cce0..9294d1a70 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/spdlog/spdlog_1.5.0.bb +@@ -4,7 +4,7 @@ LICENSE = "MIT" + LIC_FILES_CHKSUM = "file://${COREBASE}/meta/files/common-licenses/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + + SRCREV = "cf6f1dd01e660d5865d68bf5fa78f6376b89470a" +-SRC_URI = "git://github.com/gabime/spdlog.git;protocol=git;branch=v1.x;" ++SRC_URI = "git://github.com/gabime/spdlog.git;protocol=https;branch=v1.x;" + + S = "${WORKDIR}/git" + +diff --git a/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb b/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb +index 625756873..31c87c96f 100644 +--- a/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb ++++ b/layers/meta-openembedded/meta-oe/recipes-support/spitools/spitools_git.bb +@@ -10,7 +10,7 @@ SRCREV = "4a36a84f7df291ddaebd397aecf0c8515256a8e0" + + S = "${WORKDIR}/git" + +-SRC_URI = "git://github.com/cpb-/spi-tools.git;protocol=git" ++SRC_URI = "git://github.com/cpb-/spi-tools.git;protocol=https" + + + inherit autotools diff --git a/patches/no_net_check.patch b/patches/no_net_check.patch new file mode 100644 index 0000000..4c9e306 --- /dev/null +++ b/patches/no_net_check.patch @@ -0,0 +1,12 @@ +diff --git a/layers/openembedded-core/meta/classes/sanity.bbclass b/layers/openembedded-core/meta/classes/sanity.bbclass +index 2325ee2..5d2a8d4 100644 +--- a/layers/openembedded-core/meta/classes/sanity.bbclass ++++ b/layers/openembedded-core/meta/classes/sanity.bbclass +@@ -367,6 +367,7 @@ def check_connectivity(d): + # URI's to check can be set in the CONNECTIVITY_CHECK_URIS variable + # using the same syntax as for SRC_URI. If the variable is not set + # the check is skipped ++ return "" + test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS') or "").split() + retval = "" + diff --git a/patches/openembedded-core-github-https.patch b/patches/openembedded-core-github-https.patch new file mode 100644 index 0000000..bbeea1d --- /dev/null +++ b/patches/openembedded-core-github-https.patch @@ -0,0 +1,14 @@ +github no longer allows clear git protocol, so switch to https +=============================================================== +diff --git a/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc b/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc +index 3165d13f03..8b5260bb0d 100644 +--- a/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc ++++ b/layers/openembedded-core/meta/recipes-core/systemd/systemd.inc +@@ -16,6 +16,6 @@ LIC_FILES_CHKSUM = "file://LICENSE.GPL2;md5=751419260aa954499f7abaabaa882bbe \ + + SRCREV = "3ceaa81c61b654ebf562464d142675bd4d57d7b6" + SRCBRANCH = "v244-stable" +-SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=git;branch=${SRCBRANCH}" ++SRC_URI = "git://github.com/systemd/systemd-stable.git;protocol=https;branch=${SRCBRANCH}" + + S = "${WORKDIR}/git" diff --git a/patches/rename-mtr-conflicts-with-mtr-product.patch b/patches/rename-mtr-conflicts-with-mtr-product.patch new file mode 100644 index 0000000..1a07a37 --- /dev/null +++ b/patches/rename-mtr-conflicts-with-mtr-product.patch @@ -0,0 +1,58 @@ +diff -Naru orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb +--- orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb 2021-07-14 14:05:59.000000000 -0500 ++++ new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr_0.93.bb 1969-12-31 18:00:00.000000000 -0600 +@@ -1,25 +0,0 @@ +-SUMMARY = "Combined traceroute and ping utility" +-DESCRIPTION = "mtr combines the functionality of the 'traceroute' and 'ping' programs in a single network diagnostic tool." +-HOMEPAGE = "http://www.bitwizard.nl/mtr/" +-SECTION = "net" +-DEPENDS = "ncurses" +- +-LICENSE = "GPLv2" +-LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263 \ +- file://ui/mtr.c;beginline=5;endline=16;md5=00a894a39d53726a27386534d1c4e468" +- +-SRCREV = "304349bad86229aedbc62c07d5e98a8292967991" +-SRC_URI = "git://github.com/traviscross/mtr" +- +-S = "${WORKDIR}/git" +- +-inherit autotools pkgconfig +- +-EXTRA_OECONF = "--without-gtk" +- +-PACKAGECONFIG ??= "${@bb.utils.filter('DISTRO_FEATURES', 'ipv6', d)}" +-PACKAGECONFIG[ipv6] = "--enable-ipv6,--disable-ipv6," +- +-PACKAGES += "${PN}-bash-completions" +- +-FILES_${PN}-bash-completions = "${datadir}/bash-completion/" +diff -Naru orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb +--- orig/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb 1969-12-31 18:00:00.000000000 -0600 ++++ new/layers/meta-openembedded/meta-networking/recipes-support/mtr/mtr-conflicts-with-mtr-product_0.93.bb 2021-07-14 14:05:59.000000000 -0500 +@@ -0,0 +1,25 @@ ++SUMMARY = "Combined traceroute and ping utility" ++DESCRIPTION = "mtr combines the functionality of the 'traceroute' and 'ping' programs in a single network diagnostic tool." ++HOMEPAGE = "http://www.bitwizard.nl/mtr/" ++SECTION = "net" ++DEPENDS = "ncurses" ++ ++LICENSE = "GPLv2" ++LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263 \ ++ file://ui/mtr.c;beginline=5;endline=16;md5=00a894a39d53726a27386534d1c4e468" ++ ++SRCREV = "304349bad86229aedbc62c07d5e98a8292967991" ++SRC_URI = "git://github.com/traviscross/mtr" ++ ++S = "${WORKDIR}/git" ++ ++inherit autotools pkgconfig ++ ++EXTRA_OECONF = "--without-gtk" ++ ++PACKAGECONFIG ??= "${@bb.utils.filter('DISTRO_FEATURES', 'ipv6', d)}" ++PACKAGECONFIG[ipv6] = "--enable-ipv6,--disable-ipv6," ++ ++PACKAGES += "${PN}-bash-completions" ++ ++FILES_${PN}-bash-completions = "${datadir}/bash-completion/" diff --git a/scripts/README b/scripts/README new file mode 100644 index 0000000..1b8d127 --- /dev/null +++ b/scripts/README @@ -0,0 +1 @@ +This directory contains Various useful scripts for working with OE builds diff --git a/scripts/bitbake-prserv-tool b/scripts/bitbake-prserv-tool new file mode 100755 index 0000000..fa31b52 --- /dev/null +++ b/scripts/bitbake-prserv-tool @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +help () +{ + base=`basename $0` + echo -e "Usage: $base command" + echo "Avaliable commands:" + echo -e "\texport : export and lock down the AUTOPR values from the PR service into a file for release." + echo -e "\timport : import the AUTOPR values from the exported file into the PR service." +} + +clean_cache() +{ + s=`bitbake -e | grep ^CACHE= | cut -f2 -d\"` + if [ "x${s}" != "x" ]; then + rm -rf ${s} + fi +} + +do_export () +{ + file=$1 + [ "x${file}" == "x" ] && help && exit 1 + rm -f ${file} + + clean_cache + bitbake -R conf/prexport.conf -p + s=`bitbake -R conf/prexport.conf -e | grep ^PRSERV_DUMPFILE= | cut -f2 -d\"` + if [ "x${s}" != "x" ]; + then + [ -e $s ] && mv -f $s $file && echo "Exporting to file $file succeeded!" + return 0 + fi + echo "Exporting to file $file failed!" + return 1 +} + +do_import () +{ + file=$1 + [ "x${file}" == "x" ] && help && exit 1 + + clean_cache + bitbake -R conf/primport.conf -R $file -p + ret=$? + [ $ret -eq 0 ] && echo "Importing from file $file succeeded!" || echo "Importing from file $file failed!" + return $ret +} + +do_migrate_localcount () +{ + df=`bitbake -R conf/migrate_localcount.conf -e | \ + grep ^LOCALCOUNT_DUMPFILE= | cut -f2 -d\"` + if [ "x${df}" == "x" ]; + then + echo "LOCALCOUNT_DUMPFILE is not defined!" + return 1 + fi + + rm -rf $df + clean_cache + echo "Exporting LOCALCOUNT to AUTOINCs..." + bitbake -R conf/migrate_localcount.conf -p + [ ! $? -eq 0 ] && echo "Exporting to file $df failed!" && exit 1 + + if [ -e $df ]; + then + echo "Exporting to file $df succeeded!" + else + echo "Exporting to file $df failed!" + exit 1 + fi + + echo "Importing generated AUTOINC entries..." + [ -e $df ] && do_import $df + + if [ ! $? -eq 0 ] + then + echo "Migration from LOCALCOUNT to AUTOINCs failed!" + return 1 + fi + + echo "Migration from LOCALCOUNT to AUTOINCs succeeded!" + return 0 +} + +[ $# -eq 0 ] && help && exit 1 + +case $2 in +*.conf|*.inc) + ;; +*) + echo ERROR: $2 must end with .conf or .inc! + exit 1 + ;; +esac + +case $1 in +export) + do_export $2 + ;; +import) + do_import $2 + ;; +migrate_localcount) + do_migrate_localcount + ;; +*) + help + exit 1 + ;; +esac diff --git a/scripts/bitbake-whatchanged b/scripts/bitbake-whatchanged new file mode 100755 index 0000000..af54d16 --- /dev/null +++ b/scripts/bitbake-whatchanged @@ -0,0 +1,339 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- + +# Copyright (c) 2013 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from __future__ import print_function +import os +import sys +import getopt +import shutil +import re +import warnings +import subprocess +from optparse import OptionParser + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] + +import scriptpath + +# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process +bitbakepath = scriptpath.add_bitbake_lib_path() +if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + +import bb.siggen +import bb.process + +# Match the stamp's filename +# group(1): PE_PV (may no PE) +# group(2): PR +# group(3): TASK +# group(4): HASH +stamp_re = re.compile("(?P.*)-(?Pr\d+)\.(?Pdo_\w+)\.(?P[^\.]*)") +sigdata_re = re.compile(".*\.sigdata\..*") + +def gen_dict(stamps): + """ + Generate the dict from the stamps dir. + The output dict format is: + {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}} + Where: + fake_f: pv + task + hash + path: the path to the stamp file + """ + # The member of the sub dict (A "path" will be appended below) + sub_mem = ("pv", "pr", "task") + d = {} + for dirpath, _, files in os.walk(stamps): + for f in files: + # The "bitbake -S" would generate ".sigdata", but no "_setscene". + fake_f = re.sub('_setscene.', '.', f) + fake_f = re.sub('.sigdata', '', fake_f) + subdict = {} + tmp = stamp_re.match(fake_f) + if tmp: + for i in sub_mem: + subdict[i] = tmp.group(i) + if len(subdict) != 0: + pn = os.path.basename(dirpath) + subdict['pn'] = pn + # The path will be used by os.stat() and bb.siggen + subdict['path'] = dirpath + "/" + f + fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash') + d[fake_f] = subdict + return d + +# Re-construct the dict +def recon_dict(dict_in): + """ + The output dict format is: + {pn_task: {pv: PV, pr: PR, path: PATH}} + """ + dict_out = {} + for k in dict_in.keys(): + subdict = {} + # The key + pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task')) + # If more than one stamps are found, use the latest one. + if pn_task in dict_out: + full_path_pre = dict_out.get(pn_task).get('path') + full_path_cur = dict_in.get(k).get('path') + if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime: + continue + subdict['pv'] = dict_in.get(k).get('pv') + subdict['pr'] = dict_in.get(k).get('pr') + subdict['path'] = dict_in.get(k).get('path') + dict_out[pn_task] = subdict + + return dict_out + +def split_pntask(s): + """ + Split the pn_task in to (pn, task) and return it + """ + tmp = re.match("(.*)_(do_.*)", s) + return (tmp.group(1), tmp.group(2)) + + +def print_added(d_new = None, d_old = None): + """ + Print the newly added tasks + """ + added = {} + for k in d_new.keys(): + if k not in d_old: + # Add the new one to added dict, and remove it from + # d_new, so the remaining ones are the changed ones + added[k] = d_new.get(k) + del(d_new[k]) + + if not added: + return 0 + + # Format the output, the dict format is: + # {pn: task1, task2 ...} + added_format = {} + counter = 0 + for k in added.keys(): + pn, task = split_pntask(k) + if pn in added_format: + # Append the value + added_format[pn] = "%s %s" % (added_format.get(pn), task) + else: + added_format[pn] = task + counter += 1 + print("=== Newly added tasks: (%s tasks)" % counter) + for k in added_format.keys(): + print(" %s: %s" % (k, added_format.get(k))) + + return counter + +def print_vrchanged(d_new = None, d_old = None, vr = None): + """ + Print the pv or pr changed tasks. + The arg "vr" is "pv" or "pr" + """ + pvchanged = {} + counter = 0 + for k in d_new.keys(): + if d_new.get(k).get(vr) != d_old.get(k).get(vr): + counter += 1 + pn, task = split_pntask(k) + if pn not in pvchanged: + # Format the output, we only print pn (no task) since + # all the tasks would be changed when pn or pr changed, + # the dict format is: + # {pn: pv/pr_old -> pv/pr_new} + pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr)) + del(d_new[k]) + + if not pvchanged: + return 0 + + print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter)) + for k in pvchanged.keys(): + print(" %s: %s" % (k, pvchanged.get(k))) + + return counter + +def print_depchanged(d_new = None, d_old = None, verbose = False): + """ + Print the dependency changes + """ + depchanged = {} + counter = 0 + for k in d_new.keys(): + counter += 1 + pn, task = split_pntask(k) + if (verbose): + full_path_old = d_old.get(k).get("path") + full_path_new = d_new.get(k).get("path") + # No counter since it is not ready here + if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new): + output = bb.siggen.compare_sigfiles(full_path_old, full_path_new) + if output: + print("\n=== The verbose changes of %s.%s:" % (pn, task)) + print('\n'.join(output)) + else: + # Format the output, the format is: + # {pn: task1, task2, ...} + if pn in depchanged: + depchanged[pn] = "%s %s" % (depchanged.get(pn), task) + else: + depchanged[pn] = task + + if len(depchanged) > 0: + print("\n=== Dependencies changed: (%s tasks)" % counter) + for k in depchanged.keys(): + print(" %s: %s" % (k, depchanged[k])) + + return counter + + +def main(): + """ + Print what will be done between the current and last builds: + 1) Run "STAMPS_DIR= bitbake -S recipe" to re-generate the stamps + 2) Figure out what are newly added and changed, can't figure out + what are removed since we can't know the previous stamps + clearly, for example, if there are several builds, we can't know + which stamps the last build has used exactly. + 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps + """ + + parser = OptionParser( + version = "1.0", + usage = """%prog [options] [package ...] +print what will be done between the current and last builds, for example: + + $ bitbake core-image-sato + # Edit the recipes + $ bitbake-whatchanged core-image-sato + +The changes will be printed" + +Note: + The amount of tasks is not accurate when the task is "do_build" since + it usually depends on other tasks. + The "nostamp" task is not included. +""" +) + parser.add_option("-v", "--verbose", help = "print the verbose changes", + action = "store_true", dest = "verbose") + + options, args = parser.parse_args(sys.argv) + + verbose = options.verbose + + if len(args) != 2: + parser.error("Incorrect number of arguments") + else: + recipe = args[1] + + # Get the STAMPS_DIR + print("Figuring out the STAMPS_DIR ...") + cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'" + try: + stampsdir, err = bb.process.run(cmdline) + except: + raise + if not stampsdir: + print("ERROR: No STAMPS_DIR found for '%s'" % recipe, file=sys.stderr) + return 2 + stampsdir = stampsdir.rstrip("\n") + if not os.path.isdir(stampsdir): + print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr) + return 2 + + # The new stamps dir + new_stampsdir = stampsdir + ".bbs" + if os.path.exists(new_stampsdir): + print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr) + return 2 + + try: + # Generate the new stamps dir + print("Generating the new stamps ... (need several minutes)") + cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, recipe) + # FIXME + # The "bitbake -S" may fail, not fatal error, the stamps will still + # be generated, this might be a bug of "bitbake -S". + try: + bb.process.run(cmdline) + except Exception as exc: + print(exc) + + # The dict for the new and old stamps. + old_dict = gen_dict(stampsdir) + new_dict = gen_dict(new_stampsdir) + + # Remove the same one from both stamps. + cnt_unchanged = 0 + for k in new_dict.keys(): + if k in old_dict: + cnt_unchanged += 1 + del(new_dict[k]) + del(old_dict[k]) + + # Re-construct the dict to easily find out what is added or changed. + # The dict format is: + # {pn_task: {pv: PV, pr: PR, path: PATH}} + new_recon = recon_dict(new_dict) + old_recon = recon_dict(old_dict) + + del new_dict + del old_dict + + # Figure out what are changed, the new_recon would be changed + # by the print_xxx function. + # Newly added + cnt_added = print_added(new_recon, old_recon) + + # PV (including PE) and PR changed + # Let the bb.siggen handle them if verbose + cnt_rv = {} + if not verbose: + for i in ('pv', 'pr'): + cnt_rv[i] = print_vrchanged(new_recon, old_recon, i) + + # Dependencies changed (use bitbake-diffsigs) + cnt_dep = print_depchanged(new_recon, old_recon, verbose) + + total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep + + print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged)) + if verbose: + print("Newly added: %s\nDependencies changed: %s\n" % \ + (cnt_added, cnt_dep)) + else: + print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \ + (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep)) + except: + print("ERROR occurred!") + raise + finally: + # Remove the newly generated stamps dir + if os.path.exists(new_stampsdir): + print("Removing the newly generated stamps dir ...") + shutil.rmtree(new_stampsdir) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/buildhistory-collect-srcrevs b/scripts/buildhistory-collect-srcrevs new file mode 100755 index 0000000..f3eb76b --- /dev/null +++ b/scripts/buildhistory-collect-srcrevs @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# +# Collects the recorded SRCREV values from buildhistory and reports on them +# +# Copyright 2013 Intel Corporation +# Authored-by: Paul Eggleton +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import collections +import os +import sys +import optparse +import logging + +def logger_create(): + logger = logging.getLogger("buildhistory") + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create() + +def main(): + parser = optparse.OptionParser( + description = "Collects the recorded SRCREV values from buildhistory and reports on them.", + usage = """ + %prog [options]""") + + parser.add_option("-a", "--report-all", + help = "Report all SRCREV values, not just ones where AUTOREV has been used", + action="store_true", dest="reportall") + parser.add_option("-f", "--forcevariable", + help = "Use forcevariable override for all output lines", + action="store_true", dest="forcevariable") + parser.add_option("-p", "--buildhistory-dir", + help = "Specify path to buildhistory directory (defaults to buildhistory/ under cwd)", + action="store", dest="buildhistory_dir", default='buildhistory/') + + options, args = parser.parse_args(sys.argv) + + if len(args) > 1: + sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args[1:])) + parser.print_help() + sys.exit(1) + + if not os.path.exists(options.buildhistory_dir): + sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % options.buildhistory_dir) + parser.print_help() + sys.exit(1) + + if options.forcevariable: + forcevariable = '_forcevariable' + else: + forcevariable = '' + + all_srcrevs = collections.defaultdict(list) + for root, dirs, files in os.walk(options.buildhistory_dir): + if '.git' in dirs: + dirs.remove('.git') + for fn in files: + if fn == 'latest_srcrev': + curdir = os.path.basename(os.path.dirname(root)) + fullpath = os.path.join(root, fn) + pn = os.path.basename(root) + srcrev = None + orig_srcrev = None + orig_srcrevs = {} + srcrevs = {} + with open(fullpath) as f: + for line in f: + if '=' in line: + splitval = line.split('=') + value = splitval[1].strip('" \t\n\r') + if line.startswith('# SRCREV = '): + orig_srcrev = value + elif line.startswith('# SRCREV_'): + splitval = line.split('=') + name = splitval[0].split('_')[1].strip() + orig_srcrevs[name] = value + elif line.startswith('SRCREV ='): + srcrev = value + elif line.startswith('SRCREV_'): + name = splitval[0].split('_')[1].strip() + srcrevs[name] = value + if srcrev and (options.reportall or srcrev != orig_srcrev): + all_srcrevs[curdir].append((pn, None, srcrev)) + for name, value in srcrevs.items(): + orig = orig_srcrevs.get(name, orig_srcrev) + if options.reportall or value != orig: + all_srcrevs[curdir].append((pn, name, srcrev)) + + for curdir, srcrevs in sorted(all_srcrevs.iteritems()): + if srcrevs: + print('# %s' % curdir) + for pn, name, srcrev in srcrevs: + if name: + print('SRCREV_%s_pn-%s%s = "%s"' % (name, pn, forcevariable, srcrev)) + else: + print('SRCREV_pn-%s%s = "%s"' % (pn, forcevariable, srcrev)) + + +if __name__ == "__main__": + main() diff --git a/scripts/buildhistory-diff b/scripts/buildhistory-diff new file mode 100755 index 0000000..dfebcdd --- /dev/null +++ b/scripts/buildhistory-diff @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# Report significant differences in the buildhistory repository since a specific revision +# +# Copyright (C) 2013 Intel Corporation +# Author: Paul Eggleton + +import sys +import os +import optparse +from distutils.version import LooseVersion + +# Ensure PythonGit is installed (buildhistory_analysis needs it) +try: + import git +except ImportError: + print("Please install GitPython (python-git) 0.3.1 or later in order to use this script") + sys.exit(1) + +def main(): + parser = optparse.OptionParser( + description = "Reports significant differences in the buildhistory repository.", + usage = """ + %prog [options] [from-revision [to-revision]] +(if not specified, from-revision defaults to build-minus-1, and to-revision defaults to HEAD)""") + + parser.add_option("-p", "--buildhistory-dir", + help = "Specify path to buildhistory directory (defaults to buildhistory/ under cwd)", + action="store", dest="buildhistory_dir", default='buildhistory/') + parser.add_option("-v", "--report-version", + help = "Report changes in PKGE/PKGV/PKGR even when the values are still the default (PE/PV/PR)", + action="store_true", dest="report_ver", default=False) + parser.add_option("-a", "--report-all", + help = "Report all changes, not just the default significant ones", + action="store_true", dest="report_all", default=False) + + options, args = parser.parse_args(sys.argv) + + if len(args) > 3: + sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args[3:])) + parser.print_help() + sys.exit(1) + + if LooseVersion(git.__version__) < '0.3.1': + sys.stderr.write("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script\n") + sys.exit(1) + + if not os.path.exists(options.buildhistory_dir): + sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % options.buildhistory_dir) + parser.print_help() + sys.exit(1) + + scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) + lib_path = scripts_path + '/lib' + sys.path = sys.path + [lib_path] + + import scriptpath + + # Set path to OE lib dir so we can import the buildhistory_analysis module + scriptpath.add_oe_lib_path() + # Set path to bitbake lib dir so the buildhistory_analysis module can load bb.utils + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + + import oe.buildhistory_analysis + + fromrev = 'build-minus-1' + torev = 'HEAD' + if len(args) > 1: + if len(args) == 2 and '..' in args[1]: + revs = args[1].split('..') + fromrev = revs[0] + if revs[1]: + torev = revs[1] + else: + fromrev = args[1] + if len(args) > 2: + torev = args[2] + + import gitdb + try: + changes = oe.buildhistory_analysis.process_changes(options.buildhistory_dir, fromrev, torev, options.report_all, options.report_ver) + except gitdb.exc.BadObject as e: + if len(args) == 1: + sys.stderr.write("Unable to find previous build revision in buildhistory repository\n\n") + parser.print_help() + else: + sys.stderr.write('Specified git revision "%s" is not valid\n' % e.args[0]) + sys.exit(1) + + for chg in changes: + print('%s' % chg) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/cleanup-workdir b/scripts/cleanup-workdir new file mode 100755 index 0000000..01ebd52 --- /dev/null +++ b/scripts/cleanup-workdir @@ -0,0 +1,198 @@ +#!/usr/bin/env python + +# Copyright (c) 2012 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import sys +import optparse +import re +import subprocess +import shutil + +pkg_cur_dirs = {} +obsolete_dirs = [] +parser = None + +def err_quit(msg): + print msg + parser.print_usage() + sys.exit(1) + +def parse_version(verstr): + elems = verstr.split(':') + epoch = elems[0] + if len(epoch) == 0: + return elems[1] + else: + return epoch + '_' + elems[1] + +def run_command(cmd): + pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + output = pipe.communicate()[0] + if pipe.returncode != 0: + print "Execute command '%s' failed." % cmd + sys.exit(1) + return output + +def get_cur_arch_dirs(workdir, arch_dirs): + pattern = workdir + '/(.*?)/' + + cmd = "bitbake -e | grep ^SDK_ARCH=" + output = run_command(cmd) + sdk_arch = output.split('"')[1] + + # select thest 5 packages to get the dirs of current arch + pkgs = ['hicolor-icon-theme', 'base-files', 'acl-native', 'binutils-crosssdk-' + sdk_arch, 'nativesdk-autoconf'] + + for pkg in pkgs: + cmd = "bitbake -e " + pkg + " | grep ^IMAGE_ROOTFS=" + output = run_command(cmd) + output = output.split('"')[1] + m = re.match(pattern, output) + arch_dirs.append(m.group(1)) + +def main(): + global parser + parser = optparse.OptionParser( + usage = """%prog + +%prog removes the obsolete packages' build directories in WORKDIR. +This script must be ran under BUILDDIR after source file \"oe-init-build-env\". + +Any file or directory under WORKDIR which is not created by Yocto +will be deleted. Be CAUTIOUS.""") + + options, args = parser.parse_args(sys.argv) + + builddir = run_command('echo $BUILDDIR').strip() + if len(builddir) == 0: + err_quit("Please source file \"oe-init-build-env\" first.\n") + + if os.getcwd() != builddir: + err_quit("Please run %s under: %s\n" % (os.path.basename(args[0]), builddir)) + + print 'Updating bitbake caches...' + cmd = "bitbake -s" + output = run_command(cmd) + + output = output.split('\n') + index = 0 + while len(output[index]) > 0: + index += 1 + alllines = output[index+1:] + + for line in alllines: + # empty again means end of the versions output + if len(line) == 0: + break + line = line.strip() + line = re.sub('\s+', ' ', line) + elems = line.split(' ') + if len(elems) == 2: + version = parse_version(elems[1]) + else: + version = parse_version(elems[2]) + pkg_cur_dirs[elems[0]] = version + + cmd = "bitbake -e" + output = run_command(cmd) + + tmpdir = None + image_rootfs = None + output = output.split('\n') + for line in output: + if tmpdir and image_rootfs: + break + + if not tmpdir: + m = re.match('TMPDIR="(.*)"', line) + if m: + tmpdir = m.group(1) + + if not image_rootfs: + m = re.match('IMAGE_ROOTFS="(.*)"', line) + if m: + image_rootfs = m.group(1) + + # won't fail just in case + if not tmpdir or not image_rootfs: + print "Can't get TMPDIR or IMAGE_ROOTFS." + return 1 + + pattern = tmpdir + '/(.*?)/(.*?)/' + m = re.match(pattern, image_rootfs) + if not m: + print "Can't get WORKDIR." + return 1 + + workdir = os.path.join(tmpdir, m.group(1)) + + # we only deal the dirs of current arch, total numbers of dirs are 6 + cur_arch_dirs = [m.group(2)] + get_cur_arch_dirs(workdir, cur_arch_dirs) + + for workroot, dirs, files in os.walk(workdir): + # For the files, they should NOT exist in WORKDIR. Remove them. + for f in files: + obsolete_dirs.append(os.path.join(workroot, f)) + + for d in dirs: + if d not in cur_arch_dirs: + continue + + for pkgroot, pkgdirs, filenames in os.walk(os.path.join(workroot, d)): + for f in filenames: + obsolete_dirs.append(os.path.join(pkgroot, f)) + + for pkgdir in sorted(pkgdirs): + if pkgdir not in pkg_cur_dirs: + obsolete_dirs.append(os.path.join(pkgroot, pkgdir)) + else: + for verroot, verdirs, verfiles in os.walk(os.path.join(pkgroot, pkgdir)): + for f in verfiles: + obsolete_dirs.append(os.path.join(pkgroot, f)) + for v in sorted(verdirs): + if v not in pkg_cur_dirs[pkgdir]: + obsolete_dirs.append(os.path.join(pkgroot, pkgdir, v)) + break + + # just process the top dir of every package under tmp/work/*/, + # then jump out of the above os.walk() + break + + # it is convenient to use os.walk() to get dirs and files at same time + # both of them have been dealed in the loop, so jump out + break + + for d in obsolete_dirs: + print "Deleting %s" % d + shutil.rmtree(d, True) + + if len(obsolete_dirs): + print '\nTotal %d items.' % len(obsolete_dirs) + else: + print '\nNo obsolete directory found under %s.' % workdir + + return 0 + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 2 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/combo-layer b/scripts/combo-layer new file mode 100755 index 0000000..9127041 --- /dev/null +++ b/scripts/combo-layer @@ -0,0 +1,924 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright 2011 Intel Corporation +# Authored-by: Yu Ke +# Paul Eggleton +# Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import fnmatch +import os, sys +import optparse +import logging +import subprocess +import tempfile +import ConfigParser +import re +from collections import OrderedDict +from string import Template + +__version__ = "0.2.1" + +def logger_create(): + logger = logging.getLogger("") + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger + +logger = logger_create() + +def get_current_branch(repodir=None): + try: + if not os.path.exists(os.path.join(repodir if repodir else '', ".git")): + # Repo not created yet (i.e. during init) so just assume master + return "master" + branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip() + if branchname.startswith("refs/heads/"): + branchname = branchname[11:] + return branchname + except subprocess.CalledProcessError: + return "" + +class Configuration(object): + """ + Manages the configuration + + For an example config file, see combo-layer.conf.example + + """ + def __init__(self, options): + for key, val in options.__dict__.items(): + setattr(self, key, val) + + def readsection(parser, section, repo): + for (name, value) in parser.items(section): + if value.startswith("@"): + self.repos[repo][name] = eval(value.strip("@")) + else: + # Apply special type transformations for some properties. + # Type matches the RawConfigParser.get*() methods. + types = {'signoff': 'boolean', 'update': 'boolean'} + if name in types: + value = getattr(parser, 'get' + types[name])(section, name) + self.repos[repo][name] = value + + def readglobalsection(parser, section): + for (name, value) in parser.items(section): + if name == "commit_msg": + self.commit_msg_template = value + + logger.debug("Loading config file %s" % self.conffile) + self.parser = ConfigParser.ConfigParser() + with open(self.conffile) as f: + self.parser.readfp(f) + + # initialize default values + self.commit_msg_template = "Automatic commit to update last_revision" + + self.repos = {} + for repo in self.parser.sections(): + if repo == "combo-layer-settings": + # special handling for global settings + readglobalsection(self.parser, repo) + else: + self.repos[repo] = {} + readsection(self.parser, repo, repo) + + # Load local configuration, if available + self.localconffile = None + self.localparser = None + self.combobranch = None + if self.conffile.endswith('.conf'): + lcfile = self.conffile.replace('.conf', '-local.conf') + if os.path.exists(lcfile): + # Read combo layer branch + self.combobranch = get_current_branch() + logger.debug("Combo layer branch is %s" % self.combobranch) + + self.localconffile = lcfile + logger.debug("Loading local config file %s" % self.localconffile) + self.localparser = ConfigParser.ConfigParser() + with open(self.localconffile) as f: + self.localparser.readfp(f) + + for section in self.localparser.sections(): + if '|' in section: + sectionvals = section.split('|') + repo = sectionvals[0] + if sectionvals[1] != self.combobranch: + continue + else: + repo = section + if repo in self.repos: + readsection(self.localparser, section, repo) + + def update(self, repo, option, value, initmode=False): + # If the main config has the option already, that is what we + # are expected to modify. + if self.localparser and not self.parser.has_option(repo, option): + parser = self.localparser + section = "%s|%s" % (repo, self.combobranch) + conffile = self.localconffile + if initmode and not parser.has_section(section): + parser.add_section(section) + else: + parser = self.parser + section = repo + conffile = self.conffile + parser.set(section, option, value) + with open(conffile, "w") as f: + parser.write(f) + self.repos[repo][option] = value + + def sanity_check(self, initmode=False): + required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"] + if initmode: + required_options.remove("last_revision") + msg = "" + missing_options = [] + for name in self.repos: + for option in required_options: + if option not in self.repos[name]: + msg = "%s\nOption %s is not defined for component %s" %(msg, option, name) + missing_options.append(option) + # Sanitize dest_dir so that we do not have to deal with edge cases + # (unset, empty string, double slashes) in the rest of the code. + # It not being set will still be flagged as error because it is + # listed as required option above; that could be changed now. + dest_dir = os.path.normpath(self.repos[name].get("dest_dir", ".")) + self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir + if msg != "": + logger.error("configuration file %s has the following error: %s" % (self.conffile,msg)) + if self.localconffile and 'last_revision' in missing_options: + logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch)) + sys.exit(1) + + # filterdiff is required by action_splitpatch, so check its availability + if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0: + logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)") + sys.exit(1) + +def runcmd(cmd,destdir=None,printerr=True,out=None): + """ + execute command, raise CalledProcessError if fail + return output if succeed + """ + logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) + if not out: + out = os.tmpfile() + err = out + else: + err = os.tmpfile() + try: + subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str)) + except subprocess.CalledProcessError,e: + err.seek(0) + if printerr: + logger.error("%s" % err.read()) + raise e + + err.seek(0) + output = err.read() + logger.debug("output: %s" % output ) + return output + +def action_init(conf, args): + """ + Clone component repositories + Check git is initialised; if not, copy initial data from component repos + """ + for name in conf.repos: + ldir = conf.repos[name]['local_repo_dir'] + if not os.path.exists(ldir): + logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir)) + subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True) + if not os.path.exists(".git"): + runcmd("git init") + if conf.history: + # Need a common ref for all trees. + runcmd('git commit -m "initial empty commit" --allow-empty') + startrev = runcmd('git rev-parse master').strip() + + for name in conf.repos: + repo = conf.repos[name] + ldir = repo['local_repo_dir'] + branch = repo.get('branch', "master") + lastrev = repo.get('last_revision', None) + if lastrev and lastrev != "HEAD": + initialrev = lastrev + if branch: + if not check_rev_branch(name, ldir, lastrev, branch): + sys.exit(1) + logger.info("Copying data from %s at specified revision %s..." % (name, lastrev)) + else: + lastrev = None + initialrev = branch + logger.info("Copying data from %s..." % name) + # Sanity check initialrev and turn it into hash (required for copying history, + # because resolving a name ref only works in the component repo). + rev = runcmd('git rev-parse %s' % initialrev, ldir).strip() + if rev != initialrev: + try: + refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n') + if len(set(refs)) > 1: + # Happens for example when configured to track + # "master" and there is a refs/heads/master. The + # traditional behavior from "git archive" (preserved + # here) it to choose the first one. This might not be + # intended, so at least warn about it. + logger.warn("%s: initial revision '%s' not unique, picking result of rev-parse = %s" % + (name, initialrev, refs[0])) + initialrev = rev + except: + # show-ref fails for hashes. Skip the sanity warning in that case. + pass + initialrev = rev + dest_dir = repo['dest_dir'] + if dest_dir != ".": + extract_dir = os.path.join(os.getcwd(), dest_dir) + if not os.path.exists(extract_dir): + os.makedirs(extract_dir) + else: + extract_dir = os.getcwd() + file_filter = repo.get('file_filter', "") + exclude_patterns = repo.get('file_exclude', '').split() + def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir, + subdir=""): + # When working inside a filtered branch which had the + # files already moved, we need to prepend the + # subdirectory to all filters, otherwise they would + # not match. + if subdir == '.': + subdir = '' + elif subdir: + subdir = os.path.normpath(subdir) + file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()]) + exclude_patterns = [subdir + '/' + x for x in exclude_patterns] + # To handle both cases, we cd into the target + # directory and optionally tell tar to strip the path + # prefix when the files were already moved. + subdir_components = len(subdir.split(os.path.sep)) if subdir else 0 + strip=('--strip-components=%d' % subdir_components) if subdir else '' + # TODO: file_filter wild cards do not work (and haven't worked before either), because + # a) GNU tar requires a --wildcards parameter before turning on wild card matching. + # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c, + # in contrast to the other use of file_filter as parameter of "git archive" + # where it only matches .c files directly in src). + files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" % + (initialrev, subdir, + strip, extract_dir, file_filter), + ldir) + if exclude_patterns: + # Implement file removal by letting tar create the + # file and then deleting it in the file system + # again. Uses the list of files created by tar (easier + # than walking the tree). + for file in files.split('\n'): + for pattern in exclude_patterns: + if fnmatch.fnmatch(file, pattern): + os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file]))) + break + + if not conf.history: + copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir) + else: + # First fetch remote history into local repository. + # We need a ref for that, so ensure that there is one. + refname = "combo-layer-init-%s" % name + runcmd("git branch -f %s %s" % (refname, initialrev), ldir) + runcmd("git fetch %s %s" % (ldir, refname)) + runcmd("git branch -D %s" % refname, ldir) + # Make that the head revision. + runcmd("git checkout -b %s %s" % (name, initialrev)) + # Optional: cut the history by replacing the given + # start point(s) with commits providing the same + # content (aka tree), but with commit information that + # makes it clear that this is an artifically created + # commit and nothing the original authors had anything + # to do with. + since_rev = repo.get('since_revision', '') + if since_rev: + committer = runcmd('git var GIT_AUTHOR_IDENT').strip() + # Same time stamp, no name. + author = re.sub('.* (\d+ [+-]\d+)', r'unknown \1', committer) + logger.info('author %s' % author) + for rev in since_rev.split(): + # Resolve in component repo... + rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0] + # ... and then get the tree in current + # one. The commit should be in both repos with + # the same tree, but better check here. + tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip() + with tempfile.NamedTemporaryFile() as editor: + editor.write('''cat >$1 <>$tmpname +cat >>$tmpname +if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then + echo >>$tmpname +fi +echo '---' >>$tmpname +%s $tmpname $GIT_COMMIT %s +tail -c +18 $tmpname | head -c -4 +''' % (hook, name)) + hookwrapper.flush() + filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name]) + if dest_dir != ".": + parent = os.path.dirname(dest_dir) + if not parent: + parent = '.' + # May run outside of the current directory, so do not assume that .git exists. + filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)]) + filter_branch.append('HEAD') + runcmd(filter_branch) + runcmd('git update-ref -d refs/original/refs/heads/%s' % name) + repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip() + repo['stripped_revision'] = repo['rewritten_revision'] + # Optional filter files: remove everything and re-populate using the normal filtering code. + # Override any potential .gitignore. + if file_filter or exclude_patterns: + runcmd('git rm -rf .') + if not os.path.exists(extract_dir): + os.makedirs(extract_dir) + copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.', + subdir=dest_dir) + runcmd('git add --all --force .') + if runcmd('git status --porcelain'): + # Something to commit. + runcmd(['git', 'commit', '-m', + '''%s: select file subset + +Files from the component repository were chosen based on +the following filters: +file_filter = %s +file_exclude = %s''' % (name, file_filter or '', repo.get('file_exclude', ''))]) + repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip() + + if not lastrev: + lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip() + conf.update(name, "last_revision", lastrev, initmode=True) + + if not conf.history: + runcmd("git add .") + else: + # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies + runcmd('git checkout master') + merge = ['git', 'merge', '--no-commit'] + for name in conf.repos: + repo = conf.repos[name] + # Use branch created earlier. + merge.append(name) + # Root all commits which have no parent in the common + # ancestor in the new repository. + for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s' % name).split('\n'): + runcmd('git replace --graft %s %s' % (start, startrev)) + try: + runcmd(merge) + except Exception, error: + logger.info('''Merging component repository history failed, perhaps because of merge conflicts. +It may be possible to commit anyway after resolving these conflicts. + +%s''' % error) + # Create MERGE_HEAD and MERGE_MSG. "git merge" itself + # does not create MERGE_HEAD in case of a (harmless) failure, + # and we want certain auto-generated information in the + # commit message for future reference and/or automation. + with open('.git/MERGE_HEAD', 'w') as head: + with open('.git/MERGE_MSG', 'w') as msg: + msg.write('repo: initial import of components\n\n') + # head.write('%s\n' % startrev) + for name in conf.repos: + repo = conf.repos[name] + # + msg.write('combo-layer-%s: %s %s %s\n' % (name, + repo['last_revision'], + repo['rewritten_revision'], + repo['stripped_revision'])) + rev = runcmd('git rev-parse %s' % name).strip() + head.write('%s\n' % rev) + + if conf.localconffile: + localadded = True + try: + runcmd("git rm --cached %s" % conf.localconffile, printerr=False) + except subprocess.CalledProcessError: + localadded = False + if localadded: + localrelpath = os.path.relpath(conf.localconffile) + runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath)) + runcmd("git add .gitignore") + logger.info("Added local configuration file %s to .gitignore", localrelpath) + logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.") + else: + logger.info("Repository already initialised, nothing to do.") + + +def check_repo_clean(repodir): + """ + check if the repo is clean + exit if repo is dirty + """ + output=runcmd("git status --porcelain", repodir) + r = re.compile('\?\? patch-.*/') + dirtyout = [item for item in output.splitlines() if not r.match(item)] + if dirtyout: + logger.error("git repo %s is dirty, please fix it first", repodir) + sys.exit(1) + +def check_patch(patchfile): + f = open(patchfile) + ln = f.readline() + of = None + in_patch = False + beyond_msg = False + pre_buf = '' + while ln: + if not beyond_msg: + if ln == '---\n': + if not of: + break + in_patch = False + beyond_msg = True + elif ln.startswith('--- '): + # We have a diff in the commit message + in_patch = True + if not of: + print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile) + of = open(patchfile + '.tmp', 'w') + of.write(pre_buf) + pre_buf = '' + elif in_patch and not ln[0] in '+-@ \n\r': + in_patch = False + if of: + if in_patch: + of.write(' ' + ln) + else: + of.write(ln) + else: + pre_buf += ln + ln = f.readline() + f.close() + if of: + of.close() + os.rename(patchfile + '.tmp', patchfile) + +def drop_to_shell(workdir=None): + if not sys.stdin.isatty(): + print "Not a TTY so can't drop to shell for resolution, exiting." + return False + + shell = os.environ.get('SHELL', 'bash') + print('Dropping to shell "%s"\n' \ + 'When you are finished, run the following to continue:\n' \ + ' exit -- continue to apply the patches\n' \ + ' exit 1 -- abort\n' % shell); + ret = subprocess.call([shell], cwd=workdir) + if ret != 0: + print "Aborting" + return False + else: + return True + +def check_rev_branch(component, repodir, rev, branch): + try: + actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False) + except subprocess.CalledProcessError as e: + if e.returncode == 129: + actualbranch = "" + else: + raise + + if not actualbranch: + logger.error("%s: specified revision %s is invalid!" % (component, rev)) + return False + + branches = [] + branchlist = actualbranch.split("\n") + for b in branchlist: + branches.append(b.strip().split(' ')[-1]) + + if branch not in branches: + logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch)) + return False + return True + +def get_repos(conf, repo_names): + repos = [] + for name in repo_names: + if name.startswith('-'): + break + else: + repos.append(name) + for repo in repos: + if not repo in conf.repos: + logger.error("Specified component '%s' not found in configuration" % repo) + sys.exit(1) + + if not repos: + repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ] + + return repos + +def action_pull(conf, args): + """ + update the component repos only + """ + repos = get_repos(conf, args[1:]) + + # make sure all repos are clean + for name in repos: + check_repo_clean(conf.repos[name]['local_repo_dir']) + + for name in repos: + repo = conf.repos[name] + ldir = repo['local_repo_dir'] + branch = repo.get('branch', "master") + logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir)) + if not conf.hard_reset: + # Try to pull only the configured branch. Beware that this may fail + # when the branch is currently unknown (for example, after reconfiguring + # combo-layer). In that case we need to fetch everything and try the check out + # and pull again. + try: + runcmd("git checkout %s" % branch, ldir, printerr=False) + except subprocess.CalledProcessError: + output=runcmd("git fetch", ldir) + logger.info(output) + runcmd("git checkout %s" % branch, ldir) + runcmd("git pull --ff-only", ldir) + else: + output=runcmd("git pull --ff-only", ldir) + logger.info(output) + else: + output=runcmd("git fetch", ldir) + logger.info(output) + runcmd("git checkout %s" % branch, ldir) + runcmd("git reset --hard FETCH_HEAD", ldir) + +def action_update(conf, args): + """ + update the component repos + generate the patch list + apply the generated patches + """ + components = [arg.split(':')[0] for arg in args[1:]] + revisions = {} + for arg in args[1:]: + if ':' in arg: + a = arg.split(':', 1) + revisions[a[0]] = a[1] + repos = get_repos(conf, components) + + # make sure combo repo is clean + check_repo_clean(os.getcwd()) + + import uuid + patch_dir = "patch-%s" % uuid.uuid4() + if not os.path.exists(patch_dir): + os.mkdir(patch_dir) + + # Step 1: update the component repos + if conf.nopull: + logger.info("Skipping pull (-n)") + else: + action_pull(conf, ['arg0'] + components) + + for name in repos: + revision = revisions.get(name, None) + repo = conf.repos[name] + ldir = repo['local_repo_dir'] + dest_dir = repo['dest_dir'] + branch = repo.get('branch', "master") + repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name) + + # Step 2: generate the patch list and store to patch dir + logger.info("Generating patches from %s..." % name) + top_revision = revision or branch + if not check_rev_branch(name, ldir, top_revision, branch): + sys.exit(1) + if dest_dir != ".": + prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir) + else: + prefix = "" + if repo['last_revision'] == "": + logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) + patch_cmd_range = "--root %s" % top_revision + rev_cmd_range = top_revision + else: + if not check_rev_branch(name, ldir, repo['last_revision'], branch): + sys.exit(1) + patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision) + rev_cmd_range = patch_cmd_range + + file_filter = repo.get('file_filter',".") + + # Filter out unwanted files + exclude = repo.get('file_exclude', '') + if exclude: + for path in exclude.split(): + p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path + file_filter += " ':!%s'" % p + + patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \ + (prefix,repo_patch_dir, patch_cmd_range, file_filter) + output = runcmd(patch_cmd, ldir) + logger.debug("generated patch set:\n%s" % output) + patchlist = output.splitlines() + + rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter) + revlist = runcmd(rev_cmd, ldir).splitlines() + + # Step 3: Call repo specific hook to adjust patch + if 'hook' in repo: + # hook parameter is: ./hook patchpath revision reponame + count=len(revlist)-1 + for patch in patchlist: + runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name)) + count=count-1 + + # Step 4: write patch list and revision list to file, for user to edit later + patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name) + repo['patchlist'] = patchlist_file + f = open(patchlist_file, 'w') + count=len(revlist)-1 + for patch in patchlist: + f.write("%s %s\n" % (patch, revlist[count])) + check_patch(os.path.join(patch_dir, patch)) + count=count-1 + f.close() + + # Step 5: invoke bash for user to edit patch and patch list + if conf.interactive: + print('You may now edit the patch and patch list in %s\n' \ + 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir); + if not drop_to_shell(patch_dir): + sys.exit(1) + + # Step 6: apply the generated and revised patch + apply_patchlist(conf, repos) + runcmd("rm -rf %s" % patch_dir) + + # Step 7: commit the updated config file if it's being tracked + relpath = os.path.relpath(conf.conffile) + try: + output = runcmd("git status --porcelain %s" % relpath, printerr=False) + except: + # Outside the repository + output = None + if output: + logger.info("Committing updated configuration file") + if output.lstrip().startswith("M"): + + # create the "components" string + component_str = "all components" + if len(components) > 0: + # otherwise tell which components were actually changed + component_str = ", ".join(components) + + # expand the template with known values + template = Template(conf.commit_msg_template) + raw_msg = template.substitute(components = component_str) + + # sanitize the string before using it in command line + msg = raw_msg.replace('"', '\\"') + + runcmd('git commit -m "%s" %s' % (msg, relpath)) + +def apply_patchlist(conf, repos): + """ + apply the generated patch list to combo repo + """ + for name in repos: + repo = conf.repos[name] + lastrev = repo["last_revision"] + prevrev = lastrev + + # Get non-blank lines from patch list file + patchlist = [] + if os.path.exists(repo['patchlist']) or not conf.interactive: + # Note: we want this to fail here if the file doesn't exist and we're not in + # interactive mode since the file should exist in this case + with open(repo['patchlist']) as f: + for line in f: + line = line.rstrip() + if line: + patchlist.append(line) + + ldir = conf.repos[name]['local_repo_dir'] + branch = conf.repos[name].get('branch', "master") + branchrev = runcmd("git rev-parse %s" % branch, ldir).strip() + + if patchlist: + logger.info("Applying patches from %s..." % name) + linecount = len(patchlist) + i = 1 + for line in patchlist: + patchfile = line.split()[0] + lastrev = line.split()[1] + patchdisp = os.path.relpath(patchfile) + if os.path.getsize(patchfile) == 0: + logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp)) + else: + cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile) + logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp)) + try: + runcmd(cmd) + except subprocess.CalledProcessError: + logger.info('Running "git am --abort" to cleanup repo') + runcmd("git am --abort") + logger.error('"%s" failed' % cmd) + logger.info("Please manually apply patch %s" % patchdisp) + logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped") + if not drop_to_shell(): + if prevrev != repo['last_revision']: + conf.update(name, "last_revision", prevrev) + sys.exit(1) + prevrev = lastrev + i += 1 + # Once all patches are applied, we should update + # last_revision to the branch head instead of the last + # applied patch. The two are not necessarily the same when + # the last commit is a merge commit or when the patches at + # the branch head were intentionally excluded. + # + # If we do not do that for a merge commit, the next + # combo-layer run will only exclude patches reachable from + # one of the merged branches and try to re-apply patches + # from other branches even though they were already + # copied. + # + # If patches were intentionally excluded, the next run will + # present them again instead of skipping over them. This + # may or may not be intended, so the code here is conservative + # and only addresses the "head is merge commit" case. + if lastrev != branchrev and \ + len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1: + lastrev = branchrev + else: + logger.info("No patches to apply from %s" % name) + lastrev = branchrev + + if lastrev != repo['last_revision']: + conf.update(name, "last_revision", lastrev) + +def action_splitpatch(conf, args): + """ + generate the commit patch and + split the patch per repo + """ + logger.debug("action_splitpatch") + if len(args) > 1: + commit = args[1] + else: + commit = "HEAD" + patchdir = "splitpatch-%s" % commit + if not os.path.exists(patchdir): + os.mkdir(patchdir) + + # filerange_root is for the repo whose dest_dir is root "." + # and it should be specified by excluding all other repo dest dir + # like "-x repo1 -x repo2 -x repo3 ..." + filerange_root = "" + for name in conf.repos: + dest_dir = conf.repos[name]['dest_dir'] + if dest_dir != ".": + filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir) + + for name in conf.repos: + dest_dir = conf.repos[name]['dest_dir'] + patch_filename = "%s/%s.patch" % (patchdir, name) + if dest_dir == ".": + cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename) + else: + cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename) + runcmd(cmd) + # Detect empty patches (including those produced by filterdiff above + # that contain only preamble text) + if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "": + os.remove(patch_filename) + logger.info("(skipping %s - no changes)", name) + else: + logger.info(patch_filename) + +def action_error(conf, args): + logger.info("invalid action %s" % args[0]) + +actions = { + "init": action_init, + "update": action_update, + "pull": action_pull, + "splitpatch": action_splitpatch, +} + +def main(): + parser = optparse.OptionParser( + version = "Combo Layer Repo Tool version %s" % __version__, + usage = """%prog [options] action + +Create and update a combination layer repository from multiple component repositories. + +Action: + init initialise the combo layer repo + update [components] get patches from component repos and apply them to the combo repo + pull [components] just pull component repos only + splitpatch [commit] generate commit patch and split per component, default commit is HEAD""") + + parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).", + action = "store", dest = "conffile", default = "conf/combo-layer.conf") + + parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches", + action = "store_true", dest = "interactive", default = False) + + parser.add_option("-D", "--debug", help = "output debug information", + action = "store_true", dest = "debug", default = False) + + parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update", + action = "store_true", dest = "nopull", default = False) + + parser.add_option("--hard-reset", + help = "instead of pull do fetch and hard-reset in component repos", + action = "store_true", dest = "hard_reset", default = False) + + parser.add_option("-H", "--history", help = "import full history of components during init", + action = "store_true", default = False) + + options, args = parser.parse_args(sys.argv) + + # Dispatch to action handler + if len(args) == 1: + logger.error("No action specified, exiting") + parser.print_help() + elif args[1] not in actions: + logger.error("Unsupported action %s, exiting\n" % (args[1])) + parser.print_help() + elif not os.path.exists(options.conffile): + logger.error("No valid config file, exiting\n") + parser.print_help() + else: + if options.debug: + logger.setLevel(logging.DEBUG) + confdata = Configuration(options) + initmode = (args[1] == 'init') + confdata.sanity_check(initmode) + actions.get(args[1], action_error)(confdata, args[1:]) + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/scripts/combo-layer-hook-default.sh b/scripts/combo-layer-hook-default.sh new file mode 100755 index 0000000..1e3a3b9 --- /dev/null +++ b/scripts/combo-layer-hook-default.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Hook to add source component/revision info to commit message +# Parameter: +# $1 patch-file +# $2 revision +# $3 reponame + +patchfile=$1 +rev=$2 +reponame=$3 + +sed -i -e "0,/^Subject:/s#^Subject: \[PATCH\] \($reponame: \)*\(.*\)#Subject: \[PATCH\] $reponame: \2#" $patchfile +if grep -q '^Signed-off-by:' $patchfile; then + # Insert before Signed-off-by. + sed -i -e "0,/^Signed-off-by:/s#\(^Signed-off-by:.*\)#\(From $reponame rev: $rev\)\n\n\1#" $patchfile +else + # Insert before final --- separator, with extra blank lines removed. + perl -e "\$_ = join('', <>); s/^(.*\S[ \t]*)(\n|\n\s*\n)---\n/\$1\n\nFrom $reponame rev: $rev\n---\n/s; print;" $patchfile >$patchfile.tmp + mv $patchfile.tmp $patchfile +fi diff --git a/scripts/combo-layer.conf.example b/scripts/combo-layer.conf.example new file mode 100644 index 0000000..90e2b58 --- /dev/null +++ b/scripts/combo-layer.conf.example @@ -0,0 +1,93 @@ +# combo-layer example configuration file + +# Default values for all sections. +[DEFAULT] + +# Add 'Signed-off-by' to all commits that get imported automatically. +signoff = True + +# component name +[bitbake] + +# Override signedoff default above (not very useful, but possible). +signoff = False + +# mandatory options +# git upstream uri +src_uri = git://git.openembedded.org/bitbake + +# the directory to clone the component repo +local_repo_dir = /home/kyu3/src/test/bitbake + +# the relative dir within the combo repo to put the component files +# use "." if the files should be in the root dir +dest_dir = bitbake + +# the last update revision. +# "init" will set this to the latest revision automatically, however if it +# is empty when "update" is run, the tool will start from the first commit. +# Note that this value will get updated by "update" if the component repo's +# latest revision changed and the operation completes successfully. +last_revision = + +# optional options: + +# branch: specify the branch in the component repo to pull from +# (master if not specified) + +# file_filter: only include the specified file(s) +# file_filter = [path] [path] ... +# example: +# file_filter = src/ : only include the subdir src +# file_filter = src/*.c : only include the src *.c file +# file_filter = src/main.c src/Makefile.am : only include these two files + +# file_exclude: filter out these file(s) +# file_exclude = [path] [path] ... +# +# Each entry must match a file name. In contrast do file_filter, matching +# a directory has no effect. To achieve that, use append a * wildcard +# at the end. +# +# Wildcards are applied to the complete path and also match slashes. +# +# example: +# file_exclude = src/foobar/* : exclude everything under src/foobar +# file_exclude = src/main.c : filter out main.c after including it with file_filter = src/*.c +# file_exclude = *~ : exclude backup files + +# hook: if provided, the tool will call the hook to process the generated +# patch from upstream, and then apply the modified patch to the combo +# repo. +# the hook script is called as follows: ./hook patchpath revision reponame +# example: +# hook = combo-layer-hook-default.sh + +# since_revision: +# since_revision = release-1-2 +# since_revision = 12345 abcdf +# +# If provided, truncate imported history during "combo-layer --history +# init" at the specified revision(s). More than one can be specified +# to cut off multiple component branches. +# +# The specified commits themselves do not get imported. Instead, an +# artificial commit with "unknown" author is created with a content +# that matches the original commit. + +[oe-core] +src_uri = git://git.openembedded.org/openembedded-core +local_repo_dir = /home/kyu3/src/test/oecore +dest_dir = . +last_revision = +since_revision = some-tag-or-commit-on-master-branch + +# It is also possible to embed python code in the config values. Similar +# to bitbake it considers every value starting with @ to be a python +# script. +# e.g. local_repo_dir could easily be configured using an environment +# variable: +# +# [bitbake] +# local_repo_dir = @os.getenv("LOCAL_REPO_DIR") + "/bitbake" +# diff --git a/scripts/contrib/bb-perf/bb-matrix-plot.sh b/scripts/contrib/bb-perf/bb-matrix-plot.sh new file mode 100755 index 0000000..136a255 --- /dev/null +++ b/scripts/contrib/bb-perf/bb-matrix-plot.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# This script operates on the .dat file generated by bb-matrix.sh. It tolerates +# the header by skipping the first line, but error messages and bad data records +# need to be removed first. It will generate three views of the plot, and leave +# an interactive view open for further analysis. +# +# AUTHORS +# Darren Hart +# + +# Setup the defaults +DATFILE="bb-matrix.dat" +XLABEL="BB_NUMBER_THREADS" +YLABEL="PARALLEL_MAKE" +FIELD=3 +DEF_TITLE="Elapsed Time (seconds)" +PM3D_FRAGMENT="unset surface; set pm3d at s hidden3d 100" +SIZE="640,480" + +function usage { +CMD=$(basename $0) +cat < +# + +# The following ranges are appropriate for a 4 core system with 8 logical units +# Use leading 0s to ensure all digits are the same string length, this results +# in nice log file names and columnar dat files. +BB_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16" +PM_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16" + +DATADIR="bb-matrix-$$" +BB_CMD="bitbake core-image-minimal" +RUNTIME_LOG="$DATADIR/bb-matrix.dat" + +# See TIME(1) for a description of the time format parameters +# The following all report 0: W K r s t w +TIME_STR="%e %S %U %P %c %w %R %F %M %x" + +# Prepare the DATADIR +mkdir $DATADIR +if [ $? -ne 0 ]; then + echo "Failed to create $DATADIR." + exit 1 +fi + +# Add a simple header +echo "BB PM $TIME_STR" > $RUNTIME_LOG +for BB in $BB_RANGE; do + for PM in $PM_RANGE; do + RUNDIR="$DATADIR/$BB-$PM-build" + mkdir $RUNDIR + BB_LOG=$RUNDIR/$BB-$PM-bitbake.log + date + echo "BB=$BB PM=$PM Logging to $BB_LOG" + + echo -n " Preparing the work directory... " + rm -rf pseudodone tmp sstate-cache tmp-eglibc &> /dev/null + echo "done" + + # Export the variables under test and run the bitbake command + # Strip any leading zeroes before passing to bitbake + export BB_NUMBER_THREADS=$(echo $BB | sed 's/^0*//') + export PARALLEL_MAKE="-j $(echo $PM | sed 's/^0*//')" + /usr/bin/time -f "$BB $PM $TIME_STR" -a -o $RUNTIME_LOG $BB_CMD &> $BB_LOG + + echo " $(tail -n1 $RUNTIME_LOG)" + cp -a tmp/buildstats $RUNDIR/$BB-$PM-buildstats + done +done diff --git a/scripts/contrib/bb-perf/buildstats.sh b/scripts/contrib/bb-perf/buildstats.sh new file mode 100755 index 0000000..96158a9 --- /dev/null +++ b/scripts/contrib/bb-perf/buildstats.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# Given a 'buildstats' path (created by bitbake when setting +# USER_CLASSES ?= "buildstats" on local.conf) and task names, outputs +# ' ' for all recipes. Elapsed times are in +# seconds, and task should be given without the 'do_' prefix. +# +# Some useful pipelines +# +# 1. Tasks with largest elapsed times +# $ buildstats.sh -b | sort -k3 -n -r | head +# +# 2. Min, max, sum per task (in needs GNU datamash) +# $ buildstats.sh -b | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r +# +# AUTHORS +# Leonardo Sandoval +# +BS_DIR="tmp/buildstats" +TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" + +function usage { +CMD=$(basename $0) +cat <, 2010 + + +import sys +import getopt +import os +import os.path +import re + +def usage(): + print 'Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]) + print ' -d FILENAME documentation file to search' + print ' -h, --help display this help and exit' + print ' -m METADIR meta directory to search for recipes' + print ' -t FILENAME documentation config file (for doc tags)' + print ' -T Only display variables with doc tags (requires -t)' + +def recipe_bbvars(recipe): + ''' Return a unique set of every bbvar encountered in the recipe ''' + prog = re.compile("[A-Z_]+") + vset = set() + try: + r = open(recipe) + except IOError as (errno, strerror): + print 'WARNING: Failed to open recipe ', recipe + print strerror + + for line in r: + # Strip any comments from the line + line = line.rsplit('#')[0] + vset = vset.union(set(prog.findall(line))) + r.close() + + bbvars = {} + for v in vset: + bbvars[v] = 1 + + return bbvars + +def collect_bbvars(metadir): + ''' Walk the metadir and collect the bbvars from each recipe found ''' + bbvars = {} + for root,dirs,files in os.walk(metadir): + for name in files: + if name.find(".bb") >= 0: + for key in recipe_bbvars(os.path.join(root,name)).iterkeys(): + if bbvars.has_key(key): + bbvars[key] = bbvars[key] + 1 + else: + bbvars[key] = 1 + return bbvars + +def bbvar_is_documented(var, docfiles): + prog = re.compile(".*($|[^A-Z_])%s([^A-Z_]|$)" % (var)) + for doc in docfiles: + try: + f = open(doc) + except IOError as (errno, strerror): + print 'WARNING: Failed to open doc ', doc + print strerror + for line in f: + if prog.match(line): + return True + f.close() + return False + +def bbvar_doctag(var, docconf): + prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var)) + if docconf == "": + return "?" + + try: + f = open(docconf) + except IOError as (errno, strerror): + return strerror + + for line in f: + m = prog.search(line) + if m: + return m.group(1) + + f.close() + return "" + +def main(): + docfiles = [] + metadirs = [] + bbvars = {} + undocumented = [] + docconf = "" + onlydoctags = False + + # Collect and validate input + try: + opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"]) + except getopt.GetoptError, err: + print '%s' % str(err) + usage() + sys.exit(2) + + for o, a in opts: + if o in ('-h', '--help'): + usage() + sys.exit(0) + elif o == '-d': + if os.path.isfile(a): + docfiles.append(a) + else: + print 'ERROR: documentation file %s is not a regular file' % (a) + sys.exit(3) + elif o == '-m': + if os.path.isdir(a): + metadirs.append(a) + else: + print 'ERROR: meta directory %s is not a directory' % (a) + sys.exit(4) + elif o == "-t": + if os.path.isfile(a): + docconf = a + elif o == "-T": + onlydoctags = True + else: + assert False, "unhandled option" + + if len(docfiles) == 0: + print 'ERROR: no docfile specified' + usage() + sys.exit(5) + + if len(metadirs) == 0: + print 'ERROR: no metadir specified' + usage() + sys.exit(6) + + if onlydoctags and docconf == "": + print 'ERROR: no docconf specified' + usage() + sys.exit(7) + + # Collect all the variable names from the recipes in the metadirs + for m in metadirs: + for key,cnt in collect_bbvars(m).iteritems(): + if bbvars.has_key(key): + bbvars[key] = bbvars[key] + cnt + else: + bbvars[key] = cnt + + # Check each var for documentation + varlen = 0 + for v in bbvars.iterkeys(): + if len(v) > varlen: + varlen = len(v) + if not bbvar_is_documented(v, docfiles): + undocumented.append(v) + undocumented.sort() + varlen = varlen + 1 + + # Report all undocumented variables + print 'Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)) + header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7)) + print header + print str("").ljust(len(header), '=') + for v in undocumented: + doctag = bbvar_doctag(v, docconf) + if not onlydoctags or not doctag == "": + print '%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag) + + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/build-perf-test.sh b/scripts/contrib/build-perf-test.sh new file mode 100755 index 0000000..7d99228 --- /dev/null +++ b/scripts/contrib/build-perf-test.sh @@ -0,0 +1,400 @@ +#!/bin/bash +# +# This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size) +# +# Build performance test script +# +# Copyright 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# +# AUTHORS: +# Stefan Stanacar + + +ME=$(basename $0) + +# +# usage and setup +# + +usage () { +cat << EOT +Usage: $ME [-h] + $ME [-c ] [-v] [-m ] [-j ] [-t ] [-i ] [-d ] +Options: + -h + Display this help and exit. + -c + git checkout before anything else + -v + Show bitbake output, don't redirect it to a log. + -m + Value for MACHINE. Default is qemux86. + -j + Value for PARALLEL_MAKE. Default is 8. + -t + Value for BB_NUMBER_THREADS. Default is 8. + -i + Instead of timing against core-image-sato, use + -d + Use as DL_DIR + -p + Cherry pick githash onto the commit + +Note: current working directory must be inside a poky git clone. + +EOT +} + + +if clonedir=$(git rev-parse --show-toplevel); then + cd $clonedir +else + echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME" + exit 1 +fi + +IMAGE="core-image-sato" +verbose=0 +dldir= +commit= +pmake= +cherrypicks= +while getopts "hvc:m:j:t:i:d:p:" opt; do + case $opt in + h) usage + exit 0 + ;; + v) verbose=1 + ;; + c) commit=$OPTARG + ;; + m) export MACHINE=$OPTARG + ;; + j) pmake=$OPTARG + ;; + t) export BB_NUMBER_THREADS=$OPTARG + ;; + i) IMAGE=$OPTARG + ;; + d) dldir=$OPTARG + ;; + p) cherrypicks="$cherrypicks $OPTARG" + ;; + *) usage + exit 1 + ;; + esac +done + + +#drop cached credentials and test for sudo access without a password +sudo -k -n ls > /dev/null 2>&1 +reqpass=$? +if [ $reqpass -ne 0 ]; then + echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)" + read -s -p "Please enter your sudo password: " pass + echo +fi + +if [ -n "$commit" ]; then + echo "git checkout -f $commit" + git pull > /dev/null 2>&1 + git checkout -f $commit || exit 1 + git pull > /dev/null 2>&1 +fi + +if [ -n "$cherrypicks" ]; then + for c in $cherrypicks; do + git cherry-pick $c + done +fi + +rev=$(git rev-parse --short HEAD) || exit 1 +OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`" +BUILDDIR="$OUTDIR/build" +resultsfile="$OUTDIR/results.log" +cmdoutput="$OUTDIR/commands.log" +myoutput="$OUTDIR/output.log" +globalres="$clonedir/build-perf-test/globalres.log" + +mkdir -p $OUTDIR || exit 1 + +log () { + local msg="$1" + echo "`date`: $msg" | tee -a $myoutput +} + + +# +# Config stuff +# + +branch=`git branch 2>&1 | grep "^* " | tr -d "* "` +gitcommit=$(git rev-parse HEAD) || exit 1 +log "Running on $branch:$gitcommit" + +source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1 +cd $OUTDIR/build + +[ -n "$MACHINE" ] || export MACHINE="qemux86" +[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8" + +if [ -n "$pmake" ]; then + export PARALLEL_MAKE="-j $pmake" +else + export PARALLEL_MAKE="-j 8" +fi + +if [ -n "$dldir" ]; then + echo "DL_DIR = \"$dldir\"" >> conf/local.conf +else + echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf +fi + +# Sometimes I've noticed big differences in timings for the same commit, on the same machine +# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy) +echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf + + +# +# Functions +# + +declare -a TIMES +time_count=0 +declare -a SIZES +size_count=0 + +time_cmd () { + log " Timing: $*" + + if [ $verbose -eq 0 ]; then + /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput + else + /usr/bin/time -v -o $resultsfile "$@" + fi + ret=$? + if [ $ret -eq 0 ]; then + t=`grep wall $resultsfile | sed 's/.*m:ss): //'` + log " TIME: $t" + TIMES[(( time_count++ ))]="$t" + else + log "ERROR: exit status was non-zero, will report time as 0." + TIMES[(( time_count++ ))]="0" + fi + + #time by default overwrites the output file and we want to keep the results + #it has an append option but I don't want to clobber the results in the same file + i=`ls $OUTDIR/results.log* |wc -l` + mv $resultsfile "${resultsfile}.${i}" + log "More stats can be found in ${resultsfile}.${i}" +} + +bbtime () { + time_cmd bitbake "$@" +} + +#we don't time bitbake here +bbnotime () { + local arg="$@" + log " Running: bitbake ${arg}" + if [ $verbose -eq 0 ]; then + bitbake ${arg} >> $cmdoutput + else + bitbake ${arg} + fi + ret=$? + if [ $ret -eq 0 ]; then + log " Finished bitbake ${arg}" + else + log "ERROR: exit status was non-zero. Exit.." + exit $ret + fi + +} + +do_rmtmp() { + log " Removing tmp" + rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp +} +do_rmsstate () { + log " Removing sstate-cache" + rm -rf sstate-cache +} +do_sync () { + log " Syncing and dropping caches" + sync; sync + if [ $reqpass -eq 0 ]; then + sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" + else + echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches" + echo + fi + sleep 3 +} + +write_results() { + echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres + for i in "${TIMES[@]}"; do + echo -n "$i," >> $globalres + done + for i in "${SIZES[@]}"; do + echo -n "$i," >> $globalres + done + echo >> $globalres + sed -i '$ s/,$//' $globalres +} + +#### + +# +# Test 1 +# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work) +# Pre: Downloaded sources, no sstate +# Steps: +# Part1: +# - fetchall +# - clean build dir +# - time bitbake core-image-sato +# - collect data +# Part2: +# - bitbake virtual/kernel -c cleansstate +# - time bitbake virtual/kernel +# Part3: +# - add INHERIT to local.conf +# - clean build dir +# - build +# - report size, remove INHERIT + +test1_p1 () { + log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir" + bbnotime $IMAGE -c fetchall + do_rmtmp + do_rmsstate + do_sync + bbtime $IMAGE + s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` + SIZES[(( size_count++ ))]="$s" + log "SIZE of tmp dir is: $s" + log "Buildstats are saved in $OUTDIR/buildstats-test1" + mv tmp/buildstats $OUTDIR/buildstats-test1 +} + + +test1_p2 () { + log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel" + bbnotime virtual/kernel -c cleansstate + do_sync + bbtime virtual/kernel +} + +test1_p3 () { + log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled" + echo "INHERIT += \"rm_work\"" >> conf/local.conf + do_rmtmp + do_rmsstate + do_sync + bbtime $IMAGE + sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf + s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` + SIZES[(( size_count++ ))]="$s" + log "SIZE of tmp dir is: $s" + log "Buildstats are saved in $OUTDIR/buildstats-test13" + mv tmp/buildstats $OUTDIR/buildstats-test13 +} + + +# +# Test 2 +# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir +# Pre: populated sstate cache + +test2 () { + # Assuming test 1 has run + log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate" + do_rmtmp + do_sync + bbtime $IMAGE -c rootfs +} + + +# Test 3 +# parsing time metrics +# +# Start with +# i) "rm -rf tmp/cache; time bitbake -p" +# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p" +# iii) "time bitbake -p" + + +test3 () { + log "Running Test 3: Parsing time metrics (bitbake -p)" + log " Removing tmp/cache && cache" + rm -rf tmp/cache cache + bbtime -p + log " Removing tmp/cache/default-glibc/" + rm -rf tmp/cache/default-glibc/ + bbtime -p + bbtime -p +} + +# +# Test 4 - eSDK +# Measure: eSDK size and installation time +test4 () { + log "Running Test 4: eSDK size and installation time" + bbnotime $IMAGE -c do_populate_sdk_ext + + esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh) + + if [ ${#esdk_installer[*]} -eq 1 ]; then + s=$((`stat -c %s "$esdk_installer"` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Download SIZE of eSDK is: $s kB" + + do_sync + time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy" + + s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Install SIZE of eSDK is: $s kB" + else + log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0." + SIZES[(( size_count++ ))]="0" + TIMES[(( time_count++ ))]="0" + fi + +} + + +# RUN! + +test1_p1 +test1_p2 +test1_p3 +test2 +test3 +test4 + +# if we got til here write to global results +write_results + +log "All done, cleaning up..." + +do_rmtmp +do_rmsstate diff --git a/scripts/contrib/ddimage b/scripts/contrib/ddimage new file mode 100755 index 0000000..a503f11 --- /dev/null +++ b/scripts/contrib/ddimage @@ -0,0 +1,104 @@ +#!/bin/sh + +# Default to avoiding the first two disks on typical Linux and Mac OS installs +# Better safe than sorry :-) +BLACKLIST_DEVICES="/dev/sda /dev/sdb /dev/disk1 /dev/disk2" + +# 1MB blocksize +BLOCKSIZE=1048576 + +usage() { + echo "Usage: $(basename $0) IMAGE DEVICE" +} + +image_details() { + IMG=$1 + echo "Image details" + echo "=============" + echo " image: $(basename $IMG)" + # stat format is different on Mac OS and Linux + if [ "$(uname)" = "Darwin" ]; then + echo " size: $(stat -L -f '%z bytes' $IMG)" + echo " modified: $(stat -L -f '%Sm' $IMG)" + else + echo " size: $(stat -L -c '%s bytes' $IMG)" + echo " modified: $(stat -L -c '%y' $IMG)" + fi + echo " type: $(file -L -b $IMG)" + echo "" +} + +device_details() { + DEV=$1 + BLOCK_SIZE=512 + + echo "Device details" + echo "==============" + + # Collect disk info using diskutil on Mac OS + if [ "$(uname)" = "Darwin" ]; then + diskutil info $DEVICE | egrep "(Device Node|Media Name|Total Size)" + return + fi + + # Default / Linux information collection + echo " device: $DEVICE" + if [ -f "/sys/class/block/$DEV/device/vendor" ]; then + echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" + else + echo " vendor: UNKOWN" + fi + if [ -f "/sys/class/block/$DEV/device/model" ]; then + echo " model: $(cat /sys/class/block/$DEV/device/model)" + else + echo " model: UNKNOWN" + fi + if [ -f "/sys/class/block/$DEV/size" ]; then + echo " size: $(($(cat /sys/class/block/$DEV/size) * $BLOCK_SIZE)) bytes" + else + echo " size: UNKNOWN" + fi + echo "" +} + +if [ $# -ne 2 ]; then + usage + exit 1 +fi + +IMAGE=$1 +DEVICE=$2 + +if [ ! -e "$IMAGE" ]; then + echo "ERROR: Image $IMAGE does not exist" + usage + exit 1 +fi + + +for i in ${BLACKLIST_DEVICES}; do + if [ "$i" = "$DEVICE" ]; then + echo "ERROR: Device $DEVICE is blacklisted" + exit 1 + fi +done + +if [ ! -w "$DEVICE" ]; then + echo "ERROR: Device $DEVICE does not exist or is not writable" + usage + exit 1 +fi + +image_details $IMAGE +device_details $(basename $DEVICE) + +printf "Write $IMAGE to $DEVICE [y/N]? " +read RESPONSE +if [ "$RESPONSE" != "y" ]; then + echo "Write aborted" + exit 0 +fi + +echo "Writing image..." +dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" +sync diff --git a/scripts/contrib/devtool-stress.py b/scripts/contrib/devtool-stress.py new file mode 100755 index 0000000..8cf92ca --- /dev/null +++ b/scripts/contrib/devtool-stress.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +# devtool stress tester +# +# Written by: Paul Eggleton +# +# Copyright 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import sys +import os +import os.path +import subprocess +import re +import argparse +import logging +import tempfile +import shutil +import signal +import fnmatch + +scripts_lib_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'lib')) +sys.path.insert(0, scripts_lib_path) +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('devtool-stress') + +def select_recipes(args): + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(False) + + pkg_pn = tinfoil.cooker.recipecache.pkg_pn + (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecache, pkg_pn) + + skip_classes = args.skip_classes.split(',') + + recipelist = [] + for pn in sorted(pkg_pn): + pref = preferred_versions[pn] + inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecache.inherits[pref[1]]] + for cls in skip_classes: + if cls in inherits: + break + else: + recipelist.append(pn) + + tinfoil.shutdown() + + resume_from = args.resume_from + if resume_from: + if not resume_from in recipelist: + print('%s is not a testable recipe' % resume_from) + return 1 + if args.only: + only = args.only.split(',') + for onlyitem in only: + for pn in recipelist: + if fnmatch.fnmatch(pn, onlyitem): + break + else: + print('%s does not match any testable recipe' % onlyitem) + return 1 + else: + only = None + if args.skip: + skip = args.skip.split(',') + else: + skip = [] + + recipes = [] + for pn in recipelist: + if resume_from: + if pn == resume_from: + resume_from = None + else: + continue + + if args.only: + for item in only: + if fnmatch.fnmatch(pn, item): + break + else: + continue + + skipit = False + for item in skip: + if fnmatch.fnmatch(pn, item): + skipit = True + if skipit: + continue + + recipes.append(pn) + + return recipes + + +def stress_extract(args): + import bb.process + + recipes = select_recipes(args) + + failures = 0 + tmpdir = tempfile.mkdtemp() + os.setpgrp() + try: + for pn in recipes: + sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) + sys.stdout.flush() + failed = False + + srctree = os.path.join(tmpdir, pn) + try: + bb.process.run('devtool extract %s %s' % (pn, srctree)) + except bb.process.CmdError as exc: + failed = True + with open('stress_%s_extract.log' % pn, 'w') as f: + f.write(str(exc)) + + if os.path.exists(srctree): + shutil.rmtree(srctree) + + if failed: + print('failed') + failures += 1 + else: + print('ok') + except KeyboardInterrupt: + # We want any child processes killed. This is crude, but effective. + os.killpg(0, signal.SIGTERM) + + if failures: + return 1 + else: + return 0 + + +def stress_modify(args): + import bb.process + + recipes = select_recipes(args) + + failures = 0 + tmpdir = tempfile.mkdtemp() + os.setpgrp() + try: + for pn in recipes: + sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) + sys.stdout.flush() + failed = False + reset = True + + srctree = os.path.join(tmpdir, pn) + try: + bb.process.run('devtool modify -x %s %s' % (pn, srctree)) + except bb.process.CmdError as exc: + with open('stress_%s_modify.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'modify' + reset = False + + if not failed: + try: + bb.process.run('bitbake -c install %s' % pn) + except bb.process.CmdError as exc: + with open('stress_%s_install.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'build' + if reset: + try: + bb.process.run('devtool reset %s' % pn) + except bb.process.CmdError as exc: + print('devtool reset failed: %s' % str(exc)) + break + + if os.path.exists(srctree): + shutil.rmtree(srctree) + + if failed: + print('failed (%s)' % failed) + failures += 1 + else: + print('ok') + except KeyboardInterrupt: + # We want any child processes killed. This is crude, but effective. + os.killpg(0, signal.SIGTERM) + + if failures: + return 1 + else: + return 0 + + +def main(): + parser = argparse_oe.ArgumentParser(description="devtool stress tester", + epilog="Use %(prog)s --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-r', '--resume-from', help='Resume from specified recipe', metavar='PN') + parser.add_argument('-o', '--only', help='Only test specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') + parser.add_argument('-s', '--skip', help='Skip specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') + parser.add_argument('-c', '--skip-classes', help='Skip recipes inheriting specified classes (comma-separated) - default %(default)s', metavar='CLASSLIST', default='native,nativesdk,cross,cross-canadian,image,populate_sdk,meta,packagegroup') + subparsers = parser.add_subparsers(title='subcommands', metavar='') + + parser_modify = subparsers.add_parser('modify', + help='Run "devtool modify" followed by a build with bitbake on matching recipes', + description='Runs "devtool modify" followed by a build with bitbake on matching recipes') + parser_modify.set_defaults(func=stress_modify) + + parser_extract = subparsers.add_parser('extract', + help='Run "devtool extract" on matching recipes', + description='Runs "devtool extract" on matching recipes') + parser_extract.set_defaults(func=stress_extract) + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + return 1 + logger.debug('Found bitbake path: %s' % bitbakepath) + + ret = args.func(args) + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/dialog-power-control b/scripts/contrib/dialog-power-control new file mode 100755 index 0000000..7550ea5 --- /dev/null +++ b/scripts/contrib/dialog-power-control @@ -0,0 +1,53 @@ +#!/bin/sh +# +# Simple script to show a manual power prompt for when you want to use +# automated hardware testing with testimage.bbclass but you don't have a +# web-enabled power strip or similar to do the power on/off/cycle. +# +# You can enable it by enabling testimage (see the Yocto Project +# Development manual "Performing Automated Runtime Testing" section) +# and setting the following in your local.conf: +# +# TEST_POWERCONTROL_CMD = "${COREBASE}/scripts/contrib/dialog-power-control" +# + +PROMPT="" +while true; do + case $1 in + on) + PROMPT="Please turn device power on";; + off) + PROMPT="Please turn device power off";; + cycle) + PROMPT="Please click Done, then turn the device power off then on";; + "") + break;; + esac + shift +done + +if [ "$PROMPT" = "" ] ; then + echo "ERROR: no power action specified on command line" + exit 2 +fi + +if [ "`which kdialog 2>/dev/null`" != "" ] ; then + DIALOGUTIL="kdialog" +elif [ "`which zenity 2>/dev/null`" != "" ] ; then + DIALOGUTIL="zenity" +else + echo "ERROR: couldn't find program to display a message, install kdialog or zenity" + exit 3 +fi + +if [ "$DIALOGUTIL" = "kdialog" ] ; then + kdialog --yesno "$PROMPT" --title "TestImage Power Control" --yes-label "Done" --no-label "Cancel test" +elif [ "$DIALOGUTIL" = "zenity" ] ; then + zenity --question --text="$PROMPT" --title="TestImage Power Control" --ok-label="Done" --cancel-label="Cancel test" +fi + +if [ "$?" != "0" ] ; then + echo "User cancelled test at power prompt" + exit 1 +fi + diff --git a/scripts/contrib/documentation-audit.sh b/scripts/contrib/documentation-audit.sh new file mode 100755 index 0000000..2144aac --- /dev/null +++ b/scripts/contrib/documentation-audit.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Perform an audit of which packages provide documentation and which +# are missing -doc packages. +# +# Setup requirements: be sure to be building for MACHINE=qemux86. Run +# this script after source'ing the build environment script, so you're +# running it from build/ directory. +# +# Maintainer: Scott Garman + +REPORT_DOC_SIMPLE="documentation_exists.txt" +REPORT_DOC_DETAIL="documentation_exists_detail.txt" +REPORT_MISSING_SIMPLE="documentation_missing.txt" +REPORT_MISSING_DETAIL="documentation_missing_detail.txt" +REPORT_BUILD_ERRORS="build_errors.txt" + +rm -rf $REPORT_DOC_SIMPLE $REPORT_DOC_DETAIL $REPORT_MISSING_SIMPLE $REPORT_MISSING_DETAIL + +BITBAKE=`which bitbake` +if [ -z "$BITBAKE" ]; then + echo "Error: bitbake command not found." + echo "Did you forget to source the build environment script?" + exit 1 +fi + +echo "REMINDER: you need to build for MACHINE=qemux86 or you won't get useful results" +echo "REMINDER: you need to set LICENSE_FLAGS_WHITELIST appropriately in local.conf or " +echo " you'll get false positives. For example, LICENSE_FLAGS_WHITELIST = \"Commercial\"" + +for pkg in `bitbake -s | awk '{ print \$1 }'`; do + if [[ "$pkg" == "Loading" || "$pkg" == "Loaded" || + "$pkg" == "Recipe" || + "$pkg" == "Parsing" || "$pkg" == "Package" || + "$pkg" == "NOTE:" || "$pkg" == "WARNING:" || + "$pkg" == "done." || "$pkg" == "===========" ]] + then + # Skip initial bitbake output + continue + fi + if [[ "$pkg" =~ -native$ || "$pkg" =~ -nativesdk$ || + "$pkg" =~ -cross-canadian ]]; then + # Skip native/nativesdk/cross-canadian recipes + continue + fi + if [[ "$pkg" =~ ^meta- || "$pkg" =~ ^packagegroup- || "$pkg" =~ -image ]]; then + # Skip meta, task and image recipes + continue + fi + if [[ "$pkg" =~ ^glibc- || "$pkg" =~ ^libiconv$ || + "$pkg" =~ -toolchain$ || "$pkg" =~ ^package-index$ || + "$pkg" =~ ^linux- || "$pkg" =~ ^adt-installer$ || + "$pkg" =~ ^eds-tools$ || "$pkg" =~ ^external-python-tarball$ || + "$pkg" =~ ^qt4-embedded$ || "$pkg" =~ ^qt-mobility ]]; then + # Skip glibc, libiconv, -toolchain, and other recipes known + # to cause build conflicts or trigger false positives. + continue + fi + + echo "Building package $pkg..." + bitbake $pkg > /dev/null + if [ $? -ne 0 ]; then + echo "There was an error building package $pkg" >> "$REPORT_MISSING_DETAIL" + echo "$pkg" >> $REPORT_BUILD_ERRORS + + # Do not skip the remaining tests, as sometimes the + # exit status is 1 due to QA errors, and we can still + # perform the -doc checks. + fi + + echo "$pkg built successfully, checking for a documentation package..." + WORKDIR=`bitbake -e $pkg | grep ^WORKDIR | awk -F '=' '{ print \$2 }' | awk -F '"' '{ print \$2 }'` + FIND_DOC_PKG=`find $WORKDIR/packages-split/*-doc -maxdepth 0 -type d` + if [ -z "$FIND_DOC_PKG" ]; then + # No -doc package was generated: + echo "No -doc package: $pkg" >> "$REPORT_MISSING_DETAIL" + echo "$pkg" >> $REPORT_MISSING_SIMPLE + continue + fi + + FIND_DOC_FILES=`find $FIND_DOC_PKG -type f` + if [ -z "$FIND_DOC_FILES" ]; then + # No files shipped with the -doc package: + echo "No files shipped with the -doc package: $pkg" >> "$REPORT_MISSING_DETAIL" + echo "$pkg" >> $REPORT_MISSING_SIMPLE + continue + fi + + echo "Documentation shipped with $pkg:" >> "$REPORT_DOC_DETAIL" + echo "$FIND_DOC_FILES" >> "$REPORT_DOC_DETAIL" + echo "" >> "$REPORT_DOC_DETAIL" + + echo "$pkg" >> "$REPORT_DOC_SIMPLE" +done diff --git a/scripts/contrib/graph-tool b/scripts/contrib/graph-tool new file mode 100755 index 0000000..6dc7d33 --- /dev/null +++ b/scripts/contrib/graph-tool @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +# Simple graph query utility +# useful for getting answers from .dot files produced by bitbake -g +# +# Written by: Paul Eggleton +# +# Copyright 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import sys + +def get_path_networkx(dotfile, fromnode, tonode): + try: + import networkx + except ImportError: + print('ERROR: Please install the networkx python module') + sys.exit(1) + + graph = networkx.DiGraph(networkx.read_dot(dotfile)) + + def node_missing(node): + import difflib + close_matches = difflib.get_close_matches(node, graph.nodes(), cutoff=0.7) + if close_matches: + print('ERROR: no node "%s" in graph. Close matches:\n %s' % (node, '\n '.join(close_matches))) + sys.exit(1) + + if not fromnode in graph: + node_missing(fromnode) + if not tonode in graph: + node_missing(tonode) + return networkx.all_simple_paths(graph, source=fromnode, target=tonode) + + +def find_paths(args, usage): + if len(args) < 3: + usage() + sys.exit(1) + + fromnode = args[1] + tonode = args[2] + paths = list(get_path_networkx(args[0], fromnode, tonode)) + if paths: + for path in paths: + print ' -> '.join(path) + else: + print("ERROR: no path from %s to %s in graph" % (fromnode, tonode)) + sys.exit(1) + +def main(): + import optparse + parser = optparse.OptionParser( + usage = '''%prog [options] + +Available commands: + find-paths + Find all of the paths between two nodes in a dot graph''') + + #parser.add_option("-d", "--debug", + # help = "Report all SRCREV values, not just ones where AUTOREV has been used", + # action="store_true", dest="debug", default=False) + + options, args = parser.parse_args(sys.argv) + args = args[1:] + + if len(args) < 1: + parser.print_help() + sys.exit(1) + + if args[0] == "find-paths": + find_paths(args[1:], parser.print_help) + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/list-packageconfig-flags.py b/scripts/contrib/list-packageconfig-flags.py new file mode 100755 index 0000000..2f3b8b0 --- /dev/null +++ b/scripts/contrib/list-packageconfig-flags.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation. +# +# Copyright (C) 2013 Wind River Systems, Inc. +# Copyright (C) 2014 Intel Corporation +# +# - list available recipes which have PACKAGECONFIG flags +# - list available PACKAGECONFIG flags and all affected recipes +# - list all recipes and PACKAGECONFIG information + +import sys +import optparse +import os + + +scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) +lib_path = os.path.abspath(scripts_path + '/../lib') +sys.path = sys.path + [lib_path] + +import scriptpath + +# For importing the following modules +bitbakepath = scriptpath.add_bitbake_lib_path() +if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") + sys.exit(1) + +import bb.cache +import bb.cooker +import bb.providers +import bb.tinfoil + +def get_fnlist(bbhandler, pkg_pn, preferred): + ''' Get all recipe file names ''' + if preferred: + (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecache, pkg_pn) + + fn_list = [] + for pn in sorted(pkg_pn): + if preferred: + fn_list.append(preferred_versions[pn][1]) + else: + fn_list.extend(pkg_pn[pn]) + + return fn_list + +def get_recipesdata(bbhandler, preferred): + ''' Get data of all available recipes which have PACKAGECONFIG flags ''' + pkg_pn = bbhandler.cooker.recipecache.pkg_pn + + data_dict = {} + for fn in get_fnlist(bbhandler, pkg_pn, preferred): + data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data) + flags = data.getVarFlags("PACKAGECONFIG") + flags.pop('doc', None) + if flags: + data_dict[fn] = data + + return data_dict + +def collect_pkgs(data_dict): + ''' Collect available pkgs in which have PACKAGECONFIG flags ''' + # pkg_dict = {'pkg1': ['flag1', 'flag2',...]} + pkg_dict = {} + for fn in data_dict: + pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG") + pkgconfigflags.pop('doc', None) + pkgname = data_dict[fn].getVar("P", True) + pkg_dict[pkgname] = sorted(pkgconfigflags.keys()) + + return pkg_dict + +def collect_flags(pkg_dict): + ''' Collect available PACKAGECONFIG flags and all affected pkgs ''' + # flag_dict = {'flag': ['pkg1', 'pkg2',...]} + flag_dict = {} + for pkgname, flaglist in pkg_dict.iteritems(): + for flag in flaglist: + if flag in flag_dict: + flag_dict[flag].append(pkgname) + else: + flag_dict[flag] = [pkgname] + + return flag_dict + +def display_pkgs(pkg_dict): + ''' Display available pkgs which have PACKAGECONFIG flags ''' + pkgname_len = len("RECIPE NAME") + 1 + for pkgname in pkg_dict: + if pkgname_len < len(pkgname): + pkgname_len = len(pkgname) + pkgname_len += 1 + + header = '%-*s%s' % (pkgname_len, str("RECIPE NAME"), str("PACKAGECONFIG FLAGS")) + print header + print str("").ljust(len(header), '=') + for pkgname in sorted(pkg_dict): + print('%-*s%s' % (pkgname_len, pkgname, ' '.join(pkg_dict[pkgname]))) + + +def display_flags(flag_dict): + ''' Display available PACKAGECONFIG flags and all affected pkgs ''' + flag_len = len("PACKAGECONFIG FLAG") + 5 + + header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("RECIPE NAMES")) + print header + print str("").ljust(len(header), '=') + + for flag in sorted(flag_dict): + print('%-*s%s' % (flag_len, flag, ' '.join(sorted(flag_dict[flag])))) + +def display_all(data_dict): + ''' Display all pkgs and PACKAGECONFIG information ''' + print str("").ljust(50, '=') + for fn in data_dict: + print('%s' % data_dict[fn].getVar("P", True)) + print fn + packageconfig = data_dict[fn].getVar("PACKAGECONFIG", True) or '' + if packageconfig.strip() == '': + packageconfig = 'None' + print('PACKAGECONFIG %s' % packageconfig) + + for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").iteritems(): + if flag == "doc": + continue + print('PACKAGECONFIG[%s] %s' % (flag, flag_val)) + print '' + +def main(): + pkg_dict = {} + flag_dict = {} + + # Collect and validate input + parser = optparse.OptionParser( + description = "Lists recipes and PACKAGECONFIG flags. Without -a or -f, recipes and their available PACKAGECONFIG flags are listed.", + usage = """ + %prog [options]""") + + parser.add_option("-f", "--flags", + help = "list available PACKAGECONFIG flags and affected recipes", + action="store_const", dest="listtype", const="flags", default="recipes") + parser.add_option("-a", "--all", + help = "list all recipes and PACKAGECONFIG information", + action="store_const", dest="listtype", const="all") + parser.add_option("-p", "--preferred-only", + help = "where multiple recipe versions are available, list only the preferred version", + action="store_true", dest="preferred", default=False) + + options, args = parser.parse_args(sys.argv) + + bbhandler = bb.tinfoil.Tinfoil() + bbhandler.prepare() + print("Gathering recipe data...") + data_dict = get_recipesdata(bbhandler, options.preferred) + + if options.listtype == 'flags': + pkg_dict = collect_pkgs(data_dict) + flag_dict = collect_flags(pkg_dict) + display_flags(flag_dict) + elif options.listtype == 'recipes': + pkg_dict = collect_pkgs(data_dict) + display_pkgs(pkg_dict) + elif options.listtype == 'all': + display_all(data_dict) + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/mkefidisk.sh b/scripts/contrib/mkefidisk.sh new file mode 100755 index 0000000..d8db3c0 --- /dev/null +++ b/scripts/contrib/mkefidisk.sh @@ -0,0 +1,459 @@ +#!/bin/sh +# +# Copyright (c) 2012, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +LANG=C + +# Set to 1 to enable additional output +DEBUG=0 +OUT="/dev/null" + +# +# Defaults +# +# 20 Mb for the boot partition +BOOT_SIZE=20 +# 5% for swap +SWAP_RATIO=5 + +# Cleanup after die() +cleanup() { + debug "Syncing and unmounting devices" + # Unmount anything we mounted + unmount $ROOTFS_MNT || error "Failed to unmount $ROOTFS_MNT" + unmount $BOOTFS_MNT || error "Failed to unmount $BOOTFS_MNT" + unmount $HDDIMG_ROOTFS_MNT || error "Failed to unmount $HDDIMG_ROOTFS_MNT" + unmount $HDDIMG_MNT || error "Failed to unmount $HDDIMG_MNT" + + # Remove the TMPDIR + debug "Removing temporary files" + if [ -d "$TMPDIR" ]; then + rm -rf $TMPDIR || error "Failed to remove $TMPDIR" + fi +} + +trap 'die "Signal Received, Aborting..."' HUP INT TERM + +# Logging routines +WARNINGS=0 +ERRORS=0 +CLEAR="$(tput sgr0)" +INFO="$(tput bold)" +RED="$(tput setaf 1)$(tput bold)" +GREEN="$(tput setaf 2)$(tput bold)" +YELLOW="$(tput setaf 3)$(tput bold)" +info() { + echo "${INFO}$1${CLEAR}" +} +error() { + ERRORS=$((ERRORS+1)) + echo "${RED}$1${CLEAR}" +} +warn() { + WARNINGS=$((WARNINGS+1)) + echo "${YELLOW}$1${CLEAR}" +} +success() { + echo "${GREEN}$1${CLEAR}" +} +die() { + error "$1" + cleanup + exit 1 +} +debug() { + if [ $DEBUG -eq 1 ]; then + echo "$1" + fi +} + +usage() { + echo "Usage: $(basename $0) [-v] DEVICE HDDIMG TARGET_DEVICE" + echo " -v: Verbose debug" + echo " DEVICE: The device to write the image to, e.g. /dev/sdh" + echo " HDDIMG: The hddimg file to generate the efi disk from" + echo " TARGET_DEVICE: The device the target will boot from, e.g. /dev/mmcblk0" +} + +image_details() { + IMG=$1 + info "Image details" + echo " image: $(stat --printf '%N\n' $IMG)" + echo " size: $(stat -L --printf '%s bytes\n' $IMG)" + echo " modified: $(stat -L --printf '%y\n' $IMG)" + echo " type: $(file -L -b $IMG)" + echo "" +} + +device_details() { + DEV=$1 + BLOCK_SIZE=512 + + info "Device details" + echo " device: $DEVICE" + if [ -f "/sys/class/block/$DEV/device/vendor" ]; then + echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" + else + echo " vendor: UNKOWN" + fi + if [ -f "/sys/class/block/$DEV/device/model" ]; then + echo " model: $(cat /sys/class/block/$DEV/device/model)" + else + echo " model: UNKNOWN" + fi + if [ -f "/sys/class/block/$DEV/size" ]; then + echo " size: $(($(cat /sys/class/block/$DEV/size) * $BLOCK_SIZE)) bytes" + else + echo " size: UNKNOWN" + fi + echo "" +} + +unmount_device() { + grep -q $DEVICE /proc/mounts + if [ $? -eq 0 ]; then + warn "$DEVICE listed in /proc/mounts, attempting to unmount" + umount $DEVICE* 2>/dev/null + return $? + fi + return 0 +} + +unmount() { + if [ "$1" = "" ] ; then + return 0 + fi + grep -q $1 /proc/mounts + if [ $? -eq 0 ]; then + debug "Unmounting $1" + umount $1 + return $? + fi + return 0 +} + +# +# Parse and validate arguments +# +if [ $# -lt 3 ] || [ $# -gt 4 ]; then + if [ $# -eq 1 ]; then + AVAILABLE_DISK=`lsblk | grep "disk" | cut -f 1 -d " "` + X=0 + for disk in `echo $AVAILABLE_DISK`; do + mounted=`lsblk /dev/$disk | awk {'print $7'} | sed "s/MOUNTPOINT//"` + if [ -z "$mounted" ]; then + UNMOUNTED_AVAILABLES="$UNMOUNTED_AVAILABLES /dev/$disk" + info "$X - /dev/$disk" + X=`expr $X + 1` + fi + done + if [ $X -eq 0 ]; then + die "No unmounted device found." + fi + read -p "Choose unmounted device number: " DISK_NUMBER + X=0 + for line in `echo $UNMOUNTED_AVAILABLES`; do + if [ $DISK_NUMBER -eq $X ]; then + DISK_TO_BE_FLASHED=$line + break + else + X=`expr $X + 1` + fi + done + if [ -z "$DISK_TO_BE_FLASHED" ]; then + die "Option \"$DISK_NUMBER\" is invalid. Choose a valid option" + else + if [ -z `echo $DISK_TO_BE_FLASHED | grep "mmc"` ]; then + TARGET_TO_BE_BOOT="/dev/sda" + else + TARGET_TO_BE_BOOT="/dev/mmcblk0" + fi + fi + echo "" + echo "Choose a name of the device that will be boot from" + echo -n "Recommended name is: " + info "$TARGET_TO_BE_BOOT" + read -p "Is target device okay? [y/N]: " RESPONSE + if [ "$RESPONSE" != "y" ]; then + read -p "Choose target device name: " TARGET_TO_BE_BOOT + fi + echo "" + if [ -z "$TARGET_TO_BE_BOOT" ]; then + die "Error: choose a valid target name" + fi + else + usage + exit 1 + fi +fi + +if [ "$1" = "-v" ]; then + DEBUG=1 + OUT="1" + shift +fi + +if [ -z "$AVAILABLE_DISK" ]; then + DEVICE=$1 + HDDIMG=$2 + TARGET_DEVICE=$3 +else + DEVICE=$DISK_TO_BE_FLASHED + HDDIMG=$1 + TARGET_DEVICE=$TARGET_TO_BE_BOOT +fi + +LINK=$(readlink $DEVICE) +if [ $? -eq 0 ]; then + DEVICE="$LINK" +fi + +if [ ! -w "$DEVICE" ]; then + usage + if [ ! -e "${DEVICE}" ] ; then + die "Device $DEVICE cannot be found" + else + die "Device $DEVICE is not writable (need to run under sudo?)" + fi +fi + +if [ ! -e "$HDDIMG" ]; then + usage + die "HDDIMG $HDDIMG does not exist" +fi + +# +# Ensure the hddimg is not mounted +# +unmount "$HDDIMG" || die "Failed to unmount $HDDIMG" + +# +# Check if any $DEVICE partitions are mounted +# +unmount_device || die "Failed to unmount $DEVICE" + +# +# Confirm device with user +# +image_details $HDDIMG +device_details $(basename $DEVICE) +echo -n "${INFO}Prepare EFI image on $DEVICE [y/N]?${CLEAR} " +read RESPONSE +if [ "$RESPONSE" != "y" ]; then + echo "Image creation aborted" + exit 0 +fi + + +# +# Prepare the temporary working space +# +TMPDIR=$(mktemp -d mkefidisk-XXX) || die "Failed to create temporary mounting directory." +HDDIMG_MNT=$TMPDIR/hddimg +HDDIMG_ROOTFS_MNT=$TMPDIR/hddimg_rootfs +ROOTFS_MNT=$TMPDIR/rootfs +BOOTFS_MNT=$TMPDIR/bootfs +mkdir $HDDIMG_MNT || die "Failed to create $HDDIMG_MNT" +mkdir $HDDIMG_ROOTFS_MNT || die "Failed to create $HDDIMG_ROOTFS_MNT" +mkdir $ROOTFS_MNT || die "Failed to create $ROOTFS_MNT" +mkdir $BOOTFS_MNT || die "Failed to create $BOOTFS_MNT" + + +# +# Partition $DEVICE +# +DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") +# If the device size is not reported there may not be a valid label +if [ "$DEVICE_SIZE" = "" ] ; then + parted -s $DEVICE mklabel msdos || die "Failed to create MSDOS partition table" + DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") +fi +SWAP_SIZE=$((DEVICE_SIZE*SWAP_RATIO/100)) +ROOTFS_SIZE=$((DEVICE_SIZE-BOOT_SIZE-SWAP_SIZE)) +ROOTFS_START=$((BOOT_SIZE)) +ROOTFS_END=$((ROOTFS_START+ROOTFS_SIZE)) +SWAP_START=$((ROOTFS_END)) + +# MMC devices use a partition prefix character 'p' +PART_PREFIX="" +if [ ! "${DEVICE#/dev/mmcblk}" = "${DEVICE}" ] || [ ! "${DEVICE#/dev/loop}" = "${DEVICE}" ]; then + PART_PREFIX="p" +fi +BOOTFS=$DEVICE${PART_PREFIX}1 +ROOTFS=$DEVICE${PART_PREFIX}2 +SWAP=$DEVICE${PART_PREFIX}3 + +TARGET_PART_PREFIX="" +if [ ! "${TARGET_DEVICE#/dev/mmcblk}" = "${TARGET_DEVICE}" ]; then + TARGET_PART_PREFIX="p" +fi +TARGET_ROOTFS=$TARGET_DEVICE${TARGET_PART_PREFIX}2 +TARGET_SWAP=$TARGET_DEVICE${TARGET_PART_PREFIX}3 + +echo "" +info "Boot partition size: $BOOT_SIZE MB ($BOOTFS)" +info "ROOTFS partition size: $ROOTFS_SIZE MB ($ROOTFS)" +info "Swap partition size: $SWAP_SIZE MB ($SWAP)" +echo "" + +# Use MSDOS by default as GPT cannot be reliably distributed in disk image form +# as it requires the backup table to be on the last block of the device, which +# of course varies from device to device. + +info "Partitioning installation media ($DEVICE)" + +debug "Deleting partition table on $DEVICE" +dd if=/dev/zero of=$DEVICE bs=512 count=2 >$OUT 2>&1 || die "Failed to zero beginning of $DEVICE" + +debug "Creating new partition table (MSDOS) on $DEVICE" +parted -s $DEVICE mklabel msdos >$OUT 2>&1 || die "Failed to create MSDOS partition table" + +debug "Creating boot partition on $BOOTFS" +parted -s $DEVICE mkpart primary 0% $BOOT_SIZE >$OUT 2>&1 || die "Failed to create BOOT partition" + +debug "Enabling boot flag on $BOOTFS" +parted -s $DEVICE set 1 boot on >$OUT 2>&1 || die "Failed to enable boot flag" + +debug "Creating ROOTFS partition on $ROOTFS" +parted -s $DEVICE mkpart primary $ROOTFS_START $ROOTFS_END >$OUT 2>&1 || die "Failed to create ROOTFS partition" + +debug "Creating swap partition on $SWAP" +parted -s $DEVICE mkpart primary $SWAP_START 100% >$OUT 2>&1 || die "Failed to create SWAP partition" + +if [ $DEBUG -eq 1 ]; then + parted -s $DEVICE print +fi + + +# +# Check if any $DEVICE partitions are mounted after partitioning +# +unmount_device || die "Failed to unmount $DEVICE partitions" + + +# +# Format $DEVICE partitions +# +info "Formatting partitions" +debug "Formatting $BOOTFS as vfat" +if [ ! "${DEVICE#/dev/loop}" = "${DEVICE}" ]; then + mkfs.vfat -I $BOOTFS -n "EFI" >$OUT 2>&1 || die "Failed to format $BOOTFS" +else + mkfs.vfat $BOOTFS -n "EFI" >$OUT 2>&1 || die "Failed to format $BOOTFS" +fi + +debug "Formatting $ROOTFS as ext3" +mkfs.ext3 -F $ROOTFS -L "ROOT" >$OUT 2>&1 || die "Failed to format $ROOTFS" + +debug "Formatting swap partition ($SWAP)" +mkswap $SWAP >$OUT 2>&1 || die "Failed to prepare swap" + + +# +# Installing to $DEVICE +# +debug "Mounting images and device in preparation for installation" +mount -o ro,loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" +mount -o ro,loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" +mount $ROOTFS $ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount $ROOTFS on $ROOTFS_MNT" +mount $BOOTFS $BOOTFS_MNT >$OUT 2>&1 || error "Failed to mount $BOOTFS on $BOOTFS_MNT" + +info "Preparing boot partition" +EFIDIR="$BOOTFS_MNT/EFI/BOOT" +cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy vmlinuz" +# Copy the efi loader and configs (booti*.efi and grub.cfg if it exists) +cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy EFI dir" +# Silently ignore a missing gummiboot loader dir (we might just be a GRUB image) +cp -r $HDDIMG_MNT/loader $BOOTFS_MNT >$OUT 2>&1 + +# Update the boot loaders configurations for an installed image +# Remove any existing root= kernel parameters and: +# o Add a root= parameter with the target rootfs +# o Specify ro so fsck can be run during boot +# o Specify rootwait in case the target media is an asyncronous block device +# such as MMC or USB disks +# o Specify "quiet" to minimize boot time when using slow serial consoles + +# Look for a GRUB installation +GRUB_CFG="$EFIDIR/grub.cfg" +if [ -e "$GRUB_CFG" ]; then + info "Configuring GRUB" + # Delete the install entry + sed -i "/menuentry 'install'/,/^}/d" $GRUB_CFG + # Delete the initrd lines + sed -i "/initrd /d" $GRUB_CFG + # Delete any LABEL= strings + sed -i "s/ LABEL=[^ ]*/ /" $GRUB_CFG + + sed -i "s@ root=[^ ]*@ @" $GRUB_CFG + sed -i "s@vmlinuz @vmlinuz root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GRUB_CFG +fi + +# Look for a gummiboot installation +GUMMI_ENTRIES="$BOOTFS_MNT/loader/entries" +GUMMI_CFG="$GUMMI_ENTRIES/boot.conf" +if [ -d "$GUMMI_ENTRIES" ]; then + info "Configuring Gummiboot" + # remove the install target if it exists + rm $GUMMI_ENTRIES/install.conf >$OUT 2>&1 + + if [ ! -e "$GUMMI_CFG" ]; then + echo "ERROR: $GUMMI_CFG not found" + fi + + sed -i "/initrd /d" $GUMMI_CFG + sed -i "s@ root=[^ ]*@ @" $GUMMI_CFG + sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GUMMI_CFG +fi + +# Ensure we have at least one EFI bootloader configured +if [ ! -e $GRUB_CFG ] && [ ! -e $GUMMI_CFG ]; then + die "No EFI bootloader configuration found" +fi + + +info "Copying ROOTFS files (this may take a while)" +cp -a $HDDIMG_ROOTFS_MNT/* $ROOTFS_MNT >$OUT 2>&1 || die "Root FS copy failed" + +echo "$TARGET_SWAP swap swap defaults 0 0" >> $ROOTFS_MNT/etc/fstab + +# We dont want udev to mount our root device while we're booting... +if [ -d $ROOTFS_MNT/etc/udev/ ] ; then + echo "$TARGET_DEVICE" >> $ROOTFS_MNT/etc/udev/mount.blacklist +fi + +# Add startup.nsh script for automated boot +echo "fs0:\EFI\BOOT\bootx64.efi" > $BOOTFS_MNT/startup.nsh + + +# Call cleanup to unmount devices and images and remove the TMPDIR +cleanup + +echo "" +if [ $WARNINGS -ne 0 ] && [ $ERRORS -eq 0 ]; then + echo "${YELLOW}Installation completed with warnings${CLEAR}" + echo "${YELLOW}Warnings: $WARNINGS${CLEAR}" +elif [ $ERRORS -ne 0 ]; then + echo "${RED}Installation encountered errors${CLEAR}" + echo "${RED}Errors: $ERRORS${CLEAR}" + echo "${YELLOW}Warnings: $WARNINGS${CLEAR}" +else + success "Installation completed successfully" +fi +echo "" diff --git a/scripts/contrib/python/generate-manifest-2.7.py b/scripts/contrib/python/generate-manifest-2.7.py new file mode 100755 index 0000000..d93c943 --- /dev/null +++ b/scripts/contrib/python/generate-manifest-2.7.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python + +# generate Python Manifest for the OpenEmbedded build system +# (C) 2002-2010 Michael 'Mickey' Lauer +# (C) 2007 Jeremy Laine +# licensed under MIT, see COPYING.MIT +# +# June 22, 2011 -- Mark Hatle +# * Updated to no longer generate special -dbg package, instead use the +# single system -dbg +# * Update version with ".1" to indicate this change + +import os +import sys +import time + +VERSION = "2.7.2" + +__author__ = "Michael 'Mickey' Lauer " +__version__ = "20110222.2" + +class MakefileMaker: + + def __init__( self, outfile ): + """initialize""" + self.packages = {} + self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.output = outfile + self.out( """ +# WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. +# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer +# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy +""" % ( sys.argv[0], __version__ ) ) + + # + # helper functions + # + + def out( self, data ): + """print a line to the output file""" + self.output.write( "%s\n" % data ) + + def setPrefix( self, targetPrefix ): + """set a file prefix for addPackage files""" + self.targetPrefix = targetPrefix + + def doProlog( self ): + self.out( """ """ ) + self.out( "" ) + + def addPackage( self, name, description, dependencies, filenames ): + """add a package to the Makefile""" + if type( filenames ) == type( "" ): + filenames = filenames.split() + fullFilenames = [] + for filename in filenames: + if filename[0] != "$": + fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) ) + else: + fullFilenames.append( filename ) + self.packages[name] = description, dependencies, fullFilenames + + def doBody( self ): + """generate body of Makefile""" + + global VERSION + + # + # generate provides line + # + + provideLine = 'PROVIDES+="' + for name in sorted(self.packages): + provideLine += "%s " % name + provideLine += '"' + + self.out( provideLine ) + self.out( "" ) + + # + # generate package line + # + + packageLine = 'PACKAGES="${PN}-dbg ' + for name in sorted(self.packages): + if name.startswith("${PN}-distutils"): + if name == "${PN}-distutils": + packageLine += "%s-staticdev %s " % (name, name) + elif name != '${PN}-dbg': + packageLine += "%s " % name + packageLine += '${PN}-modules"' + + self.out( packageLine ) + self.out( "" ) + + # + # generate package variables + # + + for name, data in sorted(self.packages.iteritems()): + desc, deps, files = data + + # + # write out the description, revision and dependencies + # + self.out( 'SUMMARY_%s="%s"' % ( name, desc ) ) + self.out( 'RDEPENDS_%s="%s"' % ( name, deps ) ) + + line = 'FILES_%s="' % name + + # + # check which directories to make in the temporary directory + # + + dirset = {} # if python had a set-datatype this would be sufficient. for now, we're using a dict instead. + for target in files: + dirset[os.path.dirname( target )] = True + + # + # generate which files to copy for the target (-dfR because whole directories are also allowed) + # + + for target in files: + line += "%s " % target + + line += '"' + self.out( line ) + self.out( "" ) + + self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) + line = 'RDEPENDS_${PN}-modules="' + + for name, data in sorted(self.packages.iteritems()): + if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: + line += "%s " % name + + self.out( "%s \"" % line ) + self.out( 'ALLOW_EMPTY_${PN}-modules = "1"' ) + + def doEpilog( self ): + self.out( """""" ) + self.out( "" ) + + def make( self ): + self.doProlog() + self.doBody() + self.doEpilog() + +if __name__ == "__main__": + + if len( sys.argv ) > 1: + try: + os.unlink(sys.argv[1]) + except Exception: + sys.exc_clear() + outfile = file( sys.argv[1], "w" ) + else: + outfile = sys.stdout + + m = MakefileMaker( outfile ) + + # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! + # Parameters: revision, name, description, dependencies, filenames + # + + m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re", + "__future__.* _abcoll.* abc.* ast.* copy.* copy_reg.* ConfigParser.* " + + "genericpath.* getopt.* linecache.* new.* " + + "os.* posixpath.* struct.* " + + "warnings.* site.* stat.* " + + "UserDict.* UserList.* UserString.* " + + "lib-dynload/binascii.so lib-dynload/_struct.so lib-dynload/time.so " + + "lib-dynload/xreadlines.so types.* platform.* ${bindir}/python* " + + "_weakrefset.* sysconfig.* _sysconfigdata.* config/Makefile " + + "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h " + + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") + + m.addPackage( "${PN}-dev", "Python development package", "${PN}-core", + "${includedir} " + + "${libdir}/lib*${SOLIBSDEV} " + + "${libdir}/*.la " + + "${libdir}/*.a " + + "${libdir}/*.o " + + "${libdir}/pkgconfig " + + "${base_libdir}/*.a " + + "${base_libdir}/*.o " + + "${datadir}/aclocal " + + "${datadir}/pkgconfig " ) + + m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", + "${bindir}/2to3 lib2to3" ) # package + + m.addPackage( "${PN}-idle", "Python Integrated Development Environment", "${PN}-core ${PN}-tkinter", + "${bindir}/idle idlelib" ) # package + + m.addPackage( "${PN}-pydoc", "Python interactive help support", "${PN}-core ${PN}-lang ${PN}-stringold ${PN}-re", + "${bindir}/pydoc pydoc.* pydoc_data" ) + + m.addPackage( "${PN}-smtpd", "Python Simple Mail Transport Daemon", "${PN}-core ${PN}-netserver ${PN}-email ${PN}-mime", + "${bindir}/smtpd.* smtpd.*" ) + + m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core", + "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.so lib-dynload/audioop.so audiodev.* sunaudio.* sunau.* toaiff.*" ) + + m.addPackage( "${PN}-bsddb", "Python bindings for the Berkeley Database", "${PN}-core", + "bsddb lib-dynload/_bsddb.so" ) # package + + m.addPackage( "${PN}-codecs", "Python codecs, encodings & i18n support", "${PN}-core ${PN}-lang", + "codecs.* encodings gettext.* locale.* lib-dynload/_locale.so lib-dynload/_codecs* lib-dynload/_multibytecodec.so lib-dynload/unicodedata.so stringprep.* xdrlib.*" ) + + m.addPackage( "${PN}-compile", "Python bytecode compilation support", "${PN}-core", + "py_compile.* compileall.*" ) + + m.addPackage( "${PN}-compiler", "Python compiler support", "${PN}-core", + "compiler" ) # package + + m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-zlib", + "gzip.* zipfile.* tarfile.* lib-dynload/bz2.so" ) + + m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core", + "hashlib.* md5.* sha.* lib-dynload/crypt.so lib-dynload/_hashlib.so lib-dynload/_sha256.so lib-dynload/_sha512.so" ) + + m.addPackage( "${PN}-textutils", "Python option parsing, text wrapping and CSV support", "${PN}-core ${PN}-io ${PN}-re ${PN}-stringold", + "lib-dynload/_csv.so csv.* optparse.* textwrap.*" ) + + m.addPackage( "${PN}-curses", "Python curses support", "${PN}-core", + "curses lib-dynload/_curses.so lib-dynload/_curses_panel.so" ) # directory + low level module + + m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core", + "ctypes lib-dynload/_ctypes.so lib-dynload/_ctypes_test.so" ) # directory + low level module + + m.addPackage( "${PN}-datetime", "Python calendar and time support", "${PN}-core ${PN}-codecs", + "_strptime.* calendar.* lib-dynload/datetime.so" ) + + m.addPackage( "${PN}-db", "Python file-based database support", "${PN}-core", + "anydbm.* dumbdbm.* whichdb.* " ) + + m.addPackage( "${PN}-debugger", "Python debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint", + "bdb.* pdb.*" ) + + m.addPackage( "${PN}-difflib", "Python helpers for computing deltas between objects", "${PN}-lang ${PN}-re", + "difflib.*" ) + + m.addPackage( "${PN}-distutils-staticdev", "Python distribution utilities (static libraries)", "${PN}-distutils", + "config/lib*.a" ) # package + + m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core ${PN}-email", + "config distutils" ) # package + + m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib", + "doctest.*" ) + + m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", + "imaplib.* email" ) # package + + m.addPackage( "${PN}-fcntl", "Python's fcntl interface", "${PN}-core", + "lib-dynload/fcntl.so" ) + + m.addPackage( "${PN}-hotshot", "Python hotshot performance profiler", "${PN}-core", + "hotshot lib-dynload/_hotshot.so" ) + + m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", + "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) + + m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core", + "importlib" ) + + m.addPackage( "${PN}-gdbm", "Python GNU database support", "${PN}-core", + "lib-dynload/gdbm.so" ) + + m.addPackage( "${PN}-image", "Python graphical image handling", "${PN}-core", + "colorsys.* imghdr.* lib-dynload/imageop.so lib-dynload/rgbimg.so" ) + + m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math ${PN}-textutils ${PN}-netclient ${PN}-contextlib", + "lib-dynload/_socket.so lib-dynload/_io.so lib-dynload/_ssl.so lib-dynload/select.so lib-dynload/termios.so lib-dynload/cStringIO.so " + + "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) + + m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re ${PN}-codecs", + "json lib-dynload/_json.so" ) # package + + m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", + "lib-dynload/_bisect.so lib-dynload/_collections.so lib-dynload/_heapq.so lib-dynload/_weakref.so lib-dynload/_functools.so " + + "lib-dynload/array.so lib-dynload/itertools.so lib-dynload/operator.so lib-dynload/parser.so " + + "atexit.* bisect.* code.* codeop.* collections.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* symbol.* repr.* token.* " + + "tokenize.* traceback.* weakref.*" ) + + m.addPackage( "${PN}-logging", "Python logging support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold", + "logging" ) # package + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.addPackage( "${PN}-math", "Python math support", "${PN}-core ${PN}-crypt", + "lib-dynload/cmath.so lib-dynload/math.so lib-dynload/_random.so random.* sets.*" ) + + m.addPackage( "${PN}-mime", "Python MIME handling APIs", "${PN}-core ${PN}-io", + "mimetools.* uu.* quopri.* rfc822.* MimeWriter.*" ) + + m.addPackage( "${PN}-mmap", "Python memory-mapped file support", "${PN}-core ${PN}-io", + "lib-dynload/mmap.so " ) + + m.addPackage( "${PN}-multiprocessing", "Python multiprocessing support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-threading ${PN}-ctypes ${PN}-mmap", + "lib-dynload/_multiprocessing.so multiprocessing" ) # package + + m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime", + "*Cookie*.* " + + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib.* urllib2.* urlparse.* uuid.* rfc822.* mimetools.*" ) + + m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", + "cgi.* *HTTPServer.* SocketServer.*" ) + + m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", + "decimal.* fractions.* numbers.*" ) + + m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", + "pickle.* shelve.* lib-dynload/cPickle.so pickletools.*" ) + + m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", + "pkgutil.*") + + m.addPackage( "${PN}-plistlib", "Generate and parse Mac OS X .plist files", "${PN}-core ${PN}-datetime ${PN}-io", + "plistlib.*") + + m.addPackage( "${PN}-pprint", "Python pretty-print support", "${PN}-core ${PN}-io", + "pprint.*" ) + + m.addPackage( "${PN}-profile", "Python basic performance profiling support", "${PN}-core ${PN}-textutils", + "profile.* pstats.* cProfile.* lib-dynload/_lsprof.so" ) + + m.addPackage( "${PN}-re", "Python Regular Expression APIs", "${PN}-core", + "re.* sre.* sre_compile.* sre_constants* sre_parse.*" ) # _sre is builtin + + m.addPackage( "${PN}-readline", "Python readline support", "${PN}-core", + "lib-dynload/readline.so rlcompleter.*" ) + + m.addPackage( "${PN}-resource", "Python resource control interface", "${PN}-core", + "lib-dynload/resource.so" ) + + m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re", + "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" ) + + m.addPackage( "${PN}-robotparser", "Python robots.txt parser", "${PN}-core ${PN}-netclient", + "robotparser.*") + + m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle", + "subprocess.*" ) + + m.addPackage( "${PN}-sqlite3", "Python Sqlite3 database support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading ${PN}-zlib", + "lib-dynload/_sqlite3.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" ) + + m.addPackage( "${PN}-sqlite3-tests", "Python Sqlite3 database support tests", "${PN}-core ${PN}-sqlite3", + "sqlite3/test" ) + + m.addPackage( "${PN}-stringold", "Python string APIs [deprecated]", "${PN}-core ${PN}-re", + "lib-dynload/strop.so string.* stringold.*" ) + + m.addPackage( "${PN}-syslog", "Python syslog interface", "${PN}-core", + "lib-dynload/syslog.so" ) + + m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io", + "pty.* tty.*" ) + + m.addPackage( "${PN}-tests", "Python tests", "${PN}-core", + "test" ) # package + + m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", + "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" ) + + m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", + "lib-dynload/_tkinter.so lib-tk" ) # package + + m.addPackage( "${PN}-unittest", "Python unit testing framework", "${PN}-core ${PN}-stringold ${PN}-lang ${PN}-io ${PN}-difflib ${PN}-pprint ${PN}-shell", + "unittest/" ) + + m.addPackage( "${PN}-unixadmin", "Python Unix administration support", "${PN}-core", + "lib-dynload/nis.so lib-dynload/grp.so lib-dynload/pwd.so getpass.*" ) + + m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", + "lib-dynload/_elementtree.so lib-dynload/pyexpat.so xml xmllib.*" ) # package + + m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", + "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.*" ) + + m.addPackage( "${PN}-zlib", "Python zlib compression support", "${PN}-core", + "lib-dynload/zlib.so" ) + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + + m.addPackage( "${PN}-contextlib", "Python utilities for with-statement" + + "contexts.", "${PN}-core", + "${libdir}/python${PYTHON_MAJMIN}/contextlib.*" ) + + m.make() diff --git a/scripts/contrib/python/generate-manifest-3.5.py b/scripts/contrib/python/generate-manifest-3.5.py new file mode 100755 index 0000000..eac493a --- /dev/null +++ b/scripts/contrib/python/generate-manifest-3.5.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python + +# generate Python Manifest for the OpenEmbedded build system +# (C) 2002-2010 Michael 'Mickey' Lauer +# (C) 2007 Jeremy Laine +# licensed under MIT, see COPYING.MIT +# +# June 22, 2011 -- Mark Hatle +# * Updated to no longer generate special -dbg package, instead use the +# single system -dbg +# * Update version with ".1" to indicate this change +# +# 2014 Khem Raj +# Added python3 support +# +import os +import sys +import time + +VERSION = "3.5.0" + +__author__ = "Michael 'Mickey' Lauer " +__version__ = "20140131" + +class MakefileMaker: + + def __init__( self, outfile ): + """initialize""" + self.packages = {} + self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.output = outfile + self.out( """ +# WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. +# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer +# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy +""" % ( sys.argv[0], __version__ ) ) + + # + # helper functions + # + + def out( self, data ): + """print a line to the output file""" + self.output.write( "%s\n" % data ) + + def setPrefix( self, targetPrefix ): + """set a file prefix for addPackage files""" + self.targetPrefix = targetPrefix + + def doProlog( self ): + self.out( """ """ ) + self.out( "" ) + + def addPackage( self, name, description, dependencies, filenames ): + """add a package to the Makefile""" + if type( filenames ) == type( "" ): + filenames = filenames.split() + fullFilenames = [] + for filename in filenames: + if filename[0] != "$": + fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) ) + else: + fullFilenames.append( filename ) + self.packages[name] = description, dependencies, fullFilenames + + def doBody( self ): + """generate body of Makefile""" + + global VERSION + + # + # generate provides line + # + + provideLine = 'PROVIDES+="' + for name in sorted(self.packages): + provideLine += "%s " % name + provideLine += '"' + + self.out( provideLine ) + self.out( "" ) + + # + # generate package line + # + + packageLine = 'PACKAGES="${PN}-dbg ' + for name in sorted(self.packages): + if name.startswith("${PN}-distutils"): + if name == "${PN}-distutils": + packageLine += "%s-staticdev %s " % (name, name) + elif name != '${PN}-dbg': + packageLine += "%s " % name + packageLine += '${PN}-modules"' + + self.out( packageLine ) + self.out( "" ) + + # + # generate package variables + # + + for name, data in sorted(self.packages.iteritems()): + desc, deps, files = data + + # + # write out the description, revision and dependencies + # + self.out( 'SUMMARY_%s="%s"' % ( name, desc ) ) + self.out( 'RDEPENDS_%s="%s"' % ( name, deps ) ) + + line = 'FILES_%s="' % name + + # + # check which directories to make in the temporary directory + # + + dirset = {} # if python had a set-datatype this would be sufficient. for now, we're using a dict instead. + for target in files: + dirset[os.path.dirname( target )] = True + + # + # generate which files to copy for the target (-dfR because whole directories are also allowed) + # + + for target in files: + line += "%s " % target + + line += '"' + self.out( line ) + self.out( "" ) + + self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) + line = 'RDEPENDS_${PN}-modules="' + + for name, data in sorted(self.packages.iteritems()): + if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: + line += "%s " % name + + self.out( "%s \"" % line ) + self.out( 'ALLOW_EMPTY_${PN}-modules = "1"' ) + + def doEpilog( self ): + self.out( """""" ) + self.out( "" ) + + def make( self ): + self.doProlog() + self.doBody() + self.doEpilog() + +if __name__ == "__main__": + + if len( sys.argv ) > 1: + try: + os.unlink(sys.argv[1]) + except Exception: + sys.exc_clear() + outfile = file( sys.argv[1], "w" ) + else: + outfile = sys.stdout + + m = MakefileMaker( outfile ) + + # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! + # Parameters: revision, name, description, dependencies, filenames + # + + m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re ${PN}-reprlib ${PN}-codecs ${PN}-io ${PN}-math", + "__future__.* _abcoll.* abc.* ast.* copy.* copyreg.* ConfigParser.* " + + "genericpath.* getopt.* linecache.* new.* " + + "os.* posixpath.* struct.* " + + "warnings.* site.* stat.* " + + "UserDict.* UserList.* UserString.* " + + "lib-dynload/binascii.*.so lib-dynload/_struct.*.so lib-dynload/time.*.so " + + "lib-dynload/xreadlines.*.so types.* platform.* ${bindir}/python* " + + "_weakrefset.* sysconfig.* _sysconfigdata.* config/Makefile " + + "${includedir}/python${PYTHON_BINABI}/pyconfig*.h " + + "${libdir}/python${PYTHON_MAJMIN}/collections " + + "${libdir}/python${PYTHON_MAJMIN}/_collections_abc.* " + + "${libdir}/python${PYTHON_MAJMIN}/_sitebuiltins.* " + + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") + + m.addPackage( "${PN}-dev", "Python development package", "${PN}-core", + "${includedir} " + + "${libdir}/lib*${SOLIBSDEV} " + + "${libdir}/*.la " + + "${libdir}/*.a " + + "${libdir}/*.o " + + "${libdir}/pkgconfig " + + "${base_libdir}/*.a " + + "${base_libdir}/*.o " + + "${datadir}/aclocal " + + "${datadir}/pkgconfig " ) + + m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", + "lib2to3" ) # package + + m.addPackage( "${PN}-idle", "Python Integrated Development Environment", "${PN}-core ${PN}-tkinter", + "${bindir}/idle idlelib" ) # package + + m.addPackage( "${PN}-pydoc", "Python interactive help support", "${PN}-core ${PN}-lang ${PN}-stringold ${PN}-re", + "${bindir}/pydoc pydoc.* pydoc_data" ) + + m.addPackage( "${PN}-smtpd", "Python Simple Mail Transport Daemon", "${PN}-core ${PN}-netserver ${PN}-email ${PN}-mime", + "${bindir}/smtpd.* smtpd.*" ) + + m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core", + "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.*.so lib-dynload/audioop.*.so audiodev.* sunaudio.* sunau.* toaiff.*" ) + + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + + m.addPackage( "${PN}-asyncio", "Python Asynchronous I/O, event loop, coroutines and tasks", "${PN}-core", + "asyncio" ) + + m.addPackage( "${PN}-codecs", "Python codecs, encodings & i18n support", "${PN}-core ${PN}-lang", + "codecs.* encodings gettext.* locale.* lib-dynload/_locale.*.so lib-dynload/_codecs* lib-dynload/_multibytecodec.*.so lib-dynload/unicodedata.*.so stringprep.* xdrlib.*" ) + + m.addPackage( "${PN}-compile", "Python bytecode compilation support", "${PN}-core", + "py_compile.* compileall.*" ) + + m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs ${PN}-importlib ${PN}-threading ${PN}-shell", + "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so" ) + + m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core", + "hashlib.* md5.* sha.* lib-dynload/crypt.*.so lib-dynload/_hashlib.*.so lib-dynload/_sha256.*.so lib-dynload/_sha512.*.so" ) + + m.addPackage( "${PN}-textutils", "Python option parsing, text wrapping and CSV support", "${PN}-core ${PN}-io ${PN}-re ${PN}-stringold", + "lib-dynload/_csv.*.so csv.* optparse.* textwrap.*" ) + + m.addPackage( "${PN}-curses", "Python curses support", "${PN}-core", + "curses lib-dynload/_curses.*.so lib-dynload/_curses_panel.*.so" ) # directory + low level module + + m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core", + "ctypes lib-dynload/_ctypes.*.so lib-dynload/_ctypes_test.*.so" ) # directory + low level module + + m.addPackage( "${PN}-datetime", "Python calendar and time support", "${PN}-core ${PN}-codecs", + "_strptime.* calendar.* lib-dynload/datetime.*.so" ) + + m.addPackage( "${PN}-db", "Python file-based database support", "${PN}-core", + "anydbm.* dumbdbm.* whichdb.* dbm lib-dynload/_dbm.*.so" ) + + m.addPackage( "${PN}-debugger", "Python debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint ${PN}-importlib ${PN}-pkgutil", + "bdb.* pdb.*" ) + + m.addPackage( "${PN}-difflib", "Python helpers for computing deltas between objects", "${PN}-lang ${PN}-re", + "difflib.*" ) + + m.addPackage( "${PN}-distutils-staticdev", "Python distribution utilities (static libraries)", "${PN}-distutils", + "config/lib*.a" ) # package + + m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core ${PN}-email", + "config distutils" ) # package + + m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib", + "doctest.*" ) + + m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", + "imaplib.* email" ) # package + + m.addPackage( "${PN}-enum", "Python support for enumerations", "${PN}-core", + "enum.*" ) + + m.addPackage( "${PN}-fcntl", "Python's fcntl interface", "${PN}-core", + "lib-dynload/fcntl.*.so" ) + + m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", + "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) + + m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core ${PN}-lang", + "importlib" ) + + m.addPackage( "${PN}-gdbm", "Python GNU database support", "${PN}-core", + "lib-dynload/_gdbm.*.so" ) + + m.addPackage( "${PN}-image", "Python graphical image handling", "${PN}-core", + "colorsys.* imghdr.* lib-dynload/imageop.*.so lib-dynload/rgbimg.*.so" ) + + m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math", + "lib-dynload/_socket.*.so lib-dynload/_io.*.so lib-dynload/_ssl.*.so lib-dynload/select.*.so lib-dynload/termios.*.so lib-dynload/cStringIO.*.so " + + "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) + + m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re", + "json lib-dynload/_json.*.so" ) # package + + m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", + "lib-dynload/_bisect.*.so lib-dynload/_collections.*.so lib-dynload/_heapq.*.so lib-dynload/_weakref.*.so lib-dynload/_functools.*.so " + + "lib-dynload/array.*.so lib-dynload/itertools.*.so lib-dynload/operator.*.so lib-dynload/parser.*.so " + + "atexit.* bisect.* code.* codeop.* collections.* _collections_abc.* contextlib.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* operator.* symbol.* repr.* token.* " + + "tokenize.* traceback.* weakref.*" ) + + m.addPackage( "${PN}-logging", "Python logging support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold", + "logging" ) # package + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.addPackage( "${PN}-math", "Python math support", "${PN}-core", + "lib-dynload/cmath.*.so lib-dynload/math.*.so lib-dynload/_random.*.so random.* sets.*" ) + + m.addPackage( "${PN}-mime", "Python MIME handling APIs", "${PN}-core ${PN}-io", + "mimetools.* uu.* quopri.* rfc822.* MimeWriter.*" ) + + m.addPackage( "${PN}-mmap", "Python memory-mapped file support", "${PN}-core ${PN}-io", + "lib-dynload/mmap.*.so " ) + + m.addPackage( "${PN}-multiprocessing", "Python multiprocessing support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-threading ${PN}-ctypes ${PN}-mmap", + "lib-dynload/_multiprocessing.*.so multiprocessing" ) # package + + m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime", + "*Cookie*.* " + + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib uuid.* rfc822.* mimetools.*" ) + + m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", + "cgi.* *HTTPServer.* SocketServer.*" ) + + m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", + "decimal.* fractions.* numbers.*" ) + + m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", + "pickle.* shelve.* lib-dynload/cPickle.*.so pickletools.*" ) + + m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", + "pkgutil.*") + + m.addPackage( "${PN}-pprint", "Python pretty-print support", "${PN}-core ${PN}-io", + "pprint.*" ) + + m.addPackage( "${PN}-profile", "Python basic performance profiling support", "${PN}-core ${PN}-textutils", + "profile.* pstats.* cProfile.* lib-dynload/_lsprof.*.so" ) + + m.addPackage( "${PN}-re", "Python Regular Expression APIs", "${PN}-core", + "re.* sre.* sre_compile.* sre_constants* sre_parse.*" ) # _sre is builtin + + m.addPackage( "${PN}-readline", "Python readline support", "${PN}-core", + "lib-dynload/readline.*.so rlcompleter.*" ) + + m.addPackage( "${PN}-reprlib", "Python alternate repr() implementation", "${PN}-core", + "reprlib.py" ) + + m.addPackage( "${PN}-resource", "Python resource control interface", "${PN}-core", + "lib-dynload/resource.*.so" ) + + m.addPackage( "${PN}-selectors", "Python High-level I/O multiplexing", "${PN}-core", + "selectors.*" ) + + m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re", + "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" ) + + m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle ${PN}-signal ${PN}-selectors", + "subprocess.*" ) + + m.addPackage( "${PN}-signal", "Python set handlers for asynchronous events support", "${PN}-core", + "signal.*" ) + + m.addPackage( "${PN}-sqlite3", "Python Sqlite3 database support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading", + "lib-dynload/_sqlite3.*.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" ) + + m.addPackage( "${PN}-sqlite3-tests", "Python Sqlite3 database support tests", "${PN}-core ${PN}-sqlite3", + "sqlite3/test" ) + + m.addPackage( "${PN}-stringold", "Python string APIs [deprecated]", "${PN}-core ${PN}-re", + "lib-dynload/strop.*.so string.* stringold.*" ) + + m.addPackage( "${PN}-syslog", "Python syslog interface", "${PN}-core", + "lib-dynload/syslog.*.so" ) + + m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io", + "pty.* tty.*" ) + + m.addPackage( "${PN}-tests", "Python tests", "${PN}-core", + "test" ) # package + + m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", + "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* queue.*" ) + + m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", + "lib-dynload/_tkinter.*.so lib-tk tkinter" ) # package + + m.addPackage( "${PN}-unittest", "Python unit testing framework", "${PN}-core ${PN}-stringold ${PN}-lang ${PN}-io ${PN}-difflib ${PN}-pprint ${PN}-shell", + "unittest/" ) + + m.addPackage( "${PN}-unixadmin", "Python Unix administration support", "${PN}-core", + "lib-dynload/nis.*.so lib-dynload/grp.*.so lib-dynload/pwd.*.so getpass.*" ) + + m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", + "lib-dynload/_elementtree.*.so lib-dynload/pyexpat.*.so xml xmllib.*" ) # package + + m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", + "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.* xmlrpc" ) + + m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", + "mailbox.*" ) + + m.make() diff --git a/scripts/contrib/serdevtry b/scripts/contrib/serdevtry new file mode 100755 index 0000000..74bd7b7 --- /dev/null +++ b/scripts/contrib/serdevtry @@ -0,0 +1,60 @@ +#!/bin/sh + +# Copyright (C) 2014 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +if [ "$1" = "" -o "$1" = "--help" ] ; then + echo "Usage: $0 " + echo + echo "Simple script to handle maintaining a terminal for serial devices that" + echo "disappear when a device is powered down or reset, such as the USB" + echo "serial console on the original BeagleBone (white version)." + echo + echo "e.g. $0 picocom -b 115200 /dev/ttyUSB0" + echo + exit +fi + +args="$@" +DEVICE="" +while [ "$1" != "" ]; do + case "$1" in + /dev/*) + DEVICE=$1 + break;; + esac + shift +done + +if [ "$DEVICE" != "" ] ; then + while true; do + if [ ! -e $DEVICE ] ; then + echo "serdevtry: waiting for $DEVICE to exist..." + while [ ! -e $DEVICE ]; do + sleep 0.1 + done + fi + if [ ! -w $DEVICE ] ; then + # Sometimes (presumably because of a race with udev) we get to + # the device before its permissions have been set up + RETRYNUM=0 + while [ ! -w $DEVICE ]; do + if [ "$RETRYNUM" = "2" ] ; then + echo "Device $DEVICE exists but is not writable!" + exit 1 + fi + RETRYNUM=$((RETRYNUM+1)) + sleep 0.1 + done + fi + $args + if [ -e $DEVICE ] ; then + break + fi + done +else + echo "Unable to determine device node from command: $args" + exit 1 +fi + diff --git a/scripts/contrib/test_build_time.sh b/scripts/contrib/test_build_time.sh new file mode 100755 index 0000000..9e5725a --- /dev/null +++ b/scripts/contrib/test_build_time.sh @@ -0,0 +1,237 @@ +#!/bin/bash + +# Build performance regression test script +# +# Copyright 2011 Intel Corporation +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# +# DESCRIPTION +# This script is intended to be used in conjunction with "git bisect run" +# in order to find regressions in build time, however it can also be used +# independently. It cleans out the build output directories, runs a +# specified worker script (an example is test_build_time_worker.sh) under +# TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a +# value telling "git bisect run" whether the build time is good (under +# the specified threshold) or bad (over it). There is also a tolerance +# option but it is not particularly useful as it only subtracts the +# tolerance from the given threshold and uses it as the actual threshold. +# +# It is also capable of taking a file listing git revision hashes to be +# test-applied to the repository in order to get past build failures that +# would otherwise cause certain revisions to have to be skipped; if a +# revision does not apply cleanly then the script assumes it does not +# need to be applied and ignores it. +# +# Please see the help output (syntax below) for some important setup +# instructions. +# +# AUTHORS +# Paul Eggleton + + +syntax() { + echo "syntax: $0