diff options
Diffstat (limited to 'scripts/sstate-cache-management.sh')
| -rwxr-xr-x | scripts/sstate-cache-management.sh | 469 |
1 files changed, 469 insertions, 0 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh new file mode 100755 index 0000000..2ab450a --- /dev/null +++ b/scripts/sstate-cache-management.sh @@ -0,0 +1,469 @@ +#!/bin/bash + +# Copyright (c) 2012 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# Global vars +cache_dir= +confirm= +fsym= +total_deleted=0 +verbose= +debug=0 + +usage () { + cat << EOF +Welcome to sstate cache management utilities. +sstate-cache-management.sh <OPTION> + +Options: + -h, --help + Display this help and exit. + + --cache-dir=<sstate cache dir> + Specify sstate cache directory, will use the environment + variable SSTATE_CACHE_DIR if it is not specified. + + --extra-archs=<arch1>,<arch2>...<archn> + Specify list of architectures which should be tested, this list + will be extended with native arch, allarch and empty arch. The + script won't be trying to generate list of available archs from + AVAILTUNES in tune files. + + --extra-layer=<layer1>,<layer2>...<layern> + Specify the layer which will be used for searching the archs, + it will search the meta and meta-* layers in the top dir by + default, and will search meta, meta-*, <layer1>, <layer2>, + ...<layern> when specified. Use "," as the separator. + + This is useless for --stamps-dir or when --extra-archs is used. + + -d, --remove-duplicated + Remove the duplicated sstate cache files of one package, only + the newest one will be kept. The duplicated sstate cache files + of one package must have the same arch, which means sstate cache + files with multiple archs are not considered duplicate. + + Conflicts with --stamps-dir. + + --stamps-dir=<dir1>,<dir2>...<dirn> + Specify the build directory's stamps directories, the sstate + cache file which IS USED by these build diretories will be KEPT, + other sstate cache files in cache-dir will be removed. Use "," + as the separator. For example: + --stamps-dir=build1/tmp/stamps,build2/tmp/stamps + + Conflicts with --remove-duplicated. + + -L, --follow-symlink + Remove both the symbol link and the destination file, default: no. + + -y, --yes + Automatic yes to prompts; assume "yes" as answer to all prompts + and run non-interactively. + + -v, --verbose + Explain what is being done. + + -D, --debug + Show debug info, repeat for more debug info. + +EOF +} + +if [ $# -lt 1 ]; then + usage + exit 0 +fi + +# Echo no files to remove +no_files () { + echo No files to remove +} + +# Echo nothing to do +do_nothing () { + echo Nothing to do +} + +# Read the input "y" +read_confirm () { + echo "$total_deleted out of $total_files files will be removed! " + if [ "$confirm" != "y" ]; then + echo "Do you want to continue (y/n)? " + while read confirm; do + [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \ + -o "$confirm" = "N" ] && break + echo "Invalid input \"$confirm\", please input 'y' or 'n': " + done + else + echo + fi +} + +# Print error information and exit. +echo_error () { + echo "ERROR: $1" >&2 + exit 1 +} + +# Generate the remove list: +# +# * Add .done/.siginfo to the remove list +# * Add destination of symlink to the remove list +# +# $1: output file, others: sstate cache file (.tgz) +gen_rmlist (){ + local rmlist_file="$1" + shift + local files="$@" + for i in $files; do + echo $i >> $rmlist_file + # Add the ".siginfo" + if [ -e $i.siginfo ]; then + echo $i.siginfo >> $rmlist_file + fi + # Add the destination of symlink + if [ -L "$i" ]; then + if [ "$fsym" = "y" ]; then + dest="`readlink -e $i`" + if [ -n "$dest" ]; then + echo $dest >> $rmlist_file + # Remove the .siginfo when .tgz is removed + if [ -f "$dest.siginfo" ]; then + echo $dest.siginfo >> $rmlist_file + fi + fi + fi + # Add the ".tgz.done" and ".siginfo.done" (may exist in the future) + base_fn="${i##/*/}" + t_fn="$base_fn.done" + s_fn="$base_fn.siginfo.done" + for d in $t_fn $s_fn; do + if [ -f $cache_dir/$d ]; then + echo $cache_dir/$d >> $rmlist_file + fi + done + fi + done +} + +# Remove the duplicated cache files for the pkg, keep the newest one +remove_duplicated () { + + local topdir + local oe_core_dir + local tunedirs + local all_archs + local all_machines + local ava_archs + local arch + local file_names + local sstate_files_list + local fn_tmp + local list_suffix=`mktemp` || exit 1 + + if [ -z "$extra_archs" ] ; then + # Find out the archs in all the layers + echo "Figuring out the archs in the layers ... " + oe_core_dir=$(dirname $(dirname $(readlink -e $0))) + topdir=$(dirname $oe_core_dir) + tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`" + [ -n "$tunedirs" ] || echo_error "Can't find the tune directory" + all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`" + all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'` + fi + + # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon + # Sort to remove the duplicated ones + # Add allarch and builder arch (native) + builder_arch=$(uname -m) + all_archs="$(echo allarch $all_archs $all_machines $builder_arch \ + | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs" + echo "Done" + + # Total number of files including sstate-, .siginfo and .done files + total_files=`find $cache_dir -name 'sstate*' | wc -l` + # Save all the sstate files in a file + sstate_files_list=`mktemp` || exit 1 + find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list + + echo "Figuring out the suffixes in the sstate cache dir ... " + sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`" + echo "Done" + echo "The following suffixes have been found in the cache dir:" + echo $sstate_suffixes + + echo "Figuring out the archs in the sstate cache dir ... " + # Using this SSTATE_PKGSPEC definition it's 6th colon separated field + # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" + for arch in $all_archs; do + grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list + [ $? -eq 0 ] && ava_archs="$ava_archs $arch" + # ${builder_arch}_$arch used by toolchain sstate + grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list + [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch" + done + echo "Done" + echo "The following archs have been found in the cache dir:" + echo $ava_archs + echo "" + + # Save the file list which needs to be removed + local remove_listdir=`mktemp -d` || exit 1 + for suffix in $sstate_suffixes; do + if [ "$suffix" = "populate_lic" ] ; then + echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)" + continue + fi + # Total number of files including .siginfo and .done files + total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null` + total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null` + # Save the file list to a file, some suffix's file may not exist + grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null + local deleted_tgz=0 + local deleted_files=0 + for ext in tgz tgz.siginfo tgz.done; do + echo "Figuring out the sstate:xxx_$suffix.$ext ... " + # Uniq BPNs + file_names=`for arch in $ava_archs ""; do + sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix + done | sort -u` + + fn_tmp=`mktemp` || exit 1 + rm_list="$remove_listdir/sstate:xxx_$suffix" + for fn in $file_names; do + [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}" + for arch in $ava_archs ""; do + grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp + if [ -s $fn_tmp ] ; then + [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp + # Use the modification time + to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p') + [ $debug -gt 2 ] && echo "Considering to delete: $to_del" + # The sstate file which is downloaded from the SSTATE_MIRROR is + # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to + # it, so filter it out from the remove list if it should not be + # removed. + to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p') + [ $debug -gt 2 ] && echo "Considering to keep: $to_keep" + for k in $to_keep; do + if [ -L "$k" ]; then + # The symlink's destination + k_dest="`readlink -e $k`" + # Maybe it is the one in cache_dir + k_maybe="$cache_dir/${k##/*/}" + # Remove it from the remove list if they are the same. + if [ "$k_dest" = "$k_maybe" ]; then + to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`" + fi + fi + done + rm -f $fn_tmp + [ $debug -gt 2 ] && echo "Decided to delete: $to_del" + gen_rmlist $rm_list.$ext "$to_del" + fi + done + done + done + deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l` + deleted_files=`cat $rm_list.* 2>/dev/null | wc -l` + [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.* + echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)" + let total_deleted=$total_deleted+$deleted_files + done + deleted_tgz=0 + rm_old_list=$remove_listdir/sstate-old-filenames + find $cache_dir -name 'sstate-*.tgz' >$rm_old_list + [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l` + [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l` + [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list + echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)" + let total_deleted=$total_deleted+$deleted_files + + rm -f $list_suffix + rm -f $sstate_files_list + if [ $total_deleted -gt 0 ]; then + read_confirm + if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then + for list in `ls $remove_listdir/`; do + echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... " + # Remove them one by one to avoid the argument list too long error + for i in `cat $remove_listdir/$list`; do + rm -f $verbose $i + done + echo "Done" + done + echo "$total_deleted files have been removed!" + else + do_nothing + fi + else + no_files + fi + [ -d $remove_listdir ] && rm -fr $remove_listdir +} + +# Remove the sstate file by stamps dir, the file not used by the stamps dir +# will be removed. +rm_by_stamps (){ + + local cache_list=`mktemp` || exit 1 + local keep_list=`mktemp` || exit 1 + local rm_list=`mktemp` || exit 1 + local sums + local all_sums + + # Total number of files including sstate-, .siginfo and .done files + total_files=`find $cache_dir -type f -name 'sstate*' | wc -l` + # Save all the state file list to a file + find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list + + echo "Figuring out the suffixes in the sstate cache dir ... " + local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`" + echo "Done" + echo "The following suffixes have been found in the cache dir:" + echo $sstate_suffixes + + # Figure out all the md5sums in the stamps dir. + echo "Figuring out all the md5sums in stamps dir ... " + for i in $sstate_suffixes; do + # There is no "\.sigdata" but "_setcene" when it is mirrored + # from the SSTATE_MIRRORS, use them to figure out the sum. + sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \ + -o -name "*.do_${i}_setscene.*" | \ + sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \ + sed -e 's#\..*##' | sort -u` + all_sums="$all_sums $sums" + done + echo "Done" + + echo "Figuring out the files which will be removed ... " + for i in $all_sums; do + grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list + done + echo "Done" + + if [ -s $keep_list ]; then + sort -u $keep_list -o $keep_list + to_del=`comm -1 -3 $keep_list $cache_list` + gen_rmlist $rm_list "$to_del" + let total_deleted=`cat $rm_list | sort -u | wc -w` + if [ $total_deleted -gt 0 ]; then + [ $debug -gt 0 ] && cat $rm_list | sort -u + read_confirm + if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then + echo "Removing sstate cache files ... ($total_deleted files)" + # Remove them one by one to avoid the argument list too long error + for i in `cat $rm_list | sort -u`; do + rm -f $verbose $i + done + echo "$total_deleted files have been removed" + else + do_nothing + fi + else + no_files + fi + else + echo_error "All files in cache dir will be removed! Abort!" + fi + + rm -f $cache_list + rm -f $keep_list + rm -f $rm_list +} + +# Parse arguments +while [ -n "$1" ]; do + case $1 in + --cache-dir=*) + cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e` + [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir" + shift + ;; + --remove-duplicated|-d) + rm_duplicated="y" + shift + ;; + --yes|-y) + confirm="y" + shift + ;; + --follow-symlink|-L) + fsym="y" + shift + ;; + --extra-archs=*) + extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'` + [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter" + shift + ;; + --extra-layer=*) + extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'` + [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter" + for i in $extra_layers; do + l=`readlink -e $i` + if [ -d "$l" ]; then + layers="$layers $l" + else + echo_error "Can't find layer $i" + fi + done + shift + ;; + --stamps-dir=*) + stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'` + [ -n "$stamps" ] || echo_error "Invalid stamps dir $i" + for i in $stamps; do + [ -d "$i" ] || echo_error "Invalid stamps dir $i" + done + shift + ;; + --verbose|-v) + verbose="-v" + shift + ;; + --debug|-D) + debug=`expr $debug + 1` + echo "Debug level $debug" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Invalid arguments $*" + echo_error "Try 'sstate-cache-management.sh -h' for more information." + ;; + esac +done + +# sstate cache directory, use environment variable SSTATE_CACHE_DIR +# if it was not specified, otherwise, error. +[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR +[ -n "$cache_dir" ] || echo_error "No cache dir found!" +[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\"" + +[ -n "$rm_duplicated" -a -n "$stamps" ] && \ + echo_error "Can not use both --remove-duplicated and --stamps-dir" + +[ "$rm_duplicated" = "y" ] && remove_duplicated +[ -n "$stamps" ] && rm_by_stamps +[ -z "$rm_duplicated" -a -z "$stamps" ] && \ + echo "What do you want to do?" +exit 0 |