From ee38cb5a4a60471d1e18532b43332b6a42fe3e02 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 18 Oct 2007 10:22:29 +0000 Subject: source-checker: small Python script to check local sources against conf/checksums.ini --- contrib/source-checker/oe-source-checker.py | 72 +++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 contrib/source-checker/oe-source-checker.py (limited to 'contrib/source-checker/oe-source-checker.py') diff --git a/contrib/source-checker/oe-source-checker.py b/contrib/source-checker/oe-source-checker.py new file mode 100644 index 0000000000..920ac9cd6c --- /dev/null +++ b/contrib/source-checker/oe-source-checker.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et + +# Copyright (C) 2007 OpenedHand +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# +# OpenEmbedded source checksums checker +# +# This script parse conf/checksums.ini and check does files contained in +# source archive are the same as the one used to generate checksums. +# +# Run it: +# +# oe-source-checker.py path-to-conf/checksums.ini path-to-sources +# +# +# How to deal with output: +# +# 1. download each failed entry +# 2. check md5 and sha256 of file +# 3. correct checksums.ini if needed +# 4. share changes +# + + +import sys, ConfigParser + +checksums_parser = ConfigParser.ConfigParser() +checksums_parser.read(sys.argv[1]) + +for source in checksums_parser.sections(): + archive = source.split("/")[-1] + localpath = sys.argv[2] + "/" + archive + md5 = checksums_parser.get(source, "md5") + sha = checksums_parser.get(source, "sha256") + + try: + os.stat(localpath) + try: + md5pipe = os.popen('md5sum ' + localpath) + md5data = (md5pipe.readline().split() or [ "" ])[0] + md5pipe.close() + except OSError: + raise Exception("Executing md5sum failed") + + try: + shapipe = os.popen("oe_sha256sum " + localpath) + shadata = (shapipe.readline().split() or [ "" ])[0] + shapipe.close() + except OSError: + raise Exception("Executing shasum failed") + + if md5 != md5data: + print "%s has wrong md5: %s instead of %s url: %s" % (archive, md5data, md5, source) + + if sha != shadata: + print "%s has wrong sha: %s instead of %s url: %s" % (archive, shadata, sha, source) + except: + pass -- cgit v1.2.3 From adcc4854f728186b90f4d09184a3d9ec5e321b37 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 18 Oct 2007 11:15:45 +0000 Subject: oe-source-checker: some improvements (usage, less exceptions) Calling with less then 2 parameters == usage info. Simplified code a bit - os.popen() does not generate exceptions if shell can not find command. --- contrib/source-checker/oe-source-checker.py | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'contrib/source-checker/oe-source-checker.py') diff --git a/contrib/source-checker/oe-source-checker.py b/contrib/source-checker/oe-source-checker.py index 920ac9cd6c..991139d44d 100644 --- a/contrib/source-checker/oe-source-checker.py +++ b/contrib/source-checker/oe-source-checker.py @@ -36,37 +36,46 @@ # -import sys, ConfigParser +import sys + +if len(sys.argv) < 3: + print """ + OpenEmbedded source checker script require two arguments: + + 1. location of conf/checksums.ini + 2. path to DL_DIR (without "/" at the end) + """ + sys.exit(0) + +import ConfigParser, os checksums_parser = ConfigParser.ConfigParser() checksums_parser.read(sys.argv[1]) for source in checksums_parser.sections(): archive = source.split("/")[-1] - localpath = sys.argv[2] + "/" + archive + localpath = os.path.join(sys.argv[2], archive) md5 = checksums_parser.get(source, "md5") sha = checksums_parser.get(source, "sha256") try: os.stat(localpath) - try: - md5pipe = os.popen('md5sum ' + localpath) - md5data = (md5pipe.readline().split() or [ "" ])[0] - md5pipe.close() - except OSError: - raise Exception("Executing md5sum failed") + except: + continue - try: - shapipe = os.popen("oe_sha256sum " + localpath) - shadata = (shapipe.readline().split() or [ "" ])[0] - shapipe.close() - except OSError: - raise Exception("Executing shasum failed") + try: + md5pipe = os.popen('md5sum ' + localpath) + md5data = (md5pipe.readline().split() or [ "" ])[0] + md5pipe.close() if md5 != md5data: print "%s has wrong md5: %s instead of %s url: %s" % (archive, md5data, md5, source) - if sha != shadata: + shapipe = os.popen("oe_sha256sum " + localpath) + shadata = (shapipe.readline().split() or [ "" ])[0] + shapipe.close() + + if shadata != "" and sha != shadata: print "%s has wrong sha: %s instead of %s url: %s" % (archive, shadata, sha, source) except: pass -- cgit v1.2.3 From 39ff8d86e924cc56ac94b7c07cf2b5f134c11748 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 18 Oct 2007 13:04:40 +0000 Subject: source-checker: added progressbar (from BitBake) and status info at end --- contrib/source-checker/oe-source-checker.py | 31 ++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) (limited to 'contrib/source-checker/oe-source-checker.py') diff --git a/contrib/source-checker/oe-source-checker.py b/contrib/source-checker/oe-source-checker.py index 991139d44d..45ddad9997 100644 --- a/contrib/source-checker/oe-source-checker.py +++ b/contrib/source-checker/oe-source-checker.py @@ -47,35 +47,60 @@ if len(sys.argv) < 3: """ sys.exit(0) -import ConfigParser, os +import ConfigParser, os, itertools checksums_parser = ConfigParser.ConfigParser() checksums_parser.read(sys.argv[1]) +parsespin = itertools.cycle( r'|/-\\' ) + +item = 1; +files_total = len(checksums_parser.sections()) +files_checked = 0 +files_good = 0 +files_wrong = 0 + for source in checksums_parser.sections(): archive = source.split("/")[-1] localpath = os.path.join(sys.argv[2], archive) md5 = checksums_parser.get(source, "md5") sha = checksums_parser.get(source, "sha256") + if os.isatty(sys.stdout.fileno()): + sys.stdout.write("\rChecking files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), item, files_total, item*100/files_total ) ) + sys.stdout.flush() + item += 1 + try: os.stat(localpath) except: continue + files_checked += 1 + file_ok = True + try: md5pipe = os.popen('md5sum ' + localpath) md5data = (md5pipe.readline().split() or [ "" ])[0] md5pipe.close() if md5 != md5data: - print "%s has wrong md5: %s instead of %s url: %s" % (archive, md5data, md5, source) + file_ok = False + print "\n%s has wrong md5: %s instead of %s url: %s" % (archive, md5data, md5, source) shapipe = os.popen("oe_sha256sum " + localpath) shadata = (shapipe.readline().split() or [ "" ])[0] shapipe.close() if shadata != "" and sha != shadata: - print "%s has wrong sha: %s instead of %s url: %s" % (archive, shadata, sha, source) + file_ok = False + print "\n%s has wrong sha: %s instead of %s url: %s" % (archive, shadata, sha, source) + + if file_ok: + files_good += 1 + else: + files_wrong += 1 except: pass + +print "\nChecked %d files. %d was OK and %d had wrong checksums." % (files_checked, files_good, files_wrong) -- cgit v1.2.3 From 5dfeeef24d1802c6590f7943ffc6db6c5265985f Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 18 Oct 2007 13:37:57 +0000 Subject: source-checker: move bad news to end of program (list of errors when program end) --- contrib/source-checker/oe-source-checker.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'contrib/source-checker/oe-source-checker.py') diff --git a/contrib/source-checker/oe-source-checker.py b/contrib/source-checker/oe-source-checker.py index 45ddad9997..9ac147b00b 100644 --- a/contrib/source-checker/oe-source-checker.py +++ b/contrib/source-checker/oe-source-checker.py @@ -58,7 +58,7 @@ item = 1; files_total = len(checksums_parser.sections()) files_checked = 0 files_good = 0 -files_wrong = 0 +files_wrong = [] for source in checksums_parser.sections(): archive = source.split("/")[-1] @@ -86,7 +86,6 @@ for source in checksums_parser.sections(): if md5 != md5data: file_ok = False - print "\n%s has wrong md5: %s instead of %s url: %s" % (archive, md5data, md5, source) shapipe = os.popen("oe_sha256sum " + localpath) shadata = (shapipe.readline().split() or [ "" ])[0] @@ -94,13 +93,21 @@ for source in checksums_parser.sections(): if shadata != "" and sha != shadata: file_ok = False - print "\n%s has wrong sha: %s instead of %s url: %s" % (archive, shadata, sha, source) if file_ok: files_good += 1 else: - files_wrong += 1 + files_wrong += [ [md5, md5data, sha, shadata, archive, source] ] except: pass -print "\nChecked %d files. %d was OK and %d had wrong checksums." % (files_checked, files_good, files_wrong) +print "\nChecked %d files. %d was OK and %d had wrong checksums." % (files_checked, files_good, len(files_wrong)) + +if len(files_wrong) > 0: + + print "\n" + + for wrong_file in files_wrong: + print "%s [%s] is wrong" % ( wrong_file[4], wrong_file[5]) + print "md5: %s instead of %s" % (wrong_file[1], wrong_file[0]) + print "sha: %s instead of %s" % (wrong_file[3], wrong_file[2]) -- cgit v1.2.3