From f995e0a4f5a60f4a1b80449d8eb7f64d0efebfef Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Sat, 31 Mar 2007 20:15:44 +0000 Subject: classes/base.bbclass: Add checksum.py code to verify md5/sha256 sums on do_fetch This code uses the checksum.py from contrib/qa/checksum to verify md5sum and sha256 on do_fetch task. This code would even allow to checksum certain local patches. --- classes/base.bbclass | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) (limited to 'classes/base.bbclass') diff --git a/classes/base.bbclass b/classes/base.bbclass index 53139e19fa..1f1766cf11 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -10,6 +10,60 @@ def base_path_join(a, *p): path += '/' + b return path +# for MD5/SHA handling +def base_chk_load_parser(config_path): + import ConfigParser, os + parser = ConfigParser.ConfigParser() + if not len(parser.read(config_path)) == 1: + raise Exception("Can not open the '%s'" % config_path) + + return parser + +def base_chk_file(parser, pn, pv, src_uri, localpath): + import os + # Try PN-PV-SRC_URI first and then try PN-SRC_URI + # we rely on the get method to create errors + pn_pv_src = "%s-%s-%s" % (pn,pv,src_uri) + pn_src = "%s-%s" % (pn,src_uri) + if parser.has_section(pn_pv_src): + md5 = parser.get(pn_pv_src, "md5") + sha256 = parser.get(pn_pv_src, "sha256") + elif parser.has_section(pn_src): + md5 = parser.get(pn_src, "md5") + sha256 = parser.get(pn_src, "sha256") + else: + return False + #raise Exception("Can not find a section for '%s' '%s' and '%s'" % (pn,pv,src_uri)) + + # md5 and sha256 should be valid now + if not os.path.exists(localpath): + raise Exception("The path does not exist '%s'" % localpath) + + + # call md5(sum) and shasum + try: + md5pipe = os.popen('md5sum ' + localpath) + md5data = (md5pipe.readline().split() or [ "" ])[0] + md5pipe.close() + except OSError: + raise Exception("Executing md5sum failed") + + try: + shapipe = os.popen('shasum -a256 -p ' + localpath) + shadata = (shapipe.readline().split() or [ "" ])[0] + shapipe.close() + except OSError: + raise Exception("Executing shasum failed") + + if not md5 == md5data: + raise Exception("MD5 Sums do not match. Wanted: '%s' Got: '%s'" % (md5, md5data)) + + if not sha256 == shadata: + raise Exception("SHA256 Sums do not match. Wanted: '%s' Got: '%s'" % (sha256, shadata)) + + return True + + def base_dep_prepend(d): import bb; # @@ -402,6 +456,41 @@ python base_do_fetch() { except bb.fetch.FetchError: (type, value, traceback) = sys.exc_info() raise bb.build.FuncFailed("Fetch failed: %s" % value) + except bb.fetch.MD5SumError: + (type, value, traceback) = sys.exc_info() + raise bb.build.FuncFailed("MD5 failed: %s" % value) + except: + (type, value, traceback) = sys.exc_info() + raise bb.build.FuncFailed("Unknown fetch Error: %s" % value) + + + # Verify the SHA and MD5 sums we have in OE and check what do + # in + check_sum = bb.which(bb.data.getVar('BBPATH', d, True), "conf/checksums.ini") + if not check_sum: + bb.note("No conf/checksums.ini found, not checking checksums") + return + + try: + parser = base_chk_load_parser(ckeck_sum) + except: + bb.note("Creating the CheckSum parser failed") + return + + pv = bb.data.getVar('PV', d, True) + pn = bb.data.getVar('PN', d, True) + + # Check each URI + for url in src_uri.split(): + localpath = bb.fetch.localpath(url,localdata) + (type,host,path,_,_,_) = bb.decodeurl(url) + print type, host, path + uri = "%s://%s%s" % (type,host,path) + try: + if not base_chk_file(parser, pn, pv,uri, localpath): + bb.note("%s-%s-%s has no section, not checking URI" % pn,pv,uri) + except Exception, e: + raise bb.func.FuncFailed("Checksum of '%s' failed", uri) } addtask fetchall after do_fetch -- cgit v1.2.3 From 8e2d1b42f56902f87817efed2a8df169d31014a4 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Sat, 31 Mar 2007 20:31:20 +0000 Subject: classes/base.bbclass: Make the checksum code work and test it with file-native conf/checksums.ini: Provide an example on how this can be used --- classes/base.bbclass | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'classes/base.bbclass') diff --git a/classes/base.bbclass b/classes/base.bbclass index 1f1766cf11..880295fc30 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -12,15 +12,16 @@ def base_path_join(a, *p): # for MD5/SHA handling def base_chk_load_parser(config_path): - import ConfigParser, os + import ConfigParser, os, bb parser = ConfigParser.ConfigParser() if not len(parser.read(config_path)) == 1: + bb.note("Can not open the '%s' ini file" % config_path) raise Exception("Can not open the '%s'" % config_path) return parser def base_chk_file(parser, pn, pv, src_uri, localpath): - import os + import os, bb # Try PN-PV-SRC_URI first and then try PN-SRC_URI # we rely on the get method to create errors pn_pv_src = "%s-%s-%s" % (pn,pv,src_uri) @@ -37,6 +38,7 @@ def base_chk_file(parser, pn, pv, src_uri, localpath): # md5 and sha256 should be valid now if not os.path.exists(localpath): + bb.note("The locapath does not exist '%s'" % localpath) raise Exception("The path does not exist '%s'" % localpath) @@ -56,9 +58,11 @@ def base_chk_file(parser, pn, pv, src_uri, localpath): raise Exception("Executing shasum failed") if not md5 == md5data: + bb.note("The MD5Sums did not match. Wanted: '%s' and Got: '%s'" % (md5,md5data)) raise Exception("MD5 Sums do not match. Wanted: '%s' Got: '%s'" % (md5, md5data)) if not sha256 == shadata: + bb.note("The SHA256 Sums do not match. Wanted: '%s' Got: '%s'" % (sha256,shadata)) raise Exception("SHA256 Sums do not match. Wanted: '%s' Got: '%s'" % (sha256, shadata)) return True @@ -472,7 +476,7 @@ python base_do_fetch() { return try: - parser = base_chk_load_parser(ckeck_sum) + parser = base_chk_load_parser(check_sum) except: bb.note("Creating the CheckSum parser failed") return @@ -484,13 +488,12 @@ python base_do_fetch() { for url in src_uri.split(): localpath = bb.fetch.localpath(url,localdata) (type,host,path,_,_,_) = bb.decodeurl(url) - print type, host, path uri = "%s://%s%s" % (type,host,path) try: - if not base_chk_file(parser, pn, pv,uri, localpath): - bb.note("%s-%s-%s has no section, not checking URI" % pn,pv,uri) - except Exception, e: - raise bb.func.FuncFailed("Checksum of '%s' failed", uri) + if not base_chk_file(parser, pn, pv,uri, localpath): + bb.note("%s-%s-%s has no section, not checking URI" % (pn,pv,uri)) + except Exception: + raise bb.build.FuncFailed("Checksum of '%s' failed" % uri) } addtask fetchall after do_fetch -- cgit v1.2.3 From afc9dea14ffb67902060f32fda79290727b74bec Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Sun, 1 Apr 2007 13:20:07 +0000 Subject: classes/base.bbclass: [ftp://ftp.trolltech.com/qt/source/qtopia-core-4.3.0beta.tar.gz] is good enough src_uri as section/groups are unique enough as well. Most of the time (almost always) scoping PN (or PN+PV) is not necessary and in this case the SRC_URI is good enough. Richard thank you for saying the obvious --- classes/base.bbclass | 3 +++ 1 file changed, 3 insertions(+) (limited to 'classes/base.bbclass') diff --git a/classes/base.bbclass b/classes/base.bbclass index 880295fc30..2ea5251609 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -32,6 +32,9 @@ def base_chk_file(parser, pn, pv, src_uri, localpath): elif parser.has_section(pn_src): md5 = parser.get(pn_src, "md5") sha256 = parser.get(pn_src, "sha256") + elif parser.has_section(src_uri): + md5 = parser.get(src_uri, "md5") + sha256 = parser.get(src_uri, "sha256") else: return False #raise Exception("Can not find a section for '%s' '%s' and '%s'" % (pn,pv,src_uri)) -- cgit v1.2.3 From 7870e6d52d38140b0789176e98aa658f8e52ed5a Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Sun, 1 Apr 2007 21:42:48 +0000 Subject: base.bbclass, sanity.bbclass: use sha256sum instead of shasum -a256 --- classes/base.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'classes/base.bbclass') diff --git a/classes/base.bbclass b/classes/base.bbclass index 2ea5251609..6f8196cf80 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -54,7 +54,7 @@ def base_chk_file(parser, pn, pv, src_uri, localpath): raise Exception("Executing md5sum failed") try: - shapipe = os.popen('shasum -a256 -p ' + localpath) + shapipe = os.popen('sha256sum -b ' + localpath) shadata = (shapipe.readline().split() or [ "" ])[0] shapipe.close() except OSError: -- cgit v1.2.3 From 745c67ffc5c79fe08cd5460ded6317d17517799a Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Mon, 2 Apr 2007 20:42:15 +0000 Subject: classes/base.bbclass: place shasum-native as the first package to build classes/base.bbclass: Make sure shasum-native does not depend on quilt classes/sanity.bbclass: No need to require sha256sum as we use oe_sha256sum packages/shasum: Change the name, make the -native version not have any deps --- classes/base.bbclass | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'classes/base.bbclass') diff --git a/classes/base.bbclass b/classes/base.bbclass index 6f8196cf80..6011790ebe 100644 --- a/classes/base.bbclass +++ b/classes/base.bbclass @@ -20,7 +20,7 @@ def base_chk_load_parser(config_path): return parser -def base_chk_file(parser, pn, pv, src_uri, localpath): +def base_chk_file(parser, pn, pv, src_uri, localpath, data): import os, bb # Try PN-PV-SRC_URI first and then try PN-SRC_URI # we rely on the get method to create errors @@ -54,7 +54,7 @@ def base_chk_file(parser, pn, pv, src_uri, localpath): raise Exception("Executing md5sum failed") try: - shapipe = os.popen('sha256sum -b ' + localpath) + shapipe = os.popen('PATH=%s oe_sha256sum %s' % (bb.data.getVar('PATH', data, True), localpath)) shadata = (shapipe.readline().split() or [ "" ])[0] shapipe.close() except OSError: @@ -78,16 +78,22 @@ def base_dep_prepend(d): # the case where host == build == target, for now we don't work in # that case though. # - deps = "" + if bb.data.getVar('PN', d, True) == "shasum-native": + deps = "" + else: + deps = "shasum-native " + + # INHIBIT_PATCH_TOOL don't apply the patch tool dependency + inhibit_patch = (bb.data.getVar("INHIBIT_PATCH_TOOL", d, True) == "1") or False # INHIBIT_DEFAULT_DEPS doesn't apply to the patch command. Whether or not # we need that built is the responsibility of the patch function / class, not # the application. patchdeps = bb.data.getVar("PATCHTOOL", d, 1) - if patchdeps: + if patchdeps and not inhibit_patch: patchdeps = "%s-native" % patchdeps if not patchdeps in bb.data.getVar("PROVIDES", d, 1): - deps = patchdeps + deps += patchdeps if not bb.data.getVar('INHIBIT_DEFAULT_DEPS', d): if (bb.data.getVar('HOST_SYS', d, 1) != @@ -439,6 +445,7 @@ python base_do_mrproper() { addtask fetch do_fetch[dirs] = "${DL_DIR}" +do_fetch[depends] = "shasum-native:do_populate_staging" python base_do_fetch() { import sys @@ -493,7 +500,7 @@ python base_do_fetch() { (type,host,path,_,_,_) = bb.decodeurl(url) uri = "%s://%s%s" % (type,host,path) try: - if not base_chk_file(parser, pn, pv,uri, localpath): + if not base_chk_file(parser, pn, pv,uri, localpath, d): bb.note("%s-%s-%s has no section, not checking URI" % (pn,pv,uri)) except Exception: raise bb.build.FuncFailed("Checksum of '%s' failed" % uri) -- cgit v1.2.3