From c2308c2dea77248652592b102358f67ba646a2fd Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Mon, 5 Mar 2007 20:02:32 +0000 Subject: image.bbclass: Create symlink to latest image generated. Also add dependency on populate staging for do_rootfs (both from poky) --- classes/image.bbclass | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'classes') diff --git a/classes/image.bbclass b/classes/image.bbclass index 69d7fb2e9b..83aa80a97e 100644 --- a/classes/image.bbclass +++ b/classes/image.bbclass @@ -4,7 +4,7 @@ PACKAGES = "" # We need to recursively follow RDEPENDS and RRECOMMENDS for images BUILD_ALL_DEPS = "1" -do_rootfs[recrdeptask] = "do_package_write do_deploy" +do_rootfs[recrdeptask] = "do_package_write do_deploy do_populate_staging" # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1" @@ -91,6 +91,10 @@ fakeroot do_rootfs () { else bbimage -n "${IMAGE_NAME}" -t "$type" -e "${FILE}" fi + + cd ${DEPLOY_DIR_IMAGE}/ + rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.* + ln -s ${IMAGE_NAME}.rootfs.$type ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.$type done ${IMAGE_POSTPROCESS_COMMAND} -- cgit v1.2.3 From 5b98bc859513b2ba2f1ccb58f187ec193233d428 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Tue, 6 Mar 2007 14:44:27 +0000 Subject: classes/seppuku.bbclass: Add a class which might report build issues to a bugzilla directly Start adding seppuku. It listens to TaskFailed and NoProvider failures logs into a bugzilla and calls empty stubs to find a bug report. We use the urllib2 and cookielib from python2.4 to handle the session cookie for us automatically. This library is sadly not present in python2.3 but it eases my life. The old BugZilla installed at OpenMoko has no support for listing the bug report as csv or xml file, this will force me to parse the HTML which I will do for now. This assumes that only one bug report with the exact name can exist. --- classes/seppuku.bbclass | 119 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 classes/seppuku.bbclass (limited to 'classes') diff --git a/classes/seppuku.bbclass b/classes/seppuku.bbclass new file mode 100644 index 0000000000..b3e3fcb8c2 --- /dev/null +++ b/classes/seppuku.bbclass @@ -0,0 +1,119 @@ +# +# Small event handler to automatically open URLs and file +# bug reports at a bugzilla of your choiche +# +# This class requires python2.4 because of the urllib2 usage +# + + +def seppuku_login(opener, login, user, password): + """ + We need to post to query.cgi with the parameters + Bugzilla_login and Bugzilla_password and will scan + the resulting page then + + @param opened = cookie enabled urllib2 opener + @param login = http://bugzilla.openmoko.org/cgi-bin/bugzilla/query.cgi? + @param user = Your username + @param pass = Your password + """ + import urllib + param = urllib.urlencode( {"GoAheadAndLogIn" : 1, "Bugzilla_login" : user, "Bugzilla_password" : password } ) + result = opener.open(login + param) + + if result.code != 200: + return False + txt = result.read() + if not 'Log out' in txt: + return False + + return True + + +def seppuku_find_bug_report(opener, query, product, component, bugname): + """ + Find a bug report with the sane name and return the bug id + and the status. + + @param opener = urllib2 opener + @param query = e.g. https://bugzilla.openmoko.org/cgi-bin/bugzilla/query.cgi? + @param product = search for this product + @param component = search for this component + @param bugname = the bug to search for + + https://bugzilla.openmoko.org/cgi-bin/bugzilla/buglist.cgi?short_desc_type=substring&short_desc=manual+test+bug&product=OpenMoko&emailreporter2=1&emailtype2=substring&email2=freyther%40yahoo.com + but it does not support ctype=csv... + """ + pass + +def seppuku_append_or_file(opener, file, product, component, bugname, text): + """ + Add a comment or open a bug report + """ + + + +addhandler seppuku_do_report +python seppuku_do_report() { + """ + Report task failures to the bugzilla + and succeeded builds to the box + """ + from bb.event import NotHandled, getName + from bb import data, mkdirhier, build + import bb, os, glob + + try: + import urllib2, cookielib + except: + bb.note("Failed to import the cookielib and urllib2, make sure to use python2.4") + return NotHandled + + event = e + data = e.data + name = getName(event) + if name == "PkgFailed": + if not data.getVar('SEPPUKU_AUTOBUILD', data, True) == "0": + build.exec_task('do_clean', data) + elif name == "TaskFailed" or name == "NoProvider": + cj = cookielib.CookieJar() + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) + login = bb.data.getVar("SEPPUKU_LOGIN", data, True) + query = bb.data.getVar("SEPPUKU_QUERY", data, True) + file = bb.data.getVar("SEPPUKU_FILE", data, True) + post = bb.data.getVar("SEPPUKU_POST", data, True) + user = bb.data.getVar("SEPPUKU_USER", data, True) + pass = bb.data.getVar("SEPPUKU_PASS", data, True) + product = bb.data.getVar("SEPPUKU_PRODUCT", data, True) + component = bb.data.getVar("SEPPUKU_COMPONENT", data, True) + + if not seppuku_login(opener, login, user, pass): + bb.note("Login to bugzilla failed") + return NotHandled + + if name == "TaskFailed": + bugname = "%(package)s-%(pv)s-%(pr)s-%(task)s" % { "package" : bb.data.getVar("PN", data, True), + "pv" : bb.data.getVar("PV", data, True), + "pr" : bb.data.getVar("PR", data, True), + "task" : e.task } + log_file = glob.glob("%s/log.%s.*" % (data.getVar('T', event.data, True), event.task)) + if len(log_file) != 0: + to_file = data.getVar('TINDER_LOG', event.data, True) + text = "".join(open(log_file[0], 'r').readlines()) + elif name == "NoProvider": + bugname = "noprovider for %s runtime: %s" % (event.getItem, event.getisRuntime) + text = "Please fix it" + else + assert False + + (bug_open, bug_number) = seppuku_find_bug_report(opener, query, product, component, bugname) + + # The bug is present and still open, no need to attach an error log + if bug_nunumber and bug_open: + return NotHandled + + + if seppuku_append_or_file(opener, file, produuct, component, bugname, text) + bb.note("Filing a bugreport failed") + + return NotHandled -- cgit v1.2.3 From bea4a815fb63dd0bdf329c7bfff7a5e19ef60dfc Mon Sep 17 00:00:00 2001 From: Florian Boor Date: Tue, 6 Mar 2007 15:56:42 +0000 Subject: image.bbclass: Add function to turn on autologin for device images we do not want to log in at all. --- classes/image.bbclass | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'classes') diff --git a/classes/image.bbclass b/classes/image.bbclass index 83aa80a97e..5055b5b987 100644 --- a/classes/image.bbclass +++ b/classes/image.bbclass @@ -165,7 +165,14 @@ make_zimage_symlink_relative () { fi } +# Make login manager(s) enable automatic login. +# Useful for devices where we do not want to log in at all (e.g. phones) +set_image_autologin () { + sed -i 's%^AUTOLOGIN=\"false"%AUTOLOGIN="true"%g' ${IMAGE_ROOTFS}/etc/sysconfig/gpelogin +} + + # export the zap_root_password, create_etc_timestamp and remote_init_link -EXPORT_FUNCTIONS zap_root_password create_etc_timestamp remove_init_link do_rootfs make_zimage_symlink_relative +EXPORT_FUNCTIONS zap_root_password create_etc_timestamp remove_init_link do_rootfs make_zimage_symlink_relative set_image_autologin addtask rootfs before do_build after do_install -- cgit v1.2.3 From ec2f31c4c67656bb4b51384954baece702983cfc Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Tue, 6 Mar 2007 20:20:42 +0000 Subject: classes/seppuku.bbclass: Add the BugQuery code, add code to post a bug Posting bugs should work (besides typos), now only reopening bugs is missing. --- classes/seppuku.bbclass | 103 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 7 deletions(-) (limited to 'classes') diff --git a/classes/seppuku.bbclass b/classes/seppuku.bbclass index b3e3fcb8c2..4a3eced73d 100644 --- a/classes/seppuku.bbclass +++ b/classes/seppuku.bbclass @@ -29,6 +29,62 @@ def seppuku_login(opener, login, user, password): return True +def seppuku_find_bug_report_old(): + from HTMLParser import HTMLParser + + class BugQueryExtractor(HTMLParser): + STATE_NONE = 0 + STATE_FOUND_TR = 1 + STATE_FOUND_NUMBER = 2 + STATE_FOUND_PRIO = 3 + STATE_FOUND_PRIO2 = 4 + STATE_FOUND_NAME = 5 + STATE_FOUND_PLATFORM = 6 + STATE_FOUND_STATUS = 7 + STATE_FOUND_WHATEVER = 8 # I don't know this field + STATE_FOUND_DESCRIPTION =9 + + def __init__(self): + HTMLParser.__init__(self) + self.state = self.STATE_NONE + self.bugs = [] + + def handle_starttag(self, tag, attr): + if self.state == self.STATE_NONE and tag.lower() == "tr": + if len(attr) == 1 and attr[0] == ('class', 'bz_normal bz_P2 '): + print "Found tr %s %s" % (tag, attr) + self.state = self.STATE_FOUND_TR + elif self.state == self.STATE_FOUND_TR and tag.lower() == "td": + self.state += 1 + + def handle_endtag(self, tag): + if tag.lower() == "tr": + print "Going back" + if self.state != self.STATE_NONE: + self.bugs.append( (self.bug,self.status) ) + self.state = self.STATE_NONE + if self.state > 1 and tag.lower() == "td": + print "Next TD" + self.state += 1 + + def handle_data(self,data): + data = data.strip() + + # skip garbage + if len(data) == 0: + return + + if self.state == self.STATE_FOUND_NUMBER: + self.bug = data + elif self.state == self.STATE_FOUND_STATUS: + self.status = data + + def result(self): + return self.bugs + + return BugQueryExtractor() + + def seppuku_find_bug_report(opener, query, product, component, bugname): """ @@ -44,13 +100,43 @@ def seppuku_find_bug_report(opener, query, product, component, bugname): https://bugzilla.openmoko.org/cgi-bin/bugzilla/buglist.cgi?short_desc_type=substring&short_desc=manual+test+bug&product=OpenMoko&emailreporter2=1&emailtype2=substring&email2=freyther%40yahoo.com but it does not support ctype=csv... """ - pass - -def seppuku_append_or_file(opener, file, product, component, bugname, text): + result = opener.open("%(query)s?product=%(product)s&component=%(component)s&short_desc_type=substring&short_desc=%(bugname)s" % vars()) + if result.code != 200: + raise "Can not query the bugzilla at all" + txt = result.read() + scanner = seppuku_find_bug_report_old() + scanner.feed(txt) + if len(scanner.result()) == 0: + return (False,None) + else: # silently pick the first result + (number,status) = scanner.result()[0] + return (status != "CLOSED",number) + +def seppuku_file_bug(opener, file, product, component, bugname, text): """ - Add a comment or open a bug report + Create a completely new bug report + + + http://bugzilla.openmoko.org/cgi-bin/bugzilla/post_bug.cgi?bug_file_loc=http%3A%2F%2F&version=2007&product=OpenMoko&component=autobuilds&short_desc=foo&comment=bla&priority=P2&bug_severity=normal&op_sys=Linux&rep_platform=Neo1973 + + You are forced to add some default values to the bugzilla query and stop with '&' + + @param opener urllib2 opener + @param file The url used to file a bug report + @param product Product + @param component Component + @param bugname Name of the to be created bug + @param text Text """ + import urllib + param = urllib.urlencode( { "product" : product, "component" : component, "short_desc" : bugname, "comment" : text } ) + result = opener.open( file + param ) + if result.code != 200: + return False + else: + return True + addhandler seppuku_do_report @@ -109,11 +195,14 @@ python seppuku_do_report() { (bug_open, bug_number) = seppuku_find_bug_report(opener, query, product, component, bugname) # The bug is present and still open, no need to attach an error log - if bug_nunumber and bug_open: + if bug_number and bug_open: + bb.note("The bug is known as '%s'" % bug_number" return NotHandled - - if seppuku_append_or_file(opener, file, produuct, component, bugname, text) + if bug_number and not bug_open: + if not seppuku_reopen_bug(opener, file, product, component, bug_number, bugname, text): + bb.note("Failed to reopen the bug report") + else seppuku_file_bug(opener, file, product, component, bugname, text): bb.note("Filing a bugreport failed") return NotHandled -- cgit v1.2.3 From b30c26b35a9d7b8e326b793efc43706aea2d6375 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Tue, 6 Mar 2007 23:37:35 +0000 Subject: classes/seppuku.bbclass: This could be feature complete now It is time to try it --- classes/seppuku.bbclass | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'classes') diff --git a/classes/seppuku.bbclass b/classes/seppuku.bbclass index 4a3eced73d..716fdabe0d 100644 --- a/classes/seppuku.bbclass +++ b/classes/seppuku.bbclass @@ -112,6 +112,24 @@ def seppuku_find_bug_report(opener, query, product, component, bugname): (number,status) = scanner.result()[0] return (status != "CLOSED",number) +def seppuku_reopen_bug(opener, file, product, component, bug_number, bugname, text): + """ + Reopen a bug report and append to the comment + + Same as with opening a new report, some bits need to be inside the url + + http://bugzilla.openmoko.org/cgi-bin/bugzilla/process_bug.cgi?id=239&bug_file_loc=http%3A%2F%2F&version=2007&longdesclength=2&product=OpenMoko&component=autobuilds&comment=bla&priority=P2&bug_severity=normal&op_sys=Linux&rep_platform=Neo1973&knob=reopen&target_milestone=Phase+0&short_desc=foo + """ + + import urllib + param = urllib.urlencode( { "product" : product, "component" : component, "longdesclength" : 2, + "short_desc" : bugname, "comment" : text, "knob" : "reopen" } ) + result = opener.open( file + param ) + if result.code != 200: + return False + else + return True + def seppuku_file_bug(opener, file, product, component, bugname, text): """ Create a completely new bug report -- cgit v1.2.3 From b7d3b2793c6142f295862593a1c9c3544139a79f Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Tue, 6 Mar 2007 23:56:27 +0000 Subject: conf/documentation.conf: Document the keys influencing seppuku.bbclass --- classes/seppuku.bbclass | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'classes') diff --git a/classes/seppuku.bbclass b/classes/seppuku.bbclass index 716fdabe0d..9fdcbbd236 100644 --- a/classes/seppuku.bbclass +++ b/classes/seppuku.bbclass @@ -123,7 +123,7 @@ def seppuku_reopen_bug(opener, file, product, component, bug_number, bugname, te import urllib param = urllib.urlencode( { "product" : product, "component" : component, "longdesclength" : 2, - "short_desc" : bugname, "comment" : text, "knob" : "reopen" } ) + "short_desc" : bugname, "comment" : text, "knob" : "reopen", "id" : bug_number } ) result = opener.open( file + param ) if result.code != 200: return False @@ -184,8 +184,8 @@ python seppuku_do_report() { opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) login = bb.data.getVar("SEPPUKU_LOGIN", data, True) query = bb.data.getVar("SEPPUKU_QUERY", data, True) - file = bb.data.getVar("SEPPUKU_FILE", data, True) - post = bb.data.getVar("SEPPUKU_POST", data, True) + newbug = bb.data.getVar("SEPPUKU_NEWREPORT", data, True) + reopen = bb.data.getVar("SEPPUKU_ADDCOMMENT", data, True) user = bb.data.getVar("SEPPUKU_USER", data, True) pass = bb.data.getVar("SEPPUKU_PASS", data, True) product = bb.data.getVar("SEPPUKU_PRODUCT", data, True) @@ -218,9 +218,9 @@ python seppuku_do_report() { return NotHandled if bug_number and not bug_open: - if not seppuku_reopen_bug(opener, file, product, component, bug_number, bugname, text): + if not seppuku_reopen_bug(opener, reopen, product, component, bug_number, bugname, text): bb.note("Failed to reopen the bug report") - else seppuku_file_bug(opener, file, product, component, bugname, text): + else seppuku_file_bug(opener, newbug, product, component, bugname, text): bb.note("Filing a bugreport failed") return NotHandled -- cgit v1.2.3 From deacf77e13e6f1f450d1f9e91a466c387c8494f0 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Wed, 7 Mar 2007 00:42:41 +0000 Subject: classes/seppuku.bbclass: Make it almost work This class can find the broken bug, reopen it, or leave it open. There is some issue with sending the logfile to the server, the url itself is correct, when opened in a browser the right thing happens but not from within python *sigh* need a working tcpdump to look into this (which means I will continue tomorrow) --- classes/seppuku.bbclass | 59 +++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 21 deletions(-) (limited to 'classes') diff --git a/classes/seppuku.bbclass b/classes/seppuku.bbclass index 9fdcbbd236..5757df7efb 100644 --- a/classes/seppuku.bbclass +++ b/classes/seppuku.bbclass @@ -15,7 +15,7 @@ def seppuku_login(opener, login, user, password): @param opened = cookie enabled urllib2 opener @param login = http://bugzilla.openmoko.org/cgi-bin/bugzilla/query.cgi? @param user = Your username - @param pass = Your password + @param password = Your password """ import urllib param = urllib.urlencode( {"GoAheadAndLogIn" : 1, "Bugzilla_login" : user, "Bugzilla_password" : password } ) @@ -52,19 +52,16 @@ def seppuku_find_bug_report_old(): def handle_starttag(self, tag, attr): if self.state == self.STATE_NONE and tag.lower() == "tr": if len(attr) == 1 and attr[0] == ('class', 'bz_normal bz_P2 '): - print "Found tr %s %s" % (tag, attr) self.state = self.STATE_FOUND_TR elif self.state == self.STATE_FOUND_TR and tag.lower() == "td": self.state += 1 def handle_endtag(self, tag): if tag.lower() == "tr": - print "Going back" if self.state != self.STATE_NONE: self.bugs.append( (self.bug,self.status) ) self.state = self.STATE_NONE if self.state > 1 and tag.lower() == "td": - print "Next TD" self.state += 1 def handle_data(self,data): @@ -110,7 +107,7 @@ def seppuku_find_bug_report(opener, query, product, component, bugname): return (False,None) else: # silently pick the first result (number,status) = scanner.result()[0] - return (status != "CLOSED",number) + return (not status in ["CLOS", "RESO", "VERI"],number) def seppuku_reopen_bug(opener, file, product, component, bug_number, bugname, text): """ @@ -121,13 +118,19 @@ def seppuku_reopen_bug(opener, file, product, component, bug_number, bugname, te http://bugzilla.openmoko.org/cgi-bin/bugzilla/process_bug.cgi?id=239&bug_file_loc=http%3A%2F%2F&version=2007&longdesclength=2&product=OpenMoko&component=autobuilds&comment=bla&priority=P2&bug_severity=normal&op_sys=Linux&rep_platform=Neo1973&knob=reopen&target_milestone=Phase+0&short_desc=foo """ - import urllib + import urllib, urllib2 param = urllib.urlencode( { "product" : product, "component" : component, "longdesclength" : 2, - "short_desc" : bugname, "comment" : text, "knob" : "reopen", "id" : bug_number } ) - result = opener.open( file + param ) + "short_desc" : bugname, "knob" : "reopen", "id" : bug_number, "comment" : text } ) + try: + result = opener.open( file + param ) + except urllib2.HTTPError, e: + print e.geturl() + print e.info() + return False + if result.code != 200: return False - else + else: return True def seppuku_file_bug(opener, file, product, component, bugname, text): @@ -147,9 +150,16 @@ def seppuku_file_bug(opener, file, product, component, bugname, text): @param text Text """ - import urllib + import urllib,urllib2 param = urllib.urlencode( { "product" : product, "component" : component, "short_desc" : bugname, "comment" : text } ) - result = opener.open( file + param ) + try: + result = opener.open( file + param ) + except urllib2.HTTPError, e: + print e.geturl() + print e.info() + raise e + return False + if result.code != 200: return False else: @@ -157,8 +167,8 @@ def seppuku_file_bug(opener, file, product, component, bugname, text): -addhandler seppuku_do_report -python seppuku_do_report() { +addhandler seppuku_eventhandler +python seppuku_eventhandler() { """ Report task failures to the bugzilla and succeeded builds to the box @@ -167,6 +177,8 @@ python seppuku_do_report() { from bb import data, mkdirhier, build import bb, os, glob + bb.note( "Ran" ) + try: import urllib2, cookielib except: @@ -177,7 +189,7 @@ python seppuku_do_report() { data = e.data name = getName(event) if name == "PkgFailed": - if not data.getVar('SEPPUKU_AUTOBUILD', data, True) == "0": + if not bb.data.getVar('SEPPUKU_AUTOBUILD', data, True) == "0": build.exec_task('do_clean', data) elif name == "TaskFailed" or name == "NoProvider": cj = cookielib.CookieJar() @@ -187,40 +199,45 @@ python seppuku_do_report() { newbug = bb.data.getVar("SEPPUKU_NEWREPORT", data, True) reopen = bb.data.getVar("SEPPUKU_ADDCOMMENT", data, True) user = bb.data.getVar("SEPPUKU_USER", data, True) - pass = bb.data.getVar("SEPPUKU_PASS", data, True) + passw = bb.data.getVar("SEPPUKU_PASS", data, True) product = bb.data.getVar("SEPPUKU_PRODUCT", data, True) component = bb.data.getVar("SEPPUKU_COMPONENT", data, True) - if not seppuku_login(opener, login, user, pass): + if not seppuku_login(opener, login, user, passw): bb.note("Login to bugzilla failed") return NotHandled + else: + print "Logged into the box" if name == "TaskFailed": bugname = "%(package)s-%(pv)s-%(pr)s-%(task)s" % { "package" : bb.data.getVar("PN", data, True), "pv" : bb.data.getVar("PV", data, True), "pr" : bb.data.getVar("PR", data, True), "task" : e.task } - log_file = glob.glob("%s/log.%s.*" % (data.getVar('T', event.data, True), event.task)) + log_file = glob.glob("%s/log.%s.*" % (bb.data.getVar('T', event.data, True), event.task)) if len(log_file) != 0: - to_file = data.getVar('TINDER_LOG', event.data, True) + to_file = bb.data.getVar('TINDER_LOG', event.data, True) text = "".join(open(log_file[0], 'r').readlines()) elif name == "NoProvider": bugname = "noprovider for %s runtime: %s" % (event.getItem, event.getisRuntime) text = "Please fix it" - else + else: assert False (bug_open, bug_number) = seppuku_find_bug_report(opener, query, product, component, bugname) + bb.note("Bug is open: %s and bug number: %s" % (bug_open, bug_number)) + # The bug is present and still open, no need to attach an error log if bug_number and bug_open: - bb.note("The bug is known as '%s'" % bug_number" + bb.note("The bug is known as '%s'" % bug_number) return NotHandled if bug_number and not bug_open: if not seppuku_reopen_bug(opener, reopen, product, component, bug_number, bugname, text): bb.note("Failed to reopen the bug report") - else seppuku_file_bug(opener, newbug, product, component, bugname, text): + elif not seppuku_file_bug(opener, newbug, product, component, bugname, text): bb.note("Filing a bugreport failed") return NotHandled +} -- cgit v1.2.3