From fe05301aea2fe0677f0cfe4e0256a55c4f472ff8 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Sun, 19 Feb 2006 17:28:55 +0000 Subject: classes/tinderclient.bbclass: Begin of the rewrite to support the tbox3 protocol --- classes/tinderclient.bbclass | 260 ++++++++++++++++++++++++++++++------------- 1 file changed, 181 insertions(+), 79 deletions(-) diff --git a/classes/tinderclient.bbclass b/classes/tinderclient.bbclass index 290166bb03..a50cdda297 100644 --- a/classes/tinderclient.bbclass +++ b/classes/tinderclient.bbclass @@ -1,68 +1,186 @@ -def tinder_tz_offset(off): - # get the offset.n minutes Either it is a number like - # +200 or -300 - try: - return int(off) - except ValueError: - if off == "Europe/Berlin": - return 120 - else: - return 0 - -def tinder_tinder_time(offset): - import datetime - td = datetime.timedelta(minutes=tinder_tz_offset(offset)) - time = datetime.datetime.utcnow() + td - return time.strftime('%m/%d/%Y %H:%M:%S') - -def tinder_tinder_start(date,offset): - import datetime, time - td = datetime.timedelta(minutes=tinder_tz_offset(offset)) - ti = time.strptime(date, "%m/%d/%Y %H:%M:%S") - time = datetime.datetime(*ti[0:7])+td - return time.strftime('%m/%d/%Y %H:%M:%S') - -def tinder_send_email(da, header, log): - import smtplib +def tinder_form_data(bound, dict, log): + """ + Create the boundary for the HTTP Post + """ + output = [] + + # for each key in the dictionary + for name in dict: + output.append( "--" + bound ) + output.append( 'Content-Disposition: form-data; name="%s"' % name ) + output.append( "" ) + output.append( dict[name] ) + if log: + output.append( "--" + bound ) + output.append( 'Content-Disposition: form-data; name="log"; filename="log.txt"' ) + output.append( '' ) + output.append( log ) + output.append( '--' + bound + '--' ) + output.append( '' ) + + return "\r\n".join(output) + +def tinder_send_http(d, log): + """ + Send the log via HTTP... + """ + +def tinder_time_string(): + """ + Return the time as GMT + """ + return "" + +def tinder_format_http_post(d,log): + """ + Format the Tinderbox HTTP post with the data needed + for the tinderbox to be happy. + """ + from bb import data - from email.MIMEText import MIMEText - msg = MIMEText(header +'\n' + log) - msg['Subject'] = data.getVar('TINDER_SUBJECT',da, True) or "Tinder-Client build log" - msg['To'] = data.getVar('TINDER_MAILTO' ,da, True) - msg['From'] = data.getVar('TINDER_FROM', da, True) + import os,random + + # the variables we will need to send on this form post + variables = { + "tree" : data.getVar('TINDER_TREE', d, True), + "machine_name" : data.getVar('TINDER_MACHINE', d, True), + "os" : os.uname()[0], + "os_version" : os.uname()[2], + "compiler" : "gcc", + "clobber" : data.getVar('TINDER_CLOBBER', d, True) + } + # the boundary we will need + boundary = "----------------------------------%d" % int(random.random()*1000000000000) - s = smtplib.SMTP() - s.connect() - s.sendmail(data.getVar('TINDER_FROM', da, True), [data.getVar('TINDER_MAILTO', da, True)], msg.as_string()) - s.close() + # now format the body + body = tinder_form_data( boundary, variables, log ) -def tinder_send_http(da, header, log): + return ("multipart/form-data; boundary=%s" % boundary),body + + +def tinder_build_start(d): + """ + Inform the tinderbox that a build is starting. We do this + by posting our name and tree to the build_start.pl script + on the server. + """ from bb import data - import httplib, urllib - cont = "\n%s\n%s" % ( header, log) - headers = {"Content-type": "multipart/form-data" } + import httplib + + # get the body and type + content_type, body = tinder_format_http_post(d,None) + server = data.getVar('TINDER_HOST', d, True ) + url = data.getVar('TINDER_URL', d, True ) + + selector = url + "/xml/build_start.pl" + + print "selector %s and url %s" % (selector, url) + + # now post it + h = httplib.HTTP(server) + h.putrequest('POST', selector) + h.putheader('content-type', content_type) + h.putheader('content-length', str(len(body))) + h.endheaders() + h.send(body) + errcode, errmsg, headers = h.getreply() + print errcode, errmsg, headers + print h.file.read() - conn = httplib.HTTPConnection(data.getVar('TINDER_HOST',da, True)) - conn.request("POST", data.getVar('TINDER_URL',da,True), cont, headers) - conn.close() + print body + print content_type +def tinder_print_info(d): + """ + Print the TinderBox Info + Including informations of the BaseSystem and the Tree + we use. + """ + + from bb import data + import os + # get the local vars + + time = tinder_time_string() + ops = os.uname()[0] + version = os.uname()[2] + url = data.getVar( 'TINDER_URL' , d, True ) + tree = data.getVar( 'TINDER_TREE', d, True ) + branch = data.getVar( 'TINDER_BRANCH', d, True ) + srcdate = data.getVar( 'SRCDATE', d, True ) + machine = data.getVar( 'MACHINE', d, True ) + distro = data.getVar( 'DISTRO', d, True ) + bbfiles = data.getVar( 'BBFILES', d, True ) + tarch = data.getVar( 'TARGET_ARCH', d, True ) + fpu = data.getVar( 'TARGET_FPU', d, True ) + + # there is a bug with tipple quoted strings + # i will work around but will fix the original + # bug as well + output = [] + output.append("== Tinderbox Info" ) + output.append("Time: %(time)s" ) + output.append("OS: %(ops)s" ) + output.append("%(version)s" ) + output.append("Compiler: gcc" ) + output.append("Tinderbox Client: 0.1" ) + output.append("Tinderbox Client Last Modified: yesterday" ) + output.append("Tinderbox Protocol: 0.1" ) + output.append("URL: %(url)s" ) + output.append("Tree: %(tree)s" ) + output.append("Config:" ) + output.append("branch = '%(branch)s'" ) + output.append("TARGET_ARCH = '%(tarch)s'" ) + output.append("TARGET_FPU = '%(fpu)s'" ) + output.append("SRCDATE = '%(srcdate)s'" ) + output.append("MACHINE = '%(machine)s'" ) + output.append("DISTRO = '%(distro)s'" ) + output.append("BBFILES = '%(bbfiles)s'" ) + output.append("== End Tinderbox Client Info" ) + + # now create the real output + return "\n".join(output) % vars() -# Prepare tinderbox mail header -def tinder_prepare_mail_header(da, status): + +def tinder_print_env(): + """ + Print the environment variables of this build + """ from bb import data + import os - str = "tinderbox: administrator: %s\n" % data.getVar('TINDER_ADMIN', da, True) - str += "tinderbox: starttime: %s\n" % tinder_tinder_start(data.getVar('TINDER_START', da, True) or data.getVar('BUILDSTART', da, True), data.getVar('TINDER_TZ', da, True)) - str += "tinderbox: buildname: %s\n" % data.getVar('TINDER_BUILD', da, True) - str += "tinderbox: errorparser: %s\n" % data.getVar('TINDER_ERROR', da, True) - str += "tinderbox: status: %s\n" % status - str += "tinderbox: timenow: %s\n" % tinder_tinder_time(data.getVar('TINDER_TZ', da, True)) - str += "tinderbox: tree: %s\n" % data.getVar('TINDER_TREE', da, True) - str += "tinderbox: buildfamily: %s\n" % "unix" - str += "tinderbox: END" + time_start = tinder_time_string() + time_end = tinder_time_string() - return str + # build the environment + env = "" + for var in os.environ: + env += "%s=%s\n" % (var, os.environ[var]) + + output = [] + output.append( "---> TINDERBOX RUNNING env %(time_start)s" ) + output.append( env ) + output.append( "<--- TINDERBOX FINISHED (SUCCESS) %(time_end)s" ) + + return "\n".join(output) % vars() + +def tinder_tinder_start(d): + """ + PRINT the configuration of this build + """ + + time_start = tinder_time_string() + config = tinder_print_info(d) + env = tinder_print_env() + time_end = tinder_time_string() + + output = [] + output.append( "---> TINDERBOX PRINTING CONFIGURATION %(time_start)s" ) + output.append( config ) + output.append( env ) + output.append( "<--- TINDERBOX FINISHED PRINTING CONFIGURATION %(time_end)s" ) + return "\n".join(output) % vars() def tinder_do_tinder_report(event): """ @@ -77,28 +195,17 @@ def tinder_do_tinder_report(event): # variables name = getName(event) log = "" - header = "" - verbose = data.getVar('TINDER_VERBOSE_REPORT', event.data, True) == "1" + logfile = None # Check what we need to do Build* shows we start or are done if name == "BuildStarted": - header = tinder_prepare_mail_header(event.data, 'building') - # generate - for var in os.environ: - log += "%s=%s\n" % (var, os.environ[var]) - - mkdirhier(data.getVar('TMPDIR', event.data, True)) - file = open(data.getVar('TINDER_LOG', event.data, True), 'w') - file.write(log) - - if not verbose: - header = "" + tinder_build_start(event.data) + log = tinder_tinder_start(event.data) if name == "PkgFailed" or name == "BuildCompleted": status = 'build_failed' if name == "BuildCompleted": status = "success" - header = tinder_prepare_mail_header(event.data, status) # append the log log_file = data.getVar('TINDER_LOG', event.data, True) file = open(log_file, 'r') @@ -106,11 +213,9 @@ def tinder_do_tinder_report(event): log += line if verbose and name == "TaskStarted": - header = tinder_prepare_mail_header(event.data, 'building') log = "Task %s started" % event.task if verbose and name == "PkgStarted": - header = tinder_prepare_mail_header(event.data, 'building') log = "Package %s started" % data.getVar('P', event.data, True) if verbose and name == "PkgSucceeded": @@ -124,8 +229,6 @@ def tinder_do_tinder_report(event): if len(log_file) != 0: to_file = data.getVar('TINDER_LOG', event.data, True) log_txt = open(log_file[0], 'r').readlines() - to_file = open(to_file, 'a') - to_file.writelines(log_txt) # append to the log @@ -134,17 +237,16 @@ def tinder_do_tinder_report(event): for line in log_txt: log += line - # now mail the log - if len(log) == 0 or len(header) == 0: + # now post the log + if len(log) == 0: return - log_post_method = tinder_send_email - if data.getVar('TINDER_SENDLOG', event.data, True) == "http": - log_post_method = tinder_send_http - - log_post_method(event.data, header, log) + # for now we will use the http post method as it is the only one + log_post_method = tinder_send_http + log_post_method(event.data, log) +# we want to be an event handler addhandler tinderclient_eventhandler python tinderclient_eventhandler() { from bb import note, error, data -- cgit v1.2.3 From aca193101e2db23661f2a49ba6c18e3192d93229 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Sun, 19 Feb 2006 18:43:26 +0000 Subject: classes/tinderclient.bbclass: -Save the assigned machine id to a file and restore it when needed. BitBake does not have a 'persistent' cache one could use across files... -Post the configuration directly to the server --- classes/tinderclient.bbclass | 94 +++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 23 deletions(-) diff --git a/classes/tinderclient.bbclass b/classes/tinderclient.bbclass index a50cdda297..67762faba8 100644 --- a/classes/tinderclient.bbclass +++ b/classes/tinderclient.bbclass @@ -20,18 +20,13 @@ def tinder_form_data(bound, dict, log): return "\r\n".join(output) -def tinder_send_http(d, log): - """ - Send the log via HTTP... - """ - def tinder_time_string(): """ Return the time as GMT """ return "" -def tinder_format_http_post(d,log): +def tinder_format_http_post(d,status,log): """ Format the Tinderbox HTTP post with the data needed for the tinderbox to be happy. @@ -50,6 +45,20 @@ def tinder_format_http_post(d,log): "clobber" : data.getVar('TINDER_CLOBBER', d, True) } + # optionally add the status + if status: + variables["status"] = str(status) + + # try to load the machine id + # we only need on build_status.pl but sending it + # always does not hurt + try: + f = file(data.getVar('TMPDIR',d,True)+'/tinder-machine.id', 'r') + id = f.read() + variables['machine_id'] = id + except: + pass + # the boundary we will need boundary = "----------------------------------%d" % int(random.random()*1000000000000) @@ -69,7 +78,7 @@ def tinder_build_start(d): import httplib # get the body and type - content_type, body = tinder_format_http_post(d,None) + content_type, body = tinder_format_http_post(d,None,None) server = data.getVar('TINDER_HOST', d, True ) url = data.getVar('TINDER_URL', d, True ) @@ -77,6 +86,46 @@ def tinder_build_start(d): print "selector %s and url %s" % (selector, url) + # now post it + h = httplib.HTTP(server) + h.putrequest('POST', selector) + h.putheader('content-type', content_type) + h.putheader('content-length', str(len(body))) + h.endheaders() + h.send(body) + errcode, errmsg, headers = h.getreply() + print errcode, errmsg, headers + report = h.file.read() + + # now let us find the machine id that was assigned to us + search = " Date: Sun, 19 Feb 2006 20:09:12 +0000 Subject: classes/tinderclient.bbclass: -Send the right messages to the TinderBox -Read the log from the logfiles and send it to the tinderbox -Add the wish for a more sane and more featurefill reporting (e.g. less data, less often) --- classes/tinderclient.bbclass | 70 +++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/classes/tinderclient.bbclass b/classes/tinderclient.bbclass index 67762faba8..d01bce1fcf 100644 --- a/classes/tinderclient.bbclass +++ b/classes/tinderclient.bbclass @@ -231,9 +231,15 @@ def tinder_tinder_start(d): def tinder_do_tinder_report(event): """ - Report to the tinderbox. Either we will report every step - (depending on TINDER_VERBOSE_REPORT) at the end we will send the - tinderclient.log + Report to the tinderbox: + On the BuildStart we will inform the box directly + On the other events we will write to the TINDER_LOG and + when the Task is finished we will send the report. + + The above is not yet fully implemented. Currently we send + information immediately. The caching/queuing needs to be + implemented. Also sending more or less information is not + implemented yet. """ from bb.event import getName from bb import data, mkdirhier @@ -242,7 +248,6 @@ def tinder_do_tinder_report(event): # variables name = getName(event) log = "" - logfile = None status = 1 # Check what we need to do Build* shows we start or are done @@ -250,40 +255,39 @@ def tinder_do_tinder_report(event): tinder_build_start(event.data) log = tinder_tinder_start(event.data) - if name == "PkgFailed" or name == "BuildCompleted": - status = 'build_failed' - if name == "BuildCompleted": - status = "success" - # append the log - log_file = data.getVar('TINDER_LOG', event.data, True) - file = open(log_file, 'r') - for line in file.readlines(): - log += line - - #if verbose and name == "TaskStarted": - # log = "Task %s started" % event.task - # - #if verbose and name == "PkgStarted": - # log = "Package %s started" % data.getVar('P', event.data, True) - # - #if verbose and name == "PkgSucceeded": - # header = tinder_prepare_mail_header(event.data, 'building') - # log = "Package %s done" % data.getVar('P', event.data, True) - - # Append the Task Log + try: + # truncate the tinder log file + f = file(data.getVar('TINDER_LOG', event.data, True), 'rw+') + f.truncate(0) + f.close() + except IOError: + pass + + # Append the Task-Log (compile,configure...) to the log file + # we will send to the server if name == "TaskSucceeded" or name == "TaskFailed": log_file = glob.glob("%s/log.%s.*" % (data.getVar('T', event.data, True), event.task)) if len(log_file) != 0: to_file = data.getVar('TINDER_LOG', event.data, True) - log_txt = open(log_file[0], 'r').readlines() - to_file.writelines(log_txt) - - # append to the log - #if verbose: - # header = tinder_prepare_mail_header(event.data, 'building') - # for line in log_txt: - # log += line + log = open(log_file[0], 'r').readlines() + + # set the right 'HEADER'/Summary for the TinderBox + if name == "TaskStarted": + log += "--> TINDERBOX Task %s started" % event.task + elif name == "TaskSucceeded": + log += "<-- TINDERBOX Task %s done (SUCCESS)" % event.task + elif name == "TaskFailed": + log += "<-- TINDERBOX Task %s failed (FAILURE)" % event.task + elif name == "PkgStarted": + log += "--> TINDERBOX Package %s started" % data.getVar('P', event.data, True) + elif name == "PkgSucceeded": + log += "<-- TINDERBOX Package %s done (SUCCESS)" % data.getVar('P', event.data, True) + elif name == "PkgFailed": + log += "<-- TINDERBOX Package %s failed (FAILURE)" % data.getVar('P', event.data, True) + status = 200 + elif name == "BuildCompleted": + status = 100 # now post the log if len(log) == 0: -- cgit v1.2.3 From 031c649fb653348559ae48ddde15b02d724e71f0 Mon Sep 17 00:00:00 2001 From: Holger Freyther Date: Sun, 19 Feb 2006 20:14:31 +0000 Subject: conf/tinder.conf: Update to the tinderclient.bbclass. We only report using HTTP as this is the only way to do reports for tinderbox3 --- conf/tinder.conf | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/conf/tinder.conf b/conf/tinder.conf index 2c50b62a17..99e263c96c 100644 --- a/conf/tinder.conf +++ b/conf/tinder.conf @@ -1,37 +1,26 @@ INHERIT += "tinderclient" -# the name of the admin (e.g you) -#TINDER_ADMIN = "you@yourhost" #Name of the client -#DO NOT USE '/' or ' ' in a name!! -#TINDER_BUILD = "Test-NameOfClient" +#TINDER_MACHINE = "Test-NameOfClient-build-image" -#What error parser to use -#TINDER_ERROR = "unix" -#One mail/post per task -#TINDER_VERBOSE_REPORT = "1" -#TINDER_LOG = "${TMPbDIR}/tinder.log" -#TINDER_TREE = "OpenEmbeddedBuild" +# Which lof file to write to, which tree to report to +#TINDER_LOG = "${TMPDIR}/tinder.log" +#TINDER_TREE = "OpenEmbeddedBuilds" -# Mail posting -#We have fixed procmail rules for the subject -#TINDER_SUBJECT = "OpenEmbedded Tinder Log" -#TINDER_MAILTO = "tinderbox-oe@gmx.net" -#TINDER_FROM = "you@yourhost" - # HTTP posting #TINDER_HOST = "ewi546.ewi.utwente.nl" #TINDER_URL = "/OE_qa/cgi-bin/process_builds.cgi" -# Select the submit method http or mail -#TINDER_SENDLOG = "http" - +# Which branch do we build +#TINDER_BRANCH = "org.openembedded.dev" -# TimeZone handling -#TINDER_TZ = "+0200" +# Clobbing +# 0 for rebuilding everything from scratch +# 1 for incremental builds +#TINDER_CLOBBER = "0" # Do a report at all #TINDER_REPORT = "1" -- cgit v1.2.3