submit-benchmark-results

#!/usr/bin/env python3

"""Utility script for submitting benchmark results to an LNT server."""

importdatetime
importerrno
importjson
importoptparse
importsubprocess
importsys
importurllib
importurllib2

# Test status codes.
PASS=0
FAIL=1
XFAIL=2

###


defcapture_with_result(args, include_stderr=False):
"""capture_with_result(command) -> (output, exit code)

 Run the given command (or argv list) in a shell and return the standard
 output and exit code.
 """
stderr=subprocess.PIPE
ifinclude_stderr:
stderr=subprocess.STDOUT
try:
p=subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr)
exceptOSErrorase:
ife.errno==errno.ENOENT:
sys.exit('no such file or directory: %r when running %s.'% (
args[0], ' '.join(args)))
raise
out, _=p.communicate()
returnout, p.wait()


defcapture(args, include_stderr=False):
"""capture(command) - Run the given command (or argv list) in a shell and
 return the standard output.
 """
returncapture_with_result(args, include_stderr)[0]


deftimestamp():
returndatetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')

###


defsubmit_results_to_server(results_data, submit_url):
# Submit the URL encoded data.
data=urllib.urlencode({'input_data': results_data,
'commit': '1'})
response=urllib2.urlopen(urllib2.Request(submit_url, data))
result_data=response.read()

# The result is expected to be a JSON object.
try:
returnjson.loads(result_data)
exceptValueError:
importtraceback
print("Unable to load result, not a valid JSON object.")
print()
print("Traceback:")
traceback.print_exc()
print()
print("Result:")
print(result_data)
return

###


defmain():
parser=optparse.OptionParser("""%prog [options] <results>""")
parser.add_option("", "--submit", dest="submit_url", metavar="URL",
help="Submit results to the given URL",
action="store", type=str, default=None)
parser.add_option("", "--output", dest="output", metavar="PATH",
help="Write raw report data to PATH",
action="store", type=str, default=None)
parser.add_option("", "--machine-name", dest="machine_name",
metavar="NAME",
help="Set the machine name to embed in the report",
action="store", type=str, default=None)
parser.add_option("", "--run-order", dest="run_order", metavar="REVISION",
help="Set the run order to embed in the report",
action="store", type=int, default=None)
opts, args=parser.parse_args()

# At least one of --submit or --output is required.
iflen(args) !=1:
parser.error("incorrect number of arguments")
ifopts.submit_urlisNoneandopts.outputisNone:
parser.error("no action given (provide --submit or --output)")
ifopts.machine_nameisNone:
parser.error("--machine-name is required")
ifopts.run_orderisNone:
parser.error("--run-order is required")

# Load the results data.
results_path, =args
withopen(results_path) asf:
data=json.load(f)

# Compute some data not present in the 'lit' report.
machine_name=opts.machine_name
run_order=str(opts.run_order)

# Estimate the end time as being now, and the start time as being that
# minus the elapsed testing time.
utcnow=datetime.datetime.utcnow()
start_time=utcnow-datetime.timedelta(seconds=data['elapsed'])
end_time=utcnow

# Create the LNT report format.
lnt_results= {}
lnt_results['Machine'] = {
'Name': machine_name,
'Info': {
'hardware': capture(["uname", "-m"], include_stderr=True).strip(),
'name': capture(["uname", "-n"], include_stderr=True).strip(),
'os': capture(["uname", "-sr"], include_stderr=True).strip(),
'uname': capture(["uname", "-a"], include_stderr=True).strip(),
 }
 }

# FIXME: Record source versions for LLVM, Swift, etc.?
lnt_results['Run'] = {
'Start Time': start_time.strftime('%Y-%m-%d %H:%M:%S'),
'End Time': end_time.strftime('%Y-%m-%d %H:%M:%S'),
'Info': {
'__report_version__': '1',
'tag': 'nts',
'inferred_run_order': run_order,
'run_order': run_order,
'sw_vers': capture(['sw_vers'], include_stderr=True).strip(),
 }
 }

lnt_results['Tests'] =lnt_tests= []
fortestindata['tests']:
# Ignore tests which have unexpected status.
code=test['code']
ifcodenotin ('PASS', 'XPASS', 'FAIL', 'XFAIL'):
sys.stderr.write("ignoring test %r with result code %r"% (
test['name'], code))
continue

# Extract the test name, which is encoded as 'suite :: name'.
test_name='nts.%s'% (test['name'].split('::', 1)[1][1:],)

# Convert this test to the 'nts' schema.
compile_success=test['metrics'].get('compile_success', 1)
compile_time=test['metrics']['compile_time']
exec_success=test['metrics'].get('exec_success', 1)
exec_time=test['metrics']['exec_time']

# FIXME: Ensure the test success flags matches the result code.
# FIXME: The XFAIL handling here isn't going to be right.

ifnotcompile_success:
lnt_tests.append({'Name': '%s.compile.status'% (test_name,),
'Info': {},
'Data': [FAIL]})
ifnotexec_success:
lnt_tests.append({'Name': '%s.exec.status'% (test_name,),
'Info': {},
'Data': [FAIL]})
lnt_tests.append({'Name': '%s.compile'% (test_name,),
'Info': {},
'Data': [compile_time]})
lnt_tests.append({'Name': '%s.exec'% (test_name,),
'Info': {},
'Data': [exec_time]})

# Create the report data.
lnt_result_data=json.dumps(lnt_results, indent=2) +'\n'

# Write the results, if requested.
ifopts.output:
sys.stderr.write('%s: generating report: %r\n'% (
timestamp(), opts.output))
withopen(opts.output, 'w') asf:
f.write(lnt_result_data)

# Submit the results to an LNT server, if requested.
ifopts.submit_url:
submit_results_to_server(lnt_result_data, opts.submit_url)


if__name__=='__main__':
main()