run_smoke_bench

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# ===--- run_smoke_bench -------------------------------------------------===//
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===---------------------------------------------------------------------===//
#
# Performs a very fast check which benchmarks regressed and improved.
#
# Initially runs the benchmarks with a low sample count and just re-runs those
# benchmarks which differ.
# Also reports code size differences.
#
# ===---------------------------------------------------------------------===//

importargparse
importglob
importos
importsubprocess
importsys
fromimpimportload_source

fromcompare_perf_testsimportPerformanceTestResult, TestComparator, create_report

# import Benchmark_Driver # doesn't work because it misses '.py' extension
Benchmark_Driver=load_source(
"Benchmark_Driver",
os.path.join(os.path.dirname(os.path.abspath(__file__)), "Benchmark_Driver"),
)
# from Benchmark_Driver import BenchmarkDriver, BenchmarkDoctor, ...
BenchmarkDriver=Benchmark_Driver.BenchmarkDriver
BenchmarkDoctor=Benchmark_Driver.BenchmarkDoctor
MarkdownReportHandler=Benchmark_Driver.MarkdownReportHandler

VERBOSE=False


classDriverArgs(object):
"""Arguments for BenchmarkDriver."""

def__init__(self, benchmark_dir, architecture, platform, optimization="O"):
"""Initialize with path to the build-dir and optimization level."""
self.benchmarks=None
self.filters=None
self.tests=os.path.join(benchmark_dir, "bin")
self.optimization=optimization
self.architecture=architecture
self.libdir=os.path.join(benchmark_dir, "lib", "swift", platform)


deflog(msg):
print(msg)
sys.stdout.flush()


defmain():
globalVERBOSE
argparser=argparse.ArgumentParser()
argparser.add_argument(
"-verbose", action="store_true", help="print verbose messages"
 )
argparser.add_argument(
"-O",
action="append_const",
const="O",
dest="opt_levels",
help="test -O benchmarks",
 )
argparser.add_argument(
"-Osize",
action="append_const",
const="Osize",
dest="opt_levels",
help="test -Osize benchmarks",
 )
argparser.add_argument(
"-Onone",
action="append_const",
const="Onone",
dest="opt_levels",
help="test -Onone benchmarks (except code size)",
 )
argparser.add_argument(
"-skip-code-size",
action="store_true",
help="Don't report code size differences",
 )
argparser.add_argument(
"-skip-performance",
action="store_true",
help="Don't report performance differences",
 )
argparser.add_argument(
"-skip-check-added",
action="store_true",
help="Don't validate newly added benchmarks",
 )
argparser.add_argument(
"-o",
type=str,
help="In addition to stdout, write the results into a markdown file",
 )
argparser.add_argument(
"-threshold",
type=float,
help="The performance threshold in %% which triggers a re-run",
default=5,
 )
argparser.add_argument(
"-num-samples",
type=int,
help="The (minimum) number of samples to run",
default=3,
 )
argparser.add_argument(
"-num-reruns",
type=int,
help="The number of re-runs until it's assumed to be a real change",
default=8,
 )
argparser.add_argument(
"-arch",
type=str,
help="The architecture. The default is x86_64",
default="x86_64",
 )
argparser.add_argument(
"-platform", type=str, help="The benchmark build platform", default="macosx"
 )
argparser.add_argument(
"oldbuilddir", nargs=1, type=str, help="old benchmark build directory"
 )
argparser.add_argument(
"newbuilddir", nargs=1, type=str, help="new benchmark build directory"
 )
args=argparser.parse_args()
VERBOSE=args.verbose

returntest_opt_levels(args)


deftest_opt_levels(args):
output_file=None
ifargs.o:
output_file=open(args.o, "w")

changes=False
foropt_levelinargs.opt_levelsor ["O", "Osize", "Onone"]:
log("####### Testing optimization level -"+opt_level+" #######")
ifnotargs.skip_performance:
iftest_performance(
opt_level,
args.oldbuilddir[0],
args.newbuilddir[0],
float(args.threshold) /100,
args.num_samples,
args.num_reruns,
output_file,
args.arch,
args.platform
 ):
changes=True

# There is no point in reporting code size for Onone.
ifnotargs.skip_code_sizeandopt_level!="Onone":
ifreport_code_size(
opt_level,
args.oldbuilddir[0],
args.newbuilddir[0],
args.arch,
args.platform,
output_file,
 ):
changes=True

ifnotargs.skip_code_size:
ifreport_code_size(
"swiftlibs",
args.oldbuilddir[0],
args.newbuilddir[0],
args.arch,
args.platform,
output_file,
 ):
changes=True

ifnotargs.skip_check_added:
check_added(args, output_file)

ifoutput_file:
ifchanges:
output_file.write(get_info_text())
else:
output_file.write("### No performance and code size changes")
output_file.close()
return0


defmeasure(driver, tests, i, min_num_samples):
"""Log and measure samples of the tests with the given driver.

 Collect increasing number of samples, depending on the iteration.
 """
num_samples=min(i+min_num_samples, 4*min_num_samples)
msg=" Iteration {0} for {1}: num samples = {2}, ".format(
i, driver.args.tests, num_samples
 )
msg+= (
"running all tests"
ifdriver.all_tests==tests
else"re-testing {0} tests".format(len(tests))
 )
log(msg)
driver.tests=tests
returndriver.run(num_samples=num_samples, sample_time=0.0025)


defmerge(results, other_results):
""""Merge the other PerformanceTestResults into the first dictionary."""
fortest, resultinother_results.items():
results[test].merge(result)
returnresults


deftest_performance(
opt_level, old_dir, new_dir, threshold, num_samples, num_reruns,
output_file, arch, platform
):
"""Detect performance changes in benchmarks.

 Start fast with few samples per benchmark and gradually spend more time
 gathering more precise measurements of the change candidates.
 """

i, unchanged_length_count=0, 0
old, new= [
BenchmarkDriver(DriverArgs(dir, architecture=arch, platform=platform,
optimization=opt_level))
fordirin [old_dir, new_dir]
 ]
results= [measure(driver, driver.tests, i, num_samples) fordriverin [old, new]]
tests=TestComparator(results[0], results[1], threshold)
changed=tests.decreased+tests.increased

whilelen(changed) >0andunchanged_length_count<num_reruns:
i+=1
ifVERBOSE:
log(" test again: "+str([test.namefortestinchanged]))
old_measurement=measure(old, [test.namefortestinchanged], i, num_samples)
old_results=merge(results[0], old_measurement)
new_measurement=measure(new, [test.namefortestinchanged], i, num_samples)
new_results=merge(results[1], new_measurement)
tests=TestComparator(old_results, new_results, threshold)
changed=tests.decreased+tests.increased

iflen(old.tests) ==len(changed):
unchanged_length_count+=1
else:
unchanged_length_count=0

log("")
report_title="Performance ({}): -{}".format(arch, opt_level)
returnreport_results(
report_title, threshold*1.4, output_file, old_results, new_results
 )


defreport_code_size(opt_level, old_dir, new_dir, architecture, platform, output_file):
ifopt_level=="swiftlibs":
p=os.path.join(old_dir, "lib", "swift", platform, architecture, "*.dylib")
files=glob.glob(p)
else:
files=glob.glob(
os.path.join(old_dir, opt_level+"-"+architecture+"*"+
platform+"*", "*.o")
 )

idx=1
old_results= {}
new_results= {}
foroldfileinfiles:
new_dir=os.path.join(new_dir, '')
newfile=oldfile.replace(old_dir, new_dir, 1)
ifos.path.isfile(newfile):
oldsize=get_codesize(oldfile)
newsize=get_codesize(newfile)
bname=os.path.basename(oldfile)
old_json= {"number": idx, "name": bname, "samples": [oldsize]}
new_json= {"number": idx, "name": bname, "samples": [newsize]}
old_results[bname] =PerformanceTestResult(old_json)
new_results[bname] =PerformanceTestResult(new_json)
idx+=1
returnreport_results(
"Code size: -"+opt_level, 0.01, output_file, old_results, new_results
 )


defget_codesize(filename):
output=subprocess.check_output(["size", filename])
lines=output.decode('utf-8').splitlines()
header_line=lines[0]
data_line=lines[1]
ifheader_line.find("__TEXT") !=0:
sys.exit("unexpected output from size command:\n"+lines)
returnint(data_line.split("\t")[0])


defreport_results(
title,
threshold,
output_file,
old_results,
new_results,
):
print("------- "+title+" -------")
print(create_report(old_results, new_results, threshold, "git"))

ifoutput_file:
report=create_report(old_results, new_results, threshold, "markdown")
ifreport!="":
output_file.write("### "+title+"\n")
output_file.write(report)
output_file.write("\n")
returnTrue
returnFalse


defget_info_text():
text="""
<details>
 <summary><strong>How to read the data</strong></summary>
The tables contain differences in performance which are larger than 8% and
differences in code size which are larger than 1%.

If you see any unexpected regressions, you should consider fixing the
regressions before you merge the PR.

**Noise**: Sometimes the performance results (not code size!) contain false
alarms. Unexpected regressions which are marked with '(?)' are probably noise.
If you see regressions which you cannot explain you can try to run the
benchmarks again. If regressions still show up, please consult with the
performance team (@eeckstein).
</details>

<details>
 <summary><strong>Hardware Overview</strong></summary>

"""
output=subprocess.check_output(["system_profiler", "SPHardwareDataType"])
po=output.decode('utf-8')
forlineinpo.splitlines():
selection= [
"Model Name",
"Model Identifier",
"Processor Name",
"Processor Speed",
"Number of Processors",
"Total Number of Cores",
"L2 Cache",
"L3 Cache",
"Memory:",
 ]

ifany(sinlineforsinselection):
text+=line+"\n"

text+="""
</details>"""
returntext


defcheck_added(args, output_file=None):
old=BenchmarkDriver(DriverArgs(args.oldbuilddir[0], architecture=args.arch,
platform=args.platform))
new=BenchmarkDriver(DriverArgs(args.newbuilddir[0], architecture=args.arch,
platform=args.platform))
added=set(new.tests).difference(set(old.tests))
new.tests=list(added)
doctor=BenchmarkDoctor(args, driver=new)
ifaddedandoutput_file:
doctor.log.addHandler(MarkdownReportHandler(output_file))
doctor.check()


if__name__=="__main__":
sys.exit(main())