process-stats-dir.py

#!/usr/bin/env python3
#
# ==-- process-stats-dir - summarize one or more Swift -stats-output-dirs --==#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014-2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ==------------------------------------------------------------------------==#
#
# This file processes the contents of one or more directories generated by
# `swiftc -stats-output-dir` and emits summary data, traces etc. for analysis.

importargparse
importcsv
importio
importitertools
importjson
importos
importplatform
importre
importsys
importtime
importurllib
fromcollectionsimportnamedtuple
fromoperatorimportattrgetter

fromjobstatsimport (list_stats_dir_profiles,
load_stats_dir, merge_all_jobstats)

ifsys.version_info[0] <3:
importurllib2
Request=urllib2.Request
URLOpen=urllib2.urlopen
else:
importurllib.request
importurllib.parse
importurllib.error
Request=urllib.request.Request
URLOpen=urllib.request.urlopen

MODULE_PAT=re.compile(r'^(\w+)\.')


defmodule_name_of_stat(name):
returnre.match(MODULE_PAT, name).groups()[0]


defstat_name_minus_module(name):
returnre.sub(MODULE_PAT, '', name)


# Perform any custom processing of args here, in particular the
# select_stats_from_csv_baseline step, which is a bit subtle.
defvars_of_args(args):
vargs=vars(args)
ifargs.select_stats_from_csv_baselineisnotNone:
withio.open(args.select_stats_from_csv_baseline, 'r', encoding='utf-8') asf:
b=read_stats_dict_from_csv(f)
# Sniff baseline stat-names to figure out if they're module-qualified
# even when the user isn't asking us to _output_ module-grouped data.
all_triples=all(len(k.split('.')) ==3forkinb.keys())
ifargs.group_by_moduleorall_triples:
vargs['select_stat'] =set(stat_name_minus_module(k)
forkinb.keys())
else:
vargs['select_stat'] =b.keys()
returnvargs


# Passed args with 2-element remainder ["old", "new"], return a list of tuples
# of the form [(name, (oldstats, newstats))] where each name is a common subdir
# of each of "old" and "new", and the stats are those found in the respective
# dirs.
defload_paired_stats_dirs(args):
assertlen(args.remainder) ==2
paired_stats= []
 (old, new) =args.remainder
vargs=vars_of_args(args)
forpinsorted(os.listdir(old)):
full_old=os.path.join(old, p)
full_new=os.path.join(new, p)
ifnot (os.path.exists(full_old) andos.path.isdir(full_old) and
os.path.exists(full_new) andos.path.isdir(full_new)):
continue
old_stats=load_stats_dir(full_old, **vargs)
new_stats=load_stats_dir(full_new, **vargs)
iflen(old_stats) ==0orlen(new_stats) ==0:
continue
paired_stats.append((p, (old_stats, new_stats)))
returnpaired_stats


defwrite_catapult_trace(args):
allstats= []
vargs=vars_of_args(args)
forpathinargs.remainder:
allstats+=load_stats_dir(path, **vargs)
allstats.sort(key=attrgetter('start_usec'))
foriinrange(len(allstats)):
allstats[i].jobid=i
json.dump([s.to_catapult_trace_obj() forsinallstats], args.output)


defwrite_lnt_values(args):
vargs=vars_of_args(args)
fordinargs.remainder:
stats=load_stats_dir(d, **vargs)
merged=merge_all_jobstats(stats, **vargs)
j=merged.to_lnt_test_obj(args)
ifargs.lnt_submitisNone:
json.dump(j, args.output, indent=4)
else:
url=args.lnt_submit
print("\nsubmitting to LNT server: "+url)
json_report= {'input_data': json.dumps(j), 'commit': '1'}
data=urllib.urlencode(json_report)
response_str=URLOpen(Request(url, data))
response=json.loads(response_str.read())
print("### response:")
print(response)
if'success'inresponse:
print("server response:\tSuccess")
else:
print("server response:\tError")
print("error:\t", response['error'])
sys.exit(1)


defshow_paired_incrementality(args):
fieldnames= ["old_pct", "old_skip",
"new_pct", "new_skip",
"delta_pct", "delta_skip",
"name"]
out=csv.DictWriter(args.output, fieldnames, dialect='excel-tab')
out.writeheader()
vargs=vars_of_args(args)

for (name, (oldstats, newstats)) inload_paired_stats_dirs(args):
olddriver=merge_all_jobstats((xforxinoldstats
ifx.is_driver_job()), **vargs)
newdriver=merge_all_jobstats((xforxinnewstats
ifx.is_driver_job()), **vargs)
ifolddriverisNoneornewdriverisNone:
continue
oldpct=olddriver.incrementality_percentage()
newpct=newdriver.incrementality_percentage()
deltapct=newpct-oldpct
oldskip=olddriver.driver_jobs_skipped()
newskip=newdriver.driver_jobs_skipped()
deltaskip=newskip-oldskip
out.writerow(dict(name=name,
old_pct=oldpct, old_skip=oldskip,
new_pct=newpct, new_skip=newskip,
delta_pct=deltapct, delta_skip=deltaskip))


defshow_incrementality(args):
fieldnames= ["incrementality", "name"]
out=csv.DictWriter(args.output, fieldnames, dialect='excel-tab')
out.writeheader()

vargs=vars_of_args(args)
forpathinargs.remainder:
stats=load_stats_dir(path, **vargs)
forsinstats:
ifs.is_driver_job():
pct=s.incrementality_percentage()
out.writerow(dict(name=os.path.basename(path),
incrementality=pct))


defdiff_and_pct(old, new):
ifold==0:
ifnew==0:
return (0, 0.0)
else:
return (new, 100.0)
delta= (new-old)
delta_pct=round((float(delta) /float(old)) *100.0, 2)
return (delta, delta_pct)


defupdate_epoch_value(d, name, epoch, value):
changed=0
ifnameind:
 (existing_epoch, existing_value) =d[name]
ifexisting_epoch>epoch:
print("note: keeping newer value %d from epoch %d for %s"
% (existing_value, existing_epoch, name))
epoch=existing_epoch
value=existing_value
elifexisting_value==value:
epoch=existing_epoch
else:
 (_, delta_pct) =diff_and_pct(existing_value, value)
print("note: changing value %d -> %d (%.2f%%) for %s"%
 (existing_value, value, delta_pct, name))
changed=1
d[name] = (epoch, value)
return (epoch, value, changed)


defread_stats_dict_from_csv(f, select_stat=''):
infieldnames= ["epoch", "name", "value"]
c=csv.DictReader(f, infieldnames,
dialect='excel-tab',
quoting=csv.QUOTE_NONNUMERIC)
d= {}
sre=re.compile('.*'iflen(select_stat) ==0else
'|'.join(select_stat))
forrowinc:
epoch=int(row["epoch"])
name=row["name"]
ifsre.search(name) isNone:
continue
value=int(row["value"])
update_epoch_value(d, name, epoch, value)
returnd


# The idea here is that a "baseline" is a (tab-separated) CSV file full of
# the counters you want to track, each prefixed by an epoch timestamp of
# the last time the value was reset.
#
# When you set a fresh baseline, all stats in the provided stats dir are
# written to the baseline. When you set against an _existing_ baseline,
# only the counters mentioned in the existing baseline are updated, and
# only if their values differ.
#
# Finally, since it's a line-oriented CSV file, you can put:
#
# mybaseline.csv merge=union
#
# in your .gitattributes file, and forget about merge conflicts. The reader
# function above will take the later epoch anytime it detects duplicates,
# so union-merging is harmless. Duplicates will be eliminated whenever the
# next baseline-set is done.
defset_csv_baseline(args):
existing=None
vargs=vars_of_args(args)
ifos.path.exists(args.set_csv_baseline):
withio.open(args.set_csv_baseline, "r", encoding='utf-8', newline='\n') asf:
ss=vargs['select_stat']
existing=read_stats_dict_from_csv(f, select_stat=ss)
print("updating %d baseline entries in %s"%
 (len(existing), args.set_csv_baseline))
else:
print("making new baseline "+args.set_csv_baseline)
fieldnames= ["epoch", "name", "value"]

def_open(path):
ifsys.version_info[0] <3:
returnopen(path, 'wb')
returnio.open(path, "w", encoding='utf-8', newline='\n')

with_open(args.set_csv_baseline) asf:
out=csv.DictWriter(f, fieldnames, dialect='excel-tab',
quoting=csv.QUOTE_NONNUMERIC)
m=merge_all_jobstats((sfordinargs.remainder
forsinload_stats_dir(d, **vargs)),
**vargs)
ifmisNone:
print("no stats found")
return1
changed=0
newepoch=int(time.time())
fornameinsorted(m.stats.keys()):
epoch=newepoch
value=m.stats[name]
ifexistingisnotNone:
ifnamenotinexisting:
continue
 (epoch, value, chg) =update_epoch_value(existing, name,
epoch, value)
changed+=chg
out.writerow(dict(epoch=int(epoch),
name=name,
value=int(value)))
ifexistingisnotNone:
print("changed %d entries in baseline"%changed)
return0


OutputRow=namedtuple("OutputRow",
 ["name", "old", "new",
"delta", "delta_pct"])


defcompare_stats(args, old_stats, new_stats):
fornameinsorted(old_stats.keys()):
old=old_stats[name]
new=new_stats.get(name, 0)
 (delta, delta_pct) =diff_and_pct(old, new)
yieldOutputRow(name=name,
old=int(old), new=int(new),
delta=int(delta),
delta_pct=delta_pct)


IMPROVED=-1
UNCHANGED=0
REGRESSED=1


defrow_state(row, args):
delta_pct_over_thresh=abs(row.delta_pct) >args.delta_pct_thresh
if (row.name.startswith("time.") or'.time.'inrow.name):
# Timers are judged as changing if they exceed
# the percentage _and_ absolute-time thresholds
delta_usec_over_thresh=abs(row.delta) >args.delta_usec_thresh
ifdelta_pct_over_threshanddelta_usec_over_thresh:
return (REGRESSEDifrow.delta>0elseIMPROVED)
elifdelta_pct_over_thresh:
return (REGRESSEDifrow.delta>0elseIMPROVED)
returnUNCHANGED


defwrite_comparison(args, old_stats, new_stats):
rows=list(compare_stats(args, old_stats, new_stats))
sort_key= (attrgetter('delta_pct')
ifargs.sort_by_delta_pct
elseattrgetter('name'))

regressed= [rforrinrowsifrow_state(r, args) ==REGRESSED]
unchanged= [rforrinrowsifrow_state(r, args) ==UNCHANGED]
improved= [rforrinrowsifrow_state(r, args) ==IMPROVED]
regressions=len(regressed)

ifargs.markdown:

defformat_time(v):
ifabs(v) >1000000:
return"{:.1f}s".format(v/1000000.0)
elifabs(v) >1000:
return"{:.1f}ms".format(v/1000.0)
else:
return"{:.1f}us".format(v)

defformat_field(field, row):
iffield=='name':
ifargs.group_by_module:
returnstat_name_minus_module(row.name)
else:
returnrow.name
eliffield=='delta_pct':
s=str(row.delta_pct) +"%"
ifargs.github_emoji:
ifrow_state(row, args) ==REGRESSED:
s+=" :no_entry:"
elifrow_state(row, args) ==IMPROVED:
s+=" :white_check_mark:"
returns
else:
v=int(getattr(row, field))
ifrow.name.startswith('time.'):
returnformat_time(v)
else:
return"{:,d}".format(v)

defformat_table(elts):
out=args.output
out.write('\n')
out.write(' | '.join(OutputRow._fields))
out.write('\n')
out.write(' | '.join('---:'for_inOutputRow._fields))
out.write('\n')
foreinelts:
out.write(' | '.join(format_field(f, e)
forfinOutputRow._fields))
out.write('\n')

defformat_details(name, elts, is_closed):
out=args.output
details='<details>\n'ifis_closedelse'<details open>\n'
out.write(details)
out.write('<summary>%s (%d)</summary>\n'
% (name, len(elts)))
ifargs.group_by_module:
defkeyfunc(e):
returnmodule_name_of_stat(e.name)
elts.sort(key=attrgetter('name'))
formod, groupinitertools.groupby(elts, keyfunc):
groupelts=list(group)
groupelts.sort(key=sort_key, reverse=args.sort_descending)
out.write(details)
out.write('<summary>%s in %s (%d)</summary>\n'
% (name, mod, len(groupelts)))
format_table(groupelts)
out.write('</details>\n')
else:
elts.sort(key=sort_key, reverse=args.sort_descending)
format_table(elts)
out.write('</details>\n')

closed_regressions= (args.close_regressionsorlen(regressed) ==0)
format_details('Regressed', regressed, closed_regressions)
format_details('Improved', improved, True)
format_details('Unchanged (delta < %s%% or delta < %s)'%
 (args.delta_pct_thresh,
format_time(args.delta_usec_thresh)),
unchanged, True)

else:
rows.sort(key=sort_key, reverse=args.sort_descending)
out=csv.DictWriter(args.output, OutputRow._fields,
dialect='excel-tab')
out.writeheader()
forrowinrows:
ifrow_state(row, args) !=UNCHANGED:
out.writerow(row._asdict())

returnregressions


defcompare_to_csv_baseline(args):
vargs=vars_of_args(args)
withio.open(args.compare_to_csv_baseline, 'r', encoding='utf-8') asf:
old_stats=read_stats_dict_from_csv(f, select_stat=vargs['select_stat'])
m=merge_all_jobstats((sfordinargs.remainder
forsinload_stats_dir(d, **vargs)),
**vargs)
old_stats=dict((k, v) for (k, (_, v)) inold_stats.items())
new_stats=m.stats

returnwrite_comparison(args, old_stats, new_stats)


# Summarize immediate difference between two stats-dirs, optionally
defcompare_stats_dirs(args):
iflen(args.remainder) !=2:
raiseValueError("Expected exactly 2 stats-dirs")

vargs=vars_of_args(args)
 (old, new) =args.remainder
old_stats=merge_all_jobstats(load_stats_dir(old, **vargs), **vargs)
new_stats=merge_all_jobstats(load_stats_dir(new, **vargs), **vargs)

returnwrite_comparison(args, old_stats.stats, new_stats.stats)


# Evaluate a boolean expression in terms of the provided stats-dir; all stats
# are projected into python dicts (thus variables in the eval expr) named by
# the last identifier in the stat definition. This means you can evaluate
# things like 'NumIRInsts < 1000' or
# 'NumTypesValidated == NumTypesDeserialized'
defevaluate(args):
iflen(args.remainder) !=1:
raiseValueError("Expected exactly 1 stats-dir to evaluate against")

d=args.remainder[0]
vargs=vars_of_args(args)
merged=merge_all_jobstats(load_stats_dir(d, **vargs), **vargs)
env= {}
ident=re.compile(r'(\w+)$')
for (k, v) inmerged.stats.items():
ifk.startswith("time.") or'.time.'ink:
continue
m=re.search(ident, k)
ifm:
i=m.groups()[0]
ifargs.verbose:
print("%s => %s"% (i, v))
env[i] =v
try:
ifeval(args.evaluate, env):
return0
else:
print("evaluate condition failed: '%s'"%args.evaluate)
return1
exceptExceptionase:
print(e)
return1


# Evaluate a boolean expression in terms of deltas between the provided two
# stats-dirs; works like evaluate() above but on absolute differences
defevaluate_delta(args):
iflen(args.remainder) !=2:
raiseValueError("Expected exactly 2 stats-dirs to evaluate-delta")

 (old, new) =args.remainder
vargs=vars_of_args(args)
old_stats=merge_all_jobstats(load_stats_dir(old, **vargs), **vargs)
new_stats=merge_all_jobstats(load_stats_dir(new, **vargs), **vargs)

env= {}
ident=re.compile(r'(\w+)$')
forrincompare_stats(args, old_stats.stats, new_stats.stats):
ifr.name.startswith("time.") or'.time.'inr.name:
continue
m=re.search(ident, r.name)
ifm:
i=m.groups()[0]
ifargs.verbose:
print("%s => %s"% (i, r.delta))
env[i] =r.delta
try:
ifeval(args.evaluate_delta, env):
return0
else:
print("evaluate-delta condition failed: '%s'"%
args.evaluate_delta)
return1
exceptExceptionase:
print(e)
return1


defrender_profiles(args):
flamegraph_pl=args.flamegraph_script
ifflamegraph_plisNone:
importdistutils.spawn
flamegraph_pl=distutils.spawn.find_executable("flamegraph.pl")
ifflamegraph_plisNone:
print("Need flamegraph.pl in $PATH, or pass --flamegraph-script")

vargs=vars_of_args(args)
forstatsdirinargs.remainder:
jobprofs=list_stats_dir_profiles(statsdir, **vargs)
index_path=os.path.join(statsdir, "profile-index.html")
all_profile_types=set([kforkeysin [j.profiles.keys()
forjinjobprofs
ifj.profilesisnotNone]
forkinkeys])
withopen(index_path, "wb") asindex:
forptypeinall_profile_types:
index.write("<h2>Profile type: "+ptype+"</h2>\n")
index.write("<ul>\n")
forjinjobprofs:
ifj.is_frontend_job():
index.write(" <li>"+
 ("Module %s :: %s"%
 (j.module, " ".join(j.jobargs))) +"\n")
index.write(" <ul>\n")
profiles=sorted(j.profiles.get(ptype, {}).items())
forcounter, pathinprofiles:
title= ("Module: %s, File: %s, "
"Counter: %s, Profile: %s"%
 (j.module, j.input, counter, ptype))
subtitle=j.triple+", -"+j.opt
svg=os.path.abspath(path+".svg")
withopen(path) asp, open(svg, "wb") asg:
importsubprocess
print("Building flamegraph "+svg)
subprocess.check_call([flamegraph_pl,
"--title", title,
"--subtitle", subtitle],
stdin=p, stdout=g)
link= ("<tt><a href=\"file://%s\">%s</a></tt>"%
 (svg, counter))
index.write(" <li>"+link+"\n")
index.write(" </ul>\n")
index.write(" </li>\n")
ifargs.browse_profiles:
importwebbrowser
webbrowser.open_new_tab("file://"+os.path.abspath(index_path))


defprocess(args):
ifargs.catapult:
write_catapult_trace(args)
elifargs.compare_stats_dirs:
returncompare_stats_dirs(args)
elifargs.set_csv_baselineisnotNone:
returnset_csv_baseline(args)
elifargs.compare_to_csv_baselineisnotNone:
returncompare_to_csv_baseline(args)
elifargs.incrementality:
ifargs.paired:
show_paired_incrementality(args)
else:
show_incrementality(args)
elifargs.lnt:
write_lnt_values(args)
elifargs.evaluate:
returnevaluate(args)
elifargs.evaluate_delta:
returnevaluate_delta(args)
elifargs.render_profiles:
returnrender_profiles(args)
returnNone


defmain():
parser=argparse.ArgumentParser()
parser.add_argument("--verbose", action="store_true",
help="Report activity verbosely")
parser.add_argument("--output", default="-",
type=argparse.FileType('w'),
help="Write output to file")
parser.add_argument("--paired", action="store_true",
help="Process two dirs-of-stats-dirs, pairwise")
parser.add_argument("--delta-pct-thresh", type=float, default=0.01,
help="Percentage change required to report")
parser.add_argument("--delta-usec-thresh", type=int, default=100000,
help="Absolute delta on times required to report")
parser.add_argument("--lnt-machine", type=str, default=platform.node(),
help="Machine name for LNT submission")
parser.add_argument("--lnt-run-info", action='append', default=[],
type=lambdakv: kv.split("="),
help="Extra key=value pairs for LNT run-info")
parser.add_argument("--lnt-machine-info", action='append', default=[],
type=lambdakv: kv.split("="),
help="Extra key=value pairs for LNT machine-info")
parser.add_argument("--lnt-order", type=str,
default=str(int(time.time())),
help="Order for LNT submission")
parser.add_argument("--lnt-tag", type=str, default="swift-compile",
help="Tag for LNT submission")
parser.add_argument("--lnt-submit", type=str, default=None,
help="URL to submit LNT data to (rather than print)")
parser.add_argument("--select-module",
default=[],
action="append",
help="Select specific modules")
parser.add_argument("--group-by-module",
default=False,
action="store_true",
help="Group stats by module")
parser.add_argument("--select-stat",
default=[],
action="append",
help="Select specific statistics")
parser.add_argument("--select-stats-from-csv-baseline",
type=str, default=None,
help="Select statistics present in a CSV baseline")
parser.add_argument("--exclude-timers",
default=False,
action="store_true",
help="only select counters, exclude timers")
parser.add_argument("--sort-by-delta-pct",
default=False,
action="store_true",
help="Sort comparison results by delta-%%, not stat")
parser.add_argument("--sort-descending",
default=False,
action="store_true",
help="Sort comparison results in descending order")
parser.add_argument("--merge-by",
default="sum",
type=str,
help="Merge identical metrics by (sum|min|max)")
parser.add_argument("--merge-timers",
default=False,
action="store_true",
help="Merge timers across modules/targets/etc.")
parser.add_argument("--divide-by",
default=1,
metavar="D",
type=int,
help="Divide stats by D (to take an average)")
parser.add_argument("--markdown",
default=False,
action="store_true",
help="Write output in markdown table format")
parser.add_argument("--include-unchanged",
default=False,
action="store_true",
help="Include unchanged stats values in comparison")
parser.add_argument("--close-regressions",
default=False,
action="store_true",
help="Close regression details in markdown")
parser.add_argument("--github-emoji",
default=False,
action="store_true",
help="Add github-emoji indicators to markdown")
modes=parser.add_mutually_exclusive_group(required=True)
modes.add_argument("--catapult", action="store_true",
help="emit a 'catapult'-compatible trace of events")
modes.add_argument("--incrementality", action="store_true",
help="summarize the 'incrementality' of a build")
modes.add_argument("--set-csv-baseline", type=str, default=None,
help="Merge stats from a stats-dir into a CSV baseline")
modes.add_argument("--compare-to-csv-baseline", type=str, default=None,
metavar="BASELINE.csv",
help="Compare stats dir to named CSV baseline")
modes.add_argument("--compare-stats-dirs",
action="store_true",
help="Compare two stats dirs directly")
modes.add_argument("--lnt", action="store_true",
help="Emit an LNT-compatible test summary")
modes.add_argument("--evaluate", type=str, default=None,
help="evaluate an expression of stat-names")
modes.add_argument("--evaluate-delta", type=str, default=None,
help="evaluate an expression of stat-deltas")
modes.add_argument("--render-profiles", action="store_true",
help="render any profiles to SVG flamegraphs")
parser.add_argument("--flamegraph-script", type=str, default=None,
help="path to flamegraph.pl")
parser.add_argument("--browse-profiles", action="store_true",
help="open web browser tabs with rendered profiles")
parser.add_argument('remainder', nargs=argparse.REMAINDER,
help="stats-dirs to process")

args=parser.parse_args()
iflen(args.remainder) ==0:
parser.print_help()
return1
try:
returnprocess(args)
finally:
args.output.close()


sys.exit(main())