demangle_tree.py

# Given a path to llvm-objdump and a directory tree, spider the directory tree
# dumping every object file encountered with correct options needed to demangle
# symbols in the object file, and collect statistics about failed / crashed
# demanglings. Useful for stress testing the demangler against a large corpus
# of inputs.

from __future__ importprint_function

importargparse
importfunctools
importos
importre
importsys
importsubprocess
importtraceback
frommultiprocessingimportPool
importmultiprocessing

args=None

defparse_line(line):
question=line.find('?')
ifquestion==-1:
returnNone, None

open_paren=line.find('(', question)
ifopen_paren==-1:
returnNone, None
close_paren=line.rfind(')', open_paren)
ifopen_paren==-1:
returnNone, None
mangled=line[question : open_paren]
demangled=line[open_paren+1 : close_paren]
returnmangled.strip(), demangled.strip()

classResult(object):
def__init__(self):
self.crashed= []
self.file=None
self.nsymbols=0
self.errors=set()
self.nfiles=0

classMapContext(object):
def__init__(self):
self.rincomplete=None
self.rcumulative=Result()
self.pending_objs= []
self.npending=0

defprocess_file(path, objdump):
r=Result()
r.file=path

popen_args= [objdump, '-t', '-demangle', path]
p=subprocess.Popen(popen_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr=p.communicate()
ifp.returncode!=0:
r.crashed= [r.file]
returnr

output=stdout.decode('utf-8')

forlineinoutput.splitlines():
mangled, demangled=parse_line(line)
ifmangledisNone:
continue
r.nsymbols+=1
if"invalid mangled name"indemangled:
r.errors.add(mangled)
returnr

defadd_results(r1, r2):
r1.crashed.extend(r2.crashed)
r1.errors.update(r2.errors)
r1.nsymbols+=r2.nsymbols
r1.nfiles+=r2.nfiles

defprint_result_row(directory, result):
print("[{0} files, {1} crashes, {2} errors, {3} symbols]: '{4}'".format(
result.nfiles, len(result.crashed), len(result.errors), result.nsymbols, directory))

defprocess_one_chunk(pool, chunk_size, objdump, context):
objs= []

incomplete=False
dir_results= {}
ordered_dirs= []
whilecontext.npending>0andlen(objs) <chunk_size:
this_dir=context.pending_objs[0][0]
ordered_dirs.append(this_dir)
re=Result()
ifcontext.rincompleteisnotNone:
re=context.rincomplete
context.rincomplete=None

dir_results[this_dir] =re
re.file=this_dir

nneeded=chunk_size-len(objs)
objs_this_dir=context.pending_objs[0][1]
navail=len(objs_this_dir)
ntaken=min(nneeded, navail)
objs.extend(objs_this_dir[0:ntaken])
remaining_objs_this_dir=objs_this_dir[ntaken:]
context.pending_objs[0] = (context.pending_objs[0][0], remaining_objs_this_dir)
context.npending-=ntaken
ifntaken==navail:
context.pending_objs.pop(0)
else:
incomplete=True

re.nfiles+=ntaken

assert(len(objs) ==chunk_sizeorcontext.npending==0)

copier=functools.partial(process_file, objdump=objdump)
mapped_results=list(pool.map(copier, objs))

formrinmapped_results:
result_dir=os.path.dirname(mr.file)
result_entry=dir_results[result_dir]
add_results(result_entry, mr)

# It's only possible that a single item is incomplete, and it has to be the
# last item.
ifincomplete:
context.rincomplete=dir_results[ordered_dirs[-1]]
ordered_dirs.pop()

# Now ordered_dirs contains a list of all directories which *did* complete.
forcinordered_dirs:
re=dir_results[c]
add_results(context.rcumulative, re)
print_result_row(c, re)

defprocess_pending_files(pool, chunk_size, objdump, context):
whilecontext.npending>=chunk_size:
process_one_chunk(pool, chunk_size, objdump, context)

defgo():
globalargs

obj_dir=args.dir
extensions=args.extensions.split(',')
extensions= [xifx[0] =='.'else'.'+xforxinextensions]


pool_size=48
pool=Pool(processes=pool_size)

try:
nfiles=0
context=MapContext()

forroot, dirs, filesinos.walk(obj_dir):
root=os.path.normpath(root)
pending= []
forfinfiles:
file, ext=os.path.splitext(f)
ifnotextinextensions:
continue

nfiles+=1
full_path=os.path.join(root, f)
full_path=os.path.normpath(full_path)
pending.append(full_path)

# If this directory had no object files, just print a default
# status line and continue with the next dir
iflen(pending) ==0:
print_result_row(root, Result())
continue

context.npending+=len(pending)
context.pending_objs.append((root, pending))
# Drain the tasks, `pool_size` at a time, until we have less than
# `pool_size` tasks remaining.
process_pending_files(pool, pool_size, args.objdump, context)

assert(context.npending<pool_size);
process_one_chunk(pool, pool_size, args.objdump, context)

total=context.rcumulative
nfailed=len(total.errors)
nsuccess=total.nsymbols-nfailed
ncrashed=len(total.crashed)

if (nfailed>0):
print("Failures:")
forminsorted(total.errors):
print(" "+m)
if (ncrashed>0):
print("Crashes:")
forfinsorted(total.crashed):
print(" "+f)
print("Summary:")
spct=float(nsuccess)/float(total.nsymbols)
fpct=float(nfailed)/float(total.nsymbols)
cpct=float(ncrashed)/float(nfiles)
print("Processed {0} object files.".format(nfiles))
print("{0}/{1} symbols successfully demangled ({2:.4%})".format(nsuccess, total.nsymbols, spct))
print("{0} symbols could not be demangled ({1:.4%})".format(nfailed, fpct))
print("{0} files crashed while demangling ({1:.4%})".format(ncrashed, cpct))

except:
traceback.print_exc()

pool.close()
pool.join()

if__name__=="__main__":
def_obj='obj'ifsys.platform=='win32'else'o'

parser=argparse.ArgumentParser(description='Demangle all symbols in a tree of object files, looking for failures.')
parser.add_argument('dir', type=str, help='the root directory at which to start crawling')
parser.add_argument('--objdump', type=str, help='path to llvm-objdump. If not specified '+
'the tool is located as if by `which llvm-objdump`.')
parser.add_argument('--extensions', type=str, default=def_obj,
help='comma separated list of extensions to demangle (e.g. `o,obj`). '+
'By default this will be `obj` on Windows and `o` otherwise.')

args=parser.parse_args()


multiprocessing.freeze_support()
go()