bisect_cmd.py

#!/usr/bin/env python3
"""
Command line tool to bisect failing CPython tests.

Find the test_os test method which alters the environment:

 ./python -m test.bisect_cmd --fail-env-changed test_os

Find a reference leak in "test_os", write the list of failing tests into the
"bisect" file:

 ./python -m test.bisect_cmd -o bisect -R 3:3 test_os

Load an existing list of tests from a file using -i option:

 ./python -m test --list-cases -m FileTests test_os > tests
 ./python -m test.bisect_cmd -i tests test_os
"""

importargparse
importdatetime
importos.path
importmath
importrandom
importsubprocess
importsys
importtempfile
importtime


defwrite_tests(filename, tests):
withopen(filename, "w") asfp:
fornameintests:
print(name, file=fp)
fp.flush()


defwrite_output(filename, tests):
ifnotfilename:
return
print("Writing %s tests into %s"% (len(tests), filename))
write_tests(filename, tests)
returnfilename


defformat_shell_args(args):
return' '.join(args)


deflist_cases(args):
cmd= [sys.executable, '-m', 'test', '--list-cases']
cmd.extend(args.test_args)
proc=subprocess.run(cmd,
stdout=subprocess.PIPE,
universal_newlines=True)
exitcode=proc.returncode
ifexitcode:
cmd=format_shell_args(cmd)
print("Failed to list tests: %s failed with exit code %s"
% (cmd, exitcode))
sys.exit(exitcode)
tests=proc.stdout.splitlines()
returntests


defrun_tests(args, tests, huntrleaks=None):
tmp=tempfile.mktemp()
try:
write_tests(tmp, tests)

cmd= [sys.executable, '-m', 'test', '--matchfile', tmp]
cmd.extend(args.test_args)
print("+ %s"%format_shell_args(cmd))
proc=subprocess.run(cmd)
returnproc.returncode
finally:
ifos.path.exists(tmp):
os.unlink(tmp)


defparse_args():
parser=argparse.ArgumentParser()
parser.add_argument('-i', '--input',
help='Test names produced by --list-tests written '
'into a file. If not set, run --list-tests')
parser.add_argument('-o', '--output',
help='Result of the bisection')
parser.add_argument('-n', '--max-tests', type=int, default=1,
help='Maximum number of tests to stop the bisection '
'(default: 1)')
parser.add_argument('-N', '--max-iter', type=int, default=100,
help='Maximum number of bisection iterations '
'(default: 100)')
# FIXME: document that following arguments are test arguments

args, test_args=parser.parse_known_args()
args.test_args=test_args
returnargs


defmain():
args=parse_args()

ifargs.input:
withopen(args.input) asfp:
tests= [line.strip() forlineinfp]
else:
tests=list_cases(args)

print("Start bisection with %s tests"%len(tests))
print("Test arguments: %s"%format_shell_args(args.test_args))
print("Bisection will stop when getting %s or less tests "
"(-n/--max-tests option), or after %s iterations "
"(-N/--max-iter option)"
% (args.max_tests, args.max_iter))
output=write_output(args.output, tests)
print()

start_time=time.monotonic()
iteration=1
try:
whilelen(tests) >args.max_testsanditeration<=args.max_iter:
ntest=len(tests)
ntest=max(ntest//2, 1)
subtests=random.sample(tests, ntest)

print("[+] Iteration %s: run %s tests/%s"
% (iteration, len(subtests), len(tests)))
print()

exitcode=run_tests(args, subtests)

print("ran %s tests/%s"% (ntest, len(tests)))
print("exit", exitcode)
ifexitcode:
print("Tests failed: continuing with this subtest")
tests=subtests
output=write_output(args.output, tests)
else:
print("Tests succeeded: skipping this subtest, trying a new subset")
print()
iteration+=1
exceptKeyboardInterrupt:
print()
print("Bisection interrupted!")
print()

print("Tests (%s):"%len(tests))
fortestintests:
print("* %s"%test)
print()

ifoutput:
print("Output written into %s"%output)

dt=math.ceil(time.monotonic() -start_time)
iflen(tests) <=args.max_tests:
print("Bisection completed in %s iterations and %s"
% (iteration, datetime.timedelta(seconds=dt)))
sys.exit(1)
else:
print("Bisection failed after %s iterations and %s"
% (iteration, datetime.timedelta(seconds=dt)))


if__name__=="__main__":
main()