swift_build_sdk_interfaces.py

#!/usr/bin/env python3

importargparse
importerrno
importitertools
importjson
importmultiprocessing
importos
importshutil
importsubprocess
importsys
importtraceback
fromshutilimportcopyfile

BARE_INTERFACE_SEARCH_PATHS= [
"usr/lib/swift",
"System/iOSSupport/usr/lib/swift"
]
DEFAULT_FRAMEWORK_INTERFACE_SEARCH_PATHS= [
"System/Library/Frameworks",
"System/iOSSupport/System/Library/Frameworks"
]
STDLIB_NAME='Swift'

MONOTONIC_VERSION=1


defcreate_parser():
parser=argparse.ArgumentParser(
description="Builds an SDK's swiftinterfaces into swiftmodules. "
"Always searches usr/lib/swift in addition to whichever "
"framework directories are passed on the command line.",
prog=os.path.basename(__file__),
usage='%(prog)s -o output/ [INTERFACE_SEARCH_DIRS]',
epilog='Environment variables: SDKROOT, SWIFT_EXEC, '
'SWIFT_FORCE_MODULE_LOADING')
parser.add_argument('interface_framework_dirs', nargs='*',
metavar='INTERFACE_SEARCH_DIRS',
help='Relative paths to search for frameworks with '
'interfaces (default: System/Library/Frameworks)')
parser.add_argument('-o', dest='output_dir',
help='Directory to which the output will be emitted '
'(required)')
parser.add_argument('-j', dest='jobs', type=int,
help='The number of parallel jobs to execute '
'(default: # of cores)')
parser.add_argument('-v', dest='verbose', action='store_true',
help='Print command invocations and progress info')
parser.add_argument('-n', dest='dry_run', action='store_true',
help='Dry run: don\'t actually run anything')
parser.add_argument('-sdk', default=os.getenv('SDKROOT'),
help='SDK to find frameworks and interfaces in '
'(default: $SDKROOT)')
parser.add_argument('-F', dest='framework_dirs', metavar='DIR',
action='append', default=[],
help='Add additional framework search paths')
parser.add_argument('-Fsystem', '-iframework',
dest='system_framework_dirs', metavar='DIR',
action='append', default=[],
help='Add additional system framework search paths')
parser.add_argument('-Fsystem-iosmac',
dest='iosmac_system_framework_dirs', metavar='DIR',
action='append', default=[],
help='Add system framework search paths '
'for iOSMac only')
parser.add_argument('-I', dest='include_dirs', metavar='DIR',
action='append', default=[],
help='Add additional header/module search paths')
parser.add_argument('-module-cache-path',
help='Temporary directory to store intermediate info')
parser.add_argument('-log-path',
help='Directory to write stdout/stderr output to')
parser.add_argument('-skip-stdlib', action='store_true',
help='Don\'t build the standard library interface')
parser.add_argument('-disable-modules-validate-system-headers',
action='store_true',
help='Disable modules verification for system headers')
parser.add_argument('-xfails', metavar='PATH',
help='JSON file containing an array of the modules '
'expected to fail')
parser.add_argument('-check-only', action='store_true',
help='Assume the resulting modules will be thrown '
'away (may be faster)')
parser.add_argument('-ignore-non-stdlib-failures', action='store_true',
help='Treat all modules but the stdlib as XFAILed')
parser.add_argument('-debug-crash-compiler', action='store_true',
help='Have the compiler crash (for testing purposes)')
parser.add_argument('-machine-parseable-monotonic-version',
action='store_true',
help='For comparing versions of this tool')
returnparser


deffatal(msg):
print(msg, file=sys.stderr)
sys.exit(1)


defrun_command(args, dry_run):
ifdry_run:
return (0, "", "")
proc=subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
out, err=proc.communicate()
exitcode=proc.returncode
return (exitcode, out.decode('utf-8'), err.decode('utf-8'))
exceptKeyboardInterrupt:
proc.terminate()
raise


defmake_dirs_if_needed(path):
try:
os.makedirs(path)
exceptOSErrorase:
ife.errno!=errno.EEXIST:
raise


classNegatedSet:
def__init__(self, contents):
self._contents=frozenset(contents)

def__contains__(self, item):
returnitemnotinself._contents


classModuleFile:
def__init__(self, name, path, is_expected_to_fail):
self.name=name
self.path=path
self.is_expected_to_fail=is_expected_to_fail


defcollect_slices(xfails, swiftmodule_dir):
ifnotos.path.isdir(swiftmodule_dir):
return
module_name, extension= \
os.path.splitext(os.path.basename(swiftmodule_dir))
assertextension==".swiftmodule"

is_xfail=module_nameinxfails
forentryinos.listdir(swiftmodule_dir):
_, extension=os.path.splitext(entry)
ifextension==".swiftinterface":
yieldModuleFile(module_name, os.path.join(swiftmodule_dir, entry),
is_xfail)


defcollect_framework_modules(sdk, xfails, sdk_relative_framework_dirs):
forsdk_relative_framework_dirinsdk_relative_framework_dirs:
framework_dir=os.path.join(sdk, sdk_relative_framework_dir)
ifnotos.access(framework_dir, os.R_OK):
continue
forentryinos.listdir(framework_dir):
path_without_extension, extension=os.path.splitext(entry)
ifextension!=".framework":
continue
module_name=os.path.basename(path_without_extension)
swiftmodule=os.path.join(framework_dir, entry, "Modules",
module_name+".swiftmodule")
ifos.access(swiftmodule, os.R_OK):
forxincollect_slices(xfails, swiftmodule):
yieldx


defcollect_non_framework_modules(sdk, xfails, sdk_relative_search_dirs):
forsdk_relative_search_dirinsdk_relative_search_dirs:
search_dir=os.path.join(sdk, sdk_relative_search_dir)
fordir_path, _, file_namesinos.walk(search_dir, followlinks=True):
ifos.path.splitext(dir_path)[1] ==".swiftmodule":
forxincollect_slices(xfails, dir_path):
yieldx
else:
forinterfaceinfile_names:
module_name, extension=os.path.splitext(interface)
ifextension==".swiftinterface":
is_xfail=module_nameinxfails
yieldModuleFile(module_name,
os.path.join(dir_path, interface),
is_xfail)


defshould_retry_compilation(stderr):
if"has been modified since the module file"instderr:
returnTrue
if"mismatched umbrella headers in submodule"instderr:
returnTrue
if"is out of date and needs to be rebuilt: signature mismatch"instderr:
returnTrue
if"current parser token 'include'"instderr:
returnTrue
if"current parser token 'import'"instderr:
returnTrue
returnFalse


defrun_with_module_cache_retry(command_args, module_cache_path, dry_run):
"""Hack: runs a command several times, clearing the module cache if we get
 an error about header files being modified during the run.

 This shouldn't be necessary (the cached PCM files should automatically be
 regenerated) but there seems to still be a bug in Clang that we haven't
 tracked down yet.
 """
RETRIES=3
attempts_stderr=""
forrinrange(RETRIES):
status, stdout, stderr=run_command(command_args, dry_run)
ifstatus==0:
break
ifnotshould_retry_compilation(stderr):
break
ifmodule_cache_path:
shutil.rmtree(module_cache_path, ignore_errors=True)
# If all retries fail, output information for each instance.
attempts_stderr+= (
"\n*** Compilation attempt {}/{} failed with modules bugs. "
"Error output:\n".format(r+1, RETRIES))
attempts_stderr+=stderr
stderr=attempts_stderr
return (status, stdout, stderr)


deflog_output_to_file(content, module_name, interface_base, label, log_path):
ifnotlog_path:
return
ifnotcontent:
return
make_dirs_if_needed(log_path)
log_name=module_name+"-"+interface_base+"-"+label+".txt"
withopen(os.path.join(log_path, log_name), "w") asoutput_file:
output_file.write(content)


deflooks_like_iosmac(interface_base):
return'ios-macabi'ininterface_base


defprocess_module(module_file):
globalargs, shared_output_lock
try:
interface_base, _= \
os.path.splitext(os.path.basename(module_file.path))

swiftc=os.getenv('SWIFT_EXEC',
os.path.join(os.path.dirname(__file__), 'swiftc'))
command_args= [
swiftc, '-frontend',
'-build-module-from-parseable-interface',
'-sdk', args.sdk,
'-prebuilt-module-cache-path', args.output_dir,
 ]
module_cache_path=""
ifargs.module_cache_path:
module_cache_path=os.path.join(args.module_cache_path,
str(os.getpid()))
command_args+= ('-module-cache-path', module_cache_path)
ifargs.debug_crash_compiler:
command_args+= ('-debug-crash-immediately',)
ifnotargs.check_only:
command_args+= (
'-serialize-parseable-module-interface-dependency-hashes',)
ifargs.disable_modules_validate_system_headers:
command_args+= (
'-disable-modules-validate-system-headers',)

# FIXME: This shouldn't be necessary, but the module name is checked
# before the frontend action is.
ifmodule_file.name==STDLIB_NAME:
command_args+= ('-parse-stdlib',)

# FIXME: Some Python installations are unable to handle Unicode
# properly. Narrow this once we figure out how to detect them.
command_args+= ('-diagnostic-style', 'llvm')

iflooks_like_iosmac(interface_base):
forsystem_framework_pathinargs.iosmac_system_framework_dirs:
command_args+= ('-Fsystem', system_framework_path)
command_args+= ('-Fsystem', os.path.join(args.sdk, "System",
"iOSSupport", "System",
"Library", "Frameworks"))

forinclude_pathinargs.include_dirs:
command_args+= ('-I', include_path)
forsystem_framework_pathinargs.system_framework_dirs:
command_args+= ('-Fsystem', system_framework_path)
forframework_pathinargs.framework_dirs:
command_args+= ('-F', framework_path)

command_args+= ('-module-name', module_file.name, module_file.path)

output_path=os.path.join(args.output_dir,
module_file.name+".swiftmodule")

ifinterface_base!=module_file.name:
make_dirs_if_needed(output_path)
output_path=os.path.join(output_path,
interface_base+".swiftmodule")

command_args+= ('-o', output_path)

ifargs.verbose:
withshared_output_lock:
print("# Starting "+module_file.path)
print(' '.join(command_args))
sys.stdout.flush()
status, stdout, stderr=run_with_module_cache_retry(
command_args, module_cache_path=module_cache_path,
dry_run=args.dry_run)
log_output_to_file(stdout, module_file.name, interface_base, "out",
log_path=args.log_path)
log_output_to_file(stderr, module_file.name, interface_base, "err",
log_path=args.log_path)
return (module_file, status, stdout, stderr)
exceptBaseException:
# We're catching everything here because we don't want to take down the
# other jobs.
return (module_file, 1, "",
"".join(traceback.format_exception(*sys.exc_info())))


defset_up_child(parent_args, lock):
globalargs, shared_output_lock
args=parent_args
shared_output_lock=lock


defprocess_module_files(pool, module_files):
results=pool.imap_unordered(process_module, module_files)

overall_exit_status=0
for (module_file, exit_status, stdout, stderr) inresults:
withshared_output_lock:
ifexit_status!=0:
print("# ", end="")
ifmodule_file.is_expected_to_fail:
print("(XFAIL) ", end="")
else:
print("(FAIL) ", end="")
print(module_file.path)
if (notmodule_file.is_expected_to_fail) orargs.verbose:
print(stdout, end="")
print(stderr, end="", file=sys.stderr)
elifmodule_file.is_expected_to_fail:
print("# (UPASS) "+module_file.path)
elifargs.verbose:
print("# (PASS) "+module_file.path)
sys.stdout.flush()
ifoverall_exit_status==0and \
notmodule_file.is_expected_to_fail:
overall_exit_status=exit_status
returnoverall_exit_status


defgetSDKVersion(sdkroot):
settingPath=os.path.join(sdkroot, 'SDKSettings.json')
withopen(settingPath) asjson_file:
data=json.load(json_file)
returndata['Version']
fatal("Failed to get SDK version from: "+settingPath)


defcopySystemVersionFile(sdkroot, output):
sysInfoPath=os.path.join(sdkroot,
'System/Library/CoreServices/SystemVersion.plist')
destInfoPath=os.path.join(output, 'SystemVersion.plist')
try:
copyfile(sysInfoPath, destInfoPath)
exceptBaseExceptionase:
print("cannot copy from "+sysInfoPath+" to "+destInfoPath+": "+str(e))


defmain():
globalargs, shared_output_lock
parser=create_parser()
args=parser.parse_args()

ifargs.machine_parseable_monotonic_version:
print(MONOTONIC_VERSION)
sys.exit(0)

if'SWIFT_FORCE_MODULE_LOADING'notinos.environ:
os.environ['SWIFT_FORCE_MODULE_LOADING'] ='prefer-serialized'

ifnotargs.output_dir:
fatal("argument -o is required")
ifnotargs.sdk:
fatal("SDKROOT must be set in the environment")
ifnotos.path.isdir(args.sdk):
fatal("invalid SDK: "+args.sdk)

# if the given output dir ends with 'prebuilt-modules', we should
# append the SDK version number so all modules will built into
# the SDK-versioned sub-directory.
ifos.path.basename(args.output_dir) =='prebuilt-modules':
args.output_dir=os.path.join(args.output_dir, getSDKVersion(args.sdk))

xfails= ()
ifargs.ignore_non_stdlib_failures:
ifargs.xfails:
print("warning: ignoring -xfails because "
"-ignore-non-stdlib-failures was provided", file=sys.stderr)
xfails=NegatedSet((STDLIB_NAME,))
elifargs.xfails:
withopen(args.xfails) asxfails_file:
xfails=json.load(xfails_file)

make_dirs_if_needed(args.output_dir)

# Copy a file containing SDK build version into the prebuilt module dir,
# so we can keep track of the SDK version we built from.
copySystemVersionFile(args.sdk, args.output_dir)
shared_output_lock=multiprocessing.Lock()
pool=multiprocessing.Pool(args.jobs, set_up_child,
 (args, shared_output_lock))

interface_framework_dirs= (args.interface_framework_dirsor
DEFAULT_FRAMEWORK_INTERFACE_SEARCH_PATHS)

module_files=list(itertools.chain(
collect_non_framework_modules(args.sdk, xfails,
BARE_INTERFACE_SEARCH_PATHS),
collect_framework_modules(args.sdk, xfails, interface_framework_dirs)))

ifnotargs.skip_stdlib:
# Always do the stdlib first, so that we can use it in later steps
stdlib_module_files= (
xforxinmodule_filesifx.name==STDLIB_NAME)
status=process_module_files(pool, stdlib_module_files)
ifstatus!=0:
sys.exit(status)

non_stdlib_module_files= (
xforxinmodule_filesifx.name!=STDLIB_NAME)
status=process_module_files(pool, non_stdlib_module_files)
ifos.name=='nt':
importctypes
Kernel32=ctypes.cdll.LoadLibrary("Kernel32.dll")
Kernel32.ExitProcess(ctypes.c_ulong(status))
sys.exit(status)


if__name__=='__main__':
main()