| #!/usr/bin/env python |
| # utils/build-parser-lib - Helper tool for building the parser library -*- python -*- |
| # |
| # This source file is part of the Swift.org open source project |
| # |
| # Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors |
| # Licensed under Apache License v2.0 with Runtime Library Exception |
| # |
| # See https://swift.org/LICENSE.txt for license information |
| # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| |
| # This is a utility for building only the syntax parser library as fast as possible |
| # by only building the necessary libraries for the parser library and nothing |
| # extraneous. To achieve this it does a single unified CMake configuration for |
| # llvm/clang/swift and builds only the parser library target. |
| # This mechanism is fundamentally different from build-script, which builds llvm/clang |
| # in a separate build directory than swift. |
| # |
| # Even though this bypasses build-script, it does share some underlying helper |
| # utilities from the python infrastructure. |
| # |
| # This utility also provides capability to gather profile data and build the parser |
| # library with PGO optimization enabled. |
| |
| from __future__ import print_function |
| |
| import copy |
| import multiprocessing |
| import os |
| import platform |
| import sys |
| |
| from build_swift.build_swift import argparse |
| from build_swift.build_swift import defaults |
| from build_swift.build_swift.constants import SWIFT_BUILD_ROOT |
| from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT |
| from build_swift.build_swift.wrappers import xcrun |
| |
| from swift_build_support.swift_build_support import shell |
| from swift_build_support.swift_build_support.toolchain import host_toolchain |
| |
| isDarwin = platform.system() == "Darwin" |
| |
| |
| class Builder(object): |
| def __init__( |
| self, toolchain, args, host, arch, profile_data=None, native_build_dir=None |
| ): |
| self.toolchain = toolchain |
| self.ninja_path = args.ninja_path |
| self.build_release = args.release |
| self.enable_assertions = not args.no_assertions |
| self.lto_type = args.lto_type |
| self.pgo_type = args.pgo_type |
| self.profile_input = args.profile_input |
| self.profile_data = profile_data |
| self.dry_run = args.dry_run |
| self.jobs = args.build_jobs |
| self.verbose = args.verbose |
| self.build_dir = args.build_dir |
| self.install_destdir = args.install_destdir |
| self.install_prefix = args.install_prefix |
| self.version = args.version |
| self.host = host |
| self.arch = arch |
| self.native_build_dir = native_build_dir |
| |
| def call(self, command, env=None, without_sleeping=False): |
| if without_sleeping: |
| shell.call_without_sleeping( |
| command, env=env, dry_run=self.dry_run, echo=self.verbose |
| ) |
| else: |
| shell.call(command, env=env, dry_run=self.dry_run, echo=self.verbose) |
| |
| def configure(self, enable_debuginfo, instrumentation=None, profile_data=None): |
| cmake_args = [self.toolchain.cmake, "-G", "Ninja"] |
| cmake_args += ["-DCMAKE_MAKE_PROGRAM=" + self.ninja_path] |
| |
| isEmbeddedHost = isDarwin and self.host != "macosx" |
| host_triple = None |
| host_sdk = None |
| llvm_c_flags = "-arch " + self.arch |
| |
| if self.host == "macosx": |
| deployment_version = "10.12" |
| host_triple = "%s-apple-macosx%s" % (self.arch, deployment_version) |
| host_sdk = "OSX" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=" + deployment_version, |
| ] |
| |
| elif self.host == "linux": |
| host_triple = "%s-unknown-linux" % (self.arch) |
| host_sdk = "LINUX" |
| |
| elif self.host == "iphonesimulator": |
| deployment_version = "10.0" |
| host_triple = "%s-apple-ios%s-simulator" % (self.arch, deployment_version) |
| host_sdk = "IOS_SIMULATOR" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version, |
| ] |
| llvm_c_flags += " -mios-simulator-version-min=" + deployment_version |
| |
| elif self.host == "iphoneos": |
| deployment_version = "10.0" |
| host_triple = "%s-apple-ios%s" % (self.arch, deployment_version) |
| host_sdk = "IOS" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version, |
| ] |
| llvm_c_flags += " -miphoneos-version-min=" + deployment_version |
| |
| elif self.host == "appletvsimulator": |
| deployment_version = "10.0" |
| host_triple = "%s-apple-tvos%s-simulator" % (self.arch, deployment_version) |
| host_sdk = "TVOS_SIMULATOR" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version, |
| ] |
| llvm_c_flags += " -mtvos-simulator-version-min=" + deployment_version |
| |
| elif self.host == "appletvos": |
| deployment_version = "10.0" |
| host_triple = "%s-apple-tvos%s" % (self.arch, deployment_version) |
| host_sdk = "TVOS" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version, |
| ] |
| llvm_c_flags += " -mtvos-version-min=" + deployment_version |
| |
| elif self.host == "watchsimulator": |
| deployment_version = "3.0" |
| host_triple = "%s-apple-watchos%s-simulator" % ( |
| self.arch, |
| deployment_version, |
| ) |
| host_sdk = "WATCHOS_SIMULATOR" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version, |
| ] |
| llvm_c_flags += " -mwatchos-simulator-version-min=" + deployment_version |
| |
| elif self.host == "watchos": |
| deployment_version = "3.0" |
| host_triple = "%s-apple-watchos%s" % (self.arch, deployment_version) |
| host_sdk = "WATCHOS" |
| cmake_args += [ |
| "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, |
| "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version, |
| ] |
| llvm_c_flags += " -mwatchos-version-min=" + deployment_version |
| |
| assert host_triple |
| assert host_sdk |
| cmake_args += [ |
| "-DLLVM_HOST_TRIPLE:STRING=" + host_triple, |
| "-DLLVM_TARGET_ARCH=" + self.arch, |
| "-DSWIFT_HOST_VARIANT=" + self.host, |
| "-DSWIFT_HOST_VARIANT_SDK=" + host_sdk, |
| "-DSWIFT_HOST_VARIANT_ARCH=" + self.arch, |
| "-DCMAKE_C_FLAGS=" + llvm_c_flags, |
| "-DCMAKE_CXX_FLAGS=" + llvm_c_flags, |
| ] |
| if isEmbeddedHost: |
| cmake_args += [ |
| "-DCMAKE_OSX_SYSROOT:PATH=" + xcrun.sdk_path(self.host), |
| # For embedded hosts CMake runs the checks and triggers crashes because |
| # the test binary was built for embedded host. |
| "-DHAVE_POSIX_REGEX:BOOL=TRUE", |
| "-DHAVE_STEADY_CLOCK:BOOL=TRUE", |
| ] |
| |
| if isDarwin: |
| if self.native_build_dir is not None: |
| cmake_args += [ |
| "-DLLVM_TABLEGEN=" |
| + os.path.join(self.native_build_dir, "bin", "llvm-tblgen"), |
| "-DCLANG_TABLEGEN=" |
| + os.path.join(self.native_build_dir, "bin", "clang-tblgen"), |
| "-DLLVM_NATIVE_BUILD=" + self.native_build_dir, |
| "-DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING=" |
| + os.path.join(self.native_build_dir, "bin"), |
| "-DSWIFT_NATIVE_CLANG_TOOLS_PATH:STRING=" |
| + os.path.join(self.native_build_dir, "bin"), |
| "-DSWIFT_NATIVE_SWIFT_TOOLS_PATH:STRING=" |
| + os.path.join(self.native_build_dir, "bin"), |
| ] |
| else: |
| dispatch_source_path = os.path.join( |
| SWIFT_SOURCE_ROOT, "swift-corelibs-libdispatch" |
| ) |
| cmake_args += [ |
| "-DSWIFT_PATH_TO_LIBDISPATCH_SOURCE:PATH=" + dispatch_source_path, |
| "-DLLVM_ENABLE_LLD=ON", |
| ] |
| cmake_args += ["-DLLVM_TARGETS_TO_BUILD=X86"] |
| if self.build_release: |
| if enable_debuginfo: |
| cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo"] |
| else: |
| cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Release"] |
| else: |
| cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Debug"] |
| if self.enable_assertions: |
| cmake_args += ["-DLLVM_ENABLE_ASSERTIONS:BOOL=ON"] |
| if instrumentation: |
| cmake_args += ["-DLLVM_BUILD_INSTRUMENTED=" + instrumentation] |
| if profile_data: |
| cmake_args += ["-DLLVM_PROFDATA_FILE=" + profile_data] |
| if self.lto_type and not instrumentation: |
| cmake_args += ["-DLLVM_ENABLE_LTO=" + self.lto_type.upper()] |
| if self.install_prefix: |
| cmake_args += ["-DCMAKE_INSTALL_PREFIX:PATH=" + self.install_prefix] |
| if self.version: |
| cmake_args += ["-DSWIFT_LIBPARSER_VER:STRING=" + self.version] |
| cmake_args += [ |
| "-DLLVM_ENABLE_PROJECTS=clang;swift", |
| "-DLLVM_EXTERNAL_PROJECTS=swift", |
| ] |
| cmake_args += ["-DSWIFT_BUILD_ONLY_SYNTAXPARSERLIB=TRUE"] |
| cmake_args += ["-DSWIFT_BUILD_PERF_TESTSUITE=NO", "-DSWIFT_INCLUDE_DOCS=NO"] |
| cmake_args += [ |
| "-DSWIFT_BUILD_REMOTE_MIRROR=FALSE", |
| "-DSWIFT_BUILD_DYNAMIC_STDLIB=FALSE", |
| "-DSWIFT_BUILD_STATIC_STDLIB=FALSE", |
| "-DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=FALSE", |
| "-DSWIFT_BUILD_STATIC_SDK_OVERLAY=FALSE", |
| ] |
| cmake_args += [ |
| "-DLLVM_ENABLE_LIBXML2=FALSE", |
| "-DLLVM_ENABLE_LIBEDIT=FALSE", |
| "-DLLVM_ENABLE_TERMINFO=FALSE", |
| "-DLLVM_ENABLE_ZLIB=FALSE", |
| ] |
| # We are not using cmark but initialize the CMARK variables to something so |
| # that configure can succeed. |
| cmake_args += [ |
| "-DCMARK_MAIN_INCLUDE_DIR=" + os.path.join(SWIFT_SOURCE_ROOT, "cmark"), |
| "-DCMARK_BUILD_INCLUDE_DIR=" + os.path.join(self.build_dir, "cmark"), |
| ] |
| cmake_args += [ |
| "-DLLVM_INCLUDE_TESTS=FALSE", |
| "-DCLANG_INCLUDE_TESTS=FALSE", |
| "-DSWIFT_INCLUDE_TESTS=FALSE", |
| ] |
| cmake_args += [os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "llvm")] |
| self.call(cmake_args) |
| |
| def build_target(self, build_dir, target, env=None): |
| invocation = [self.toolchain.cmake, "--build", build_dir] |
| invocation += ["--", "-j%d" % self.jobs] |
| if self.verbose: |
| invocation += ["-v"] |
| invocation += [target] |
| self.call(invocation, env=env, without_sleeping=True) |
| |
| def install(self): |
| print("--- Installing ---", file=sys.stderr) |
| env = None |
| if self.install_destdir: |
| env = {"DESTDIR": self.install_destdir} |
| self.build_target( |
| self.build_dir, "tools/swift/tools/libSwiftSyntaxParser/install", env=env |
| ) |
| |
| def get_profile_data(self, profile_dir): |
| shell.makedirs(profile_dir, dry_run=self.dry_run) |
| instrumentation = "IR" if self.pgo_type == "ir" else "Frontend" |
| with shell.pushd(profile_dir, dry_run=self.dry_run): |
| self.configure(enable_debuginfo=False, instrumentation=instrumentation) |
| self.build_target(profile_dir, "swift-syntax-parser-test") |
| # Delete existing profile data that were generated during building from |
| # running tablegen. |
| shell.rmtree("profiles", dry_run=self.dry_run) |
| self.call( |
| [ |
| os.path.join("bin", "swift-syntax-parser-test"), |
| self.profile_input, |
| "-time", |
| ] |
| ) |
| self.call( |
| [ |
| self.toolchain.llvm_profdata, |
| "merge", |
| "-output=profdata.prof", |
| "profiles", |
| ] |
| ) |
| |
| def run(self): |
| shell.makedirs(self.build_dir, dry_run=self.dry_run) |
| |
| with shell.pushd(self.build_dir, dry_run=self.dry_run): |
| self.configure(enable_debuginfo=True, profile_data=self.profile_data) |
| |
| self.build_target(self.build_dir, "swift-syntax-parser-test") |
| |
| if self.install_destdir: |
| self.install() |
| |
| |
| def extract_symbols(install_destdir, install_prefix, install_symroot, jobs): |
| if not isDarwin: |
| return |
| extract_script = os.path.join( |
| SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "darwin-extract-symbols" |
| ) |
| print("--- Extracting symbols ---", file=sys.stderr) |
| env = { |
| "INSTALL_DIR": install_destdir, |
| "INSTALL_PREFIX": install_prefix, |
| "INSTALL_SYMROOT": install_symroot, |
| "BUILD_JOBS": str(jobs), |
| } |
| shell.call([extract_script], env=env) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser( |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| description=""" |
| Builds Swift Syntax Parser library. |
| |
| Example invocations: |
| |
| * Building for host (macOS, linux): |
| |
| $ utils/build-parser-lib --release --no-assertions --build-dir /tmp/parser-lib-build |
| |
| * Building for iOS |
| |
| $ utils/build-parser-lib --release --no-assertions --build-dir \ |
| /tmp/parser-lib-build-iossim --host iphonesimulator --architectures x86_64 |
| $ utils/build-parser-lib --release --no-assertions --build-dir \ |
| /tmp/parser-lib-build-ios --host iphoneos --architectures arm64 |
| |
| """, |
| ) |
| optbuilder = parser.to_builder() |
| option = optbuilder.add_option |
| store = optbuilder.actions.store |
| store_true = optbuilder.actions.store_true |
| store_int = optbuilder.actions.store_int |
| store_path = optbuilder.actions.store_path |
| |
| toolchain = host_toolchain(xcrun_toolchain="default") |
| default_host = "macosx" if isDarwin else "linux" |
| default_architectures = platform.machine() |
| |
| default_profile_input = os.path.join( |
| SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "profile-input.swift" |
| ) |
| default_jobs = multiprocessing.cpu_count() |
| default_build_dir = os.path.join(SWIFT_BUILD_ROOT, "parser-lib") |
| default_install_prefix = ( |
| defaults.DARWIN_INSTALL_PREFIX if isDarwin else defaults.UNIX_INSTALL_PREFIX |
| ) |
| default_ninja = toolchain.ninja |
| |
| option("--release", store_true, help="build in release mode") |
| option( |
| "--lto", |
| store("lto_type"), |
| choices=["thin", "full"], |
| const="full", |
| metavar="LTO_TYPE", |
| help="use lto optimization." |
| "Options: thin, full. If no optional arg is provided, full is " |
| "chosen by default", |
| ) |
| option( |
| "--pgo", |
| store("pgo_type"), |
| choices=["frontend", "ir"], |
| const="ir", |
| metavar="PGO_TYPE", |
| help="use pgo optimization." |
| "Options: frontend, ir. If no optional arg is provided, ir is " |
| "chosen by default", |
| ) |
| option( |
| "--profile-input", |
| store_path, |
| default=default_profile_input, |
| help="the source file to use for PGO profiling input (default = %s)" |
| % default_profile_input, |
| ) |
| option("--no-assertions", store_true, help="disable assertions") |
| option( |
| ["-v", "--verbose"], |
| store_true, |
| help="print the commands executed during the build", |
| ) |
| option( |
| "--dry-run", |
| store_true, |
| help="print the commands to execute but not actually execute them", |
| ) |
| option( |
| ["-j", "--jobs"], |
| store_int("build_jobs"), |
| default=default_jobs, |
| help="the number of parallel build jobs to use (default = %s)" % default_jobs, |
| ) |
| option( |
| "--build-dir", |
| store_path, |
| default=default_build_dir, |
| help="the path where the build products will be placed. (default = %s)" |
| % default_build_dir, |
| ) |
| option( |
| "--host", |
| store, |
| choices=[ |
| "macosx", |
| "linux", |
| "iphonesimulator", |
| "iphoneos", |
| "appletvsimulator", |
| "appletvos", |
| "watchsimulator", |
| "watchos", |
| ], |
| default=default_host, |
| help="host platform to build for (default = %s)" % default_host, |
| ) |
| option( |
| "--architectures", |
| store, |
| default=default_architectures, |
| help="space-separated list of architectures to build for. (default = %s)" |
| % default_architectures, |
| ) |
| option( |
| "--install-symroot", store_path, help="the path to install debug symbols into" |
| ) |
| option( |
| "--install-destdir", |
| store_path, |
| help="the path to use as the filesystem root for the installation (default = " |
| "'$(build_dir)/install')", |
| ) |
| option( |
| "--install-prefix", |
| store, |
| default=default_install_prefix, |
| help="the install prefix to use (default = %s)" % default_install_prefix, |
| ) |
| option("--version", store, help="version string to use for the parser library") |
| option( |
| "--ninja-path", |
| store_path, |
| default=default_ninja, |
| help="the path to ninja (default = %s)" % default_ninja, |
| ) |
| |
| parser = optbuilder.build() |
| args = parser.parse_args() |
| |
| if not args.install_destdir: |
| args.install_destdir = os.path.join(args.build_dir, "install") |
| |
| swift_src_path = os.path.join(SWIFT_SOURCE_ROOT, "swift") |
| swift_src_in_llvm_project_path = \ |
| os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "swift") |
| # Need to symlink 'swift' into 'llvm-project' since we will be doing |
| # a unified configure with 'swift' as an external project. |
| if not os.path.exists(swift_src_in_llvm_project_path): |
| print("Symlinking '%s' to '%s'" % |
| (swift_src_path, swift_src_in_llvm_project_path), file=sys.stderr) |
| shell.symlink(swift_src_path, swift_src_in_llvm_project_path, |
| dry_run=args.dry_run, echo=args.verbose) |
| |
| architectures = args.architectures.split(" ") |
| architectures = [arch for arch in architectures if arch != ""] |
| if platform.machine() in architectures: |
| # Make sure the machine architecture is at the front. |
| architectures.remove(platform.machine()) |
| architectures = [platform.machine()] + architectures |
| |
| if isDarwin: |
| objroot = args.build_dir |
| dstroot = args.install_destdir |
| symroot = args.install_symroot |
| prefix = args.install_prefix |
| |
| native_build_dir = None |
| profile_data = None |
| dst_dirs = [] |
| |
| if args.host == "macosx" and architectures[0] == platform.machine(): |
| # Build for the native machine. |
| arch = architectures.pop(0) |
| tmpargs = copy.copy(args) |
| tmpargs.build_dir = os.path.join(objroot, arch, "obj") |
| tmpargs.install_destdir = os.path.join(objroot, arch, "dst") |
| tmpargs.install_prefix = "/" |
| |
| native_build_dir = tmpargs.build_dir |
| dst_dirs.append(tmpargs.install_destdir) |
| |
| if tmpargs.pgo_type: |
| profile_dir = os.path.join(objroot, platform.machine() + "-profiling") |
| builder = Builder(toolchain, tmpargs, tmpargs.host, arch) |
| builder.get_profile_data(profile_dir) |
| profile_data = os.path.join(profile_dir, "profdata.prof") |
| |
| builder = Builder( |
| toolchain, tmpargs, tmpargs.host, arch, profile_data=profile_data |
| ) |
| builder.run() |
| |
| else: |
| tmpargs = copy.copy(args) |
| if tmpargs.pgo_type: |
| # Build for the machine and get profile data. |
| native_build_dir = os.path.join( |
| objroot, platform.machine() + "-profiling" |
| ) |
| builder = Builder(toolchain, tmpargs, "macosx", platform.machine()) |
| builder.get_profile_data(native_build_dir) |
| profile_data = os.path.join(native_build_dir, "profdata.prof") |
| else: |
| # Build the tablegen binaries so we can use them for the cross-compile |
| # build. |
| native_build_dir = os.path.join(objroot, platform.machine() + "-tblgen") |
| tmpargs.lto_type = None |
| builder = Builder(toolchain, tmpargs, "macosx", platform.machine()) |
| shell.makedirs(native_build_dir, dry_run=tmpargs.dry_run) |
| with shell.pushd(native_build_dir, dry_run=tmpargs.dry_run): |
| builder.configure(enable_debuginfo=False) |
| builder.build_target(native_build_dir, "llvm-tblgen") |
| builder.build_target(native_build_dir, "clang-tblgen") |
| |
| for arch in architectures: |
| args.build_dir = os.path.join(objroot, arch, "obj") |
| args.install_destdir = os.path.join(objroot, arch, "dst") |
| args.install_prefix = "/" |
| |
| dst_dirs.append(args.install_destdir) |
| |
| builder = Builder( |
| toolchain, |
| args, |
| args.host, |
| arch, |
| profile_data=profile_data, |
| native_build_dir=native_build_dir, |
| ) |
| builder.run() |
| |
| lipo = os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "recursive-lipo") |
| shell.call( |
| [lipo, "-v", "--destination", os.path.join(dstroot, "./" + prefix)] |
| + dst_dirs |
| ) |
| |
| if args.install_symroot: |
| extract_symbols(dstroot, prefix, symroot, args.build_jobs) |
| |
| return 0 |
| |
| assert ( |
| args.architectures == platform.machine() |
| ), "building for non-machine architecture is not supported for non-darwin host" |
| builder = Builder(toolchain, args, args.host, args.architectures) |
| builder.run() |
| return 0 |
| |
| |
| if __name__ == "__main__": |
| try: |
| sys.exit(main()) |
| except KeyboardInterrupt: |
| sys.exit(1) |
