| #!/usr/bin/env python |
| # utils/build-script - The ultimate tool for building Swift -*- python -*- |
| # |
| # This source file is part of the Swift.org open source project |
| # |
| # Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors |
| # Licensed under Apache License v2.0 with Runtime Library Exception |
| # |
| # See https://swift.org/LICENSE.txt for license information |
| # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| |
| from __future__ import print_function |
| |
| import argparse |
| import os |
| import pipes |
| import platform |
| import sys |
| import time |
| |
| from build_swift import defaults |
| from build_swift import driver_arguments |
| from build_swift import presets |
| from build_swift.migration import migrate_swift_sdks |
| |
| from swift_build_support.swift_build_support import ( |
| arguments, |
| debug, |
| diagnostics, |
| migration, |
| products, |
| shell, |
| tar, |
| targets, |
| workspace |
| ) |
| from swift_build_support.swift_build_support.SwiftBuildSupport import ( |
| HOME, |
| SWIFT_BUILD_ROOT, |
| SWIFT_REPO_NAME, |
| SWIFT_SOURCE_ROOT, |
| ) |
| from swift_build_support.swift_build_support.cmake import CMake |
| from swift_build_support.swift_build_support.targets import \ |
| StdlibDeploymentTarget |
| from swift_build_support.swift_build_support.toolchain import host_toolchain |
| |
| build_script_impl = os.path.join( |
| SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", "build-script-impl") |
| |
| |
| def exit_rejecting_arguments(message, parser=None): |
| print(message, file=sys.stderr) |
| if parser: |
| parser.print_usage(sys.stderr) |
| sys.exit(2) # 2 is the same as `argparse` error exit code. |
| |
| |
| def call_without_sleeping(command, env=None, dry_run=False, echo=False): |
| """ |
| Execute a command during which system sleep is disabled. |
| |
| By default, this ignores the state of the `shell.dry_run` flag. |
| """ |
| |
| # Disable system sleep, if possible. |
| if platform.system() == 'Darwin': |
| # Don't mutate the caller's copy of the arguments. |
| command = ["caffeinate"] + list(command) |
| |
| shell.call(command, env=env, dry_run=dry_run, echo=echo) |
| |
| |
| class HostSpecificConfiguration(object): |
| |
| """Configuration information for an individual host.""" |
| |
| def __init__(self, host_target, invocation): |
| """Initialize for the given `host_target`.""" |
| |
| # Compute the set of deployment targets to configure/build. |
| args = invocation.args |
| if host_target == args.host_target: |
| # This host is the user's desired product, so honor the requested |
| # set of targets to configure/build. |
| stdlib_targets_to_configure = args.stdlib_deployment_targets |
| if "all" in args.build_stdlib_deployment_targets: |
| stdlib_targets_to_build = set(stdlib_targets_to_configure) |
| else: |
| stdlib_targets_to_build = set( |
| args.build_stdlib_deployment_targets).intersection( |
| set(args.stdlib_deployment_targets)) |
| else: |
| # Otherwise, this is a host we are building as part of |
| # cross-compiling, so we only need the target itself. |
| stdlib_targets_to_configure = [host_target] |
| stdlib_targets_to_build = set(stdlib_targets_to_configure) |
| |
| # Compute the lists of **CMake** targets for each use case (configure |
| # vs. build vs. run) and the SDKs to configure with. |
| self.sdks_to_configure = set() |
| self.swift_stdlib_build_targets = [] |
| self.swift_test_run_targets = [] |
| self.swift_benchmark_build_targets = [] |
| self.swift_benchmark_run_targets = [] |
| for deployment_target_name in stdlib_targets_to_configure: |
| # Get the target object. |
| deployment_target = StdlibDeploymentTarget.get_target_for_name( |
| deployment_target_name) |
| if deployment_target is None: |
| diagnostics.fatal("unknown target: %r" % ( |
| deployment_target_name,)) |
| |
| # Add the SDK to use. |
| deployment_platform = deployment_target.platform |
| self.sdks_to_configure.add(deployment_platform.sdk_name) |
| |
| # If we aren't actually building this target (only configuring |
| # it), do nothing else. |
| if deployment_target_name not in stdlib_targets_to_build: |
| continue |
| |
| # Compute which actions are desired. |
| build = ( |
| deployment_platform not in invocation.platforms_to_skip_build) |
| test = ( |
| deployment_platform not in invocation.platforms_to_skip_test) |
| test_host_only = None |
| dt_supports_benchmark = deployment_target.supports_benchmark |
| build_benchmarks = build and dt_supports_benchmark |
| build_external_benchmarks = all([build, dt_supports_benchmark, |
| args.build_external_benchmarks]) |
| |
| # FIXME: Note, `build-script-impl` computed a property here |
| # w.r.t. testing, but it was actually unused. |
| |
| # For platforms which normally require a connected device to |
| # test, the default behavior is to run tests that only require |
| # the host (i.e., they do not attempt to execute). |
| if deployment_platform.uses_host_tests and \ |
| deployment_platform not in \ |
| invocation.platforms_to_skip_test_host: |
| test_host_only = True |
| |
| name = deployment_target.name |
| |
| for skip_test_arch in invocation.platforms_archs_to_skip_test: |
| if deployment_target.name == skip_test_arch.name: |
| test = False |
| |
| if build: |
| # Validation, long, and stress tests require building the full |
| # standard library, whereas the other targets can build a |
| # slightly smaller subset which is faster to build. |
| if args.build_swift_stdlib_unittest_extra or \ |
| args.validation_test or args.long_test or \ |
| args.stress_test: |
| self.swift_stdlib_build_targets.append( |
| "swift-stdlib-" + name) |
| else: |
| self.swift_stdlib_build_targets.append( |
| "swift-test-stdlib-" + name) |
| if build_benchmarks: |
| self.swift_benchmark_build_targets.append( |
| "swift-benchmark-" + name) |
| if args.benchmark: |
| self.swift_benchmark_run_targets.append( |
| "check-swift-benchmark-" + name) |
| |
| if build_external_benchmarks: |
| # Add support for the external benchmarks. |
| self.swift_benchmark_build_targets.append( |
| "swift-benchmark-{}-external".format(name)) |
| if args.benchmark: |
| self.swift_benchmark_run_targets.append( |
| "check-swift-benchmark-{}-external".format(name)) |
| if test: |
| if test_host_only: |
| suffix = "-only_non_executable" |
| else: |
| suffix = "" |
| subset_suffix = "" |
| if args.validation_test and args.long_test and \ |
| args.stress_test: |
| subset_suffix = "-all" |
| elif args.validation_test: |
| subset_suffix = "-validation" |
| elif args.long_test: |
| subset_suffix = "-only_long" |
| elif args.stress_test: |
| subset_suffix = "-only_stress" |
| else: |
| subset_suffix = "" |
| self.swift_test_run_targets.append("check-swift{}{}-{}".format( |
| subset_suffix, suffix, name)) |
| if args.test_optimized and not test_host_only: |
| self.swift_test_run_targets.append( |
| "check-swift{}-optimize-{}".format( |
| subset_suffix, name)) |
| if args.test_optimize_for_size and not test_host_only: |
| self.swift_test_run_targets.append( |
| "check-swift{}-optimize_size-{}".format( |
| subset_suffix, name)) |
| if args.test_optimize_none_implicit_dynamic and \ |
| not test_host_only: |
| self.swift_test_run_targets.append( |
| "check-swift{}-optimize_none_implicit_dynamic-{}" |
| .format(subset_suffix, name)) |
| |
| |
| class BuildScriptInvocation(object): |
| |
| """Represent a single build script invocation.""" |
| |
| @staticmethod |
| def validate_arguments(toolchain, args): |
| if toolchain.cc is None or toolchain.cxx is None: |
| diagnostics.fatal( |
| "can't find clang (please install clang-3.5 or a " |
| "later version)") |
| |
| if toolchain.cmake is None: |
| diagnostics.fatal("can't find CMake (please install CMake)") |
| |
| if args.distcc: |
| if toolchain.distcc is None: |
| diagnostics.fatal( |
| "can't find distcc (please install distcc)") |
| if toolchain.distcc_pump is None: |
| diagnostics.fatal( |
| "can't find distcc-pump (please install distcc-pump)") |
| |
| if args.host_target is None or args.stdlib_deployment_targets is None: |
| diagnostics.fatal("unknown operating system") |
| |
| if args.symbols_package: |
| if not os.path.isabs(args.symbols_package): |
| print( |
| '--symbols-package must be an absolute path ' |
| '(was \'{}\')'.format(args.symbols_package)) |
| return 1 |
| if not args.install_symroot: |
| diagnostics.fatal( |
| "--install-symroot is required when specifying " |
| "--symbols-package.") |
| |
| if args.android: |
| if args.android_ndk is None or \ |
| args.android_api_level is None or \ |
| args.android_icu_uc is None or \ |
| args.android_icu_uc_include is None or \ |
| args.android_icu_i18n is None or \ |
| args.android_icu_i18n_include is None or \ |
| args.android_icu_data is None: |
| diagnostics.fatal( |
| "when building for Android, --android-ndk, " |
| "--android-api-level, --android-icu-uc, " |
| "--android-icu-uc-include, --android-icu-i18n, " |
| "--android-icu-i18n-include, and --android-icu-data " |
| "must be specified") |
| |
| if args.legacy_impl and (args.build_indexstoredb or |
| args.build_sourcekitlsp): |
| diagnostics.fatal( |
| "--legacy-impl is incompatible with building packages needing " |
| "a toolchain (indexstore-db or sourcekit-lsp)") |
| |
| @staticmethod |
| def apply_default_arguments(toolchain, args): |
| # Infer if ninja is required |
| ninja_required = ( |
| args.cmake_generator == 'Ninja' or args.build_foundation) |
| if ninja_required and toolchain.ninja is None: |
| args.build_ninja = True |
| |
| # Set the default stdlib-deployment-targets, if none were provided. |
| if args.stdlib_deployment_targets is None: |
| stdlib_targets = \ |
| StdlibDeploymentTarget.default_stdlib_deployment_targets() |
| args.stdlib_deployment_targets = [ |
| target.name for target in stdlib_targets] |
| |
| # SwiftPM and XCTest have a dependency on Foundation. |
| # On OS X, Foundation is built automatically using xcodebuild. |
| # On Linux, we must ensure that it is built manually. |
| if ((args.build_swiftpm or args.build_xctest) and |
| platform.system() != "Darwin"): |
| args.build_foundation = True |
| |
| # Foundation has a dependency on libdispatch. |
| # On OS X, libdispatch is provided by the OS. |
| # On Linux, we must ensure that it is built manually. |
| if (args.build_foundation and |
| platform.system() != "Darwin"): |
| args.build_libdispatch = True |
| |
| if args.build_subdir is None: |
| args.build_subdir = \ |
| workspace.compute_build_subdir(args) |
| |
| if args.install_destdir is None: |
| args.install_destdir = os.path.join( |
| SWIFT_BUILD_ROOT, args.build_subdir, |
| '{}-{}'.format('toolchain', args.host_target)) |
| |
| # Add optional stdlib-deployment-targets |
| if args.android: |
| if args.android_arch == "armv7": |
| args.stdlib_deployment_targets.append( |
| StdlibDeploymentTarget.Android.armv7.name) |
| elif args.android_arch == "aarch64": |
| args.stdlib_deployment_targets.append( |
| StdlibDeploymentTarget.Android.aarch64.name) |
| |
| # Infer platform flags from manually-specified configure targets. |
| # This doesn't apply to Darwin platforms, as they are |
| # already configured. No building without the platform flag, though. |
| |
| android_tgts = [tgt for tgt in args.stdlib_deployment_targets |
| if StdlibDeploymentTarget.Android.contains(tgt)] |
| if not args.android and len(android_tgts) > 0: |
| args.android = True |
| args.build_android = False |
| |
| # --- |
| |
| def __init__(self, toolchain, args): |
| self.toolchain = toolchain |
| self.args = args |
| |
| self.workspace = workspace.Workspace( |
| source_root=SWIFT_SOURCE_ROOT, |
| build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir)) |
| |
| # Compute derived information from the arguments. |
| # |
| # FIXME: We should move the platform-derived arguments to be entirely |
| # data driven, so that we can eliminate this code duplication and just |
| # iterate over all supported platforms. |
| |
| self.platforms_to_skip_build = set() |
| if not args.build_linux: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.Linux) |
| if not args.build_freebsd: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD) |
| if not args.build_cygwin: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.Cygwin) |
| if not args.build_osx: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.OSX) |
| if not args.build_ios_device: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.iOS) |
| if not args.build_ios_simulator: |
| self.platforms_to_skip_build.add( |
| StdlibDeploymentTarget.iOSSimulator) |
| if not args.build_tvos_device: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleTV) |
| if not args.build_tvos_simulator: |
| self.platforms_to_skip_build.add( |
| StdlibDeploymentTarget.AppleTVSimulator) |
| if not args.build_watchos_device: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleWatch) |
| if not args.build_watchos_simulator: |
| self.platforms_to_skip_build.add( |
| StdlibDeploymentTarget.AppleWatchSimulator) |
| if not args.build_android: |
| self.platforms_to_skip_build.add(StdlibDeploymentTarget.Android) |
| |
| self.platforms_to_skip_test = set() |
| self.platforms_archs_to_skip_test = set() |
| if not args.test_linux: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.Linux) |
| if not args.test_freebsd: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD) |
| if not args.test_cygwin: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.Cygwin) |
| if not args.test_osx: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.OSX) |
| if not args.test_ios_host: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOS) |
| else: |
| exit_rejecting_arguments("error: iOS device tests are not " + |
| "supported in open-source Swift.") |
| if not args.test_ios_simulator: |
| self.platforms_to_skip_test.add( |
| StdlibDeploymentTarget.iOSSimulator) |
| if not args.test_ios_32bit_simulator: |
| self.platforms_archs_to_skip_test.add( |
| StdlibDeploymentTarget.iOSSimulator.i386) |
| if not args.test_tvos_host: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV) |
| else: |
| exit_rejecting_arguments("error: tvOS device tests are not " + |
| "supported in open-source Swift.") |
| if not args.test_tvos_simulator: |
| self.platforms_to_skip_test.add( |
| StdlibDeploymentTarget.AppleTVSimulator) |
| if not args.test_watchos_host: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleWatch) |
| else: |
| exit_rejecting_arguments("error: watchOS device tests are not " + |
| "supported in open-source Swift.") |
| if not args.test_watchos_simulator: |
| self.platforms_to_skip_test.add( |
| StdlibDeploymentTarget.AppleWatchSimulator) |
| if not args.test_android: |
| self.platforms_to_skip_test.add(StdlibDeploymentTarget.Android) |
| |
| self.platforms_to_skip_test_host = set() |
| if not args.test_android_host: |
| self.platforms_to_skip_test_host.add( |
| StdlibDeploymentTarget.Android) |
| if not args.test_ios_host: |
| self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS) |
| if not args.test_tvos_host: |
| self.platforms_to_skip_test_host.add( |
| StdlibDeploymentTarget.AppleTV) |
| if not args.test_watchos_host: |
| self.platforms_to_skip_test_host.add( |
| StdlibDeploymentTarget.AppleWatch) |
| self.build_libparser_only = args.build_libparser_only |
| |
| def initialize_runtime_environment(self): |
| """Change the program environment for building.""" |
| |
| # Set an appropriate default umask. |
| os.umask(0o022) |
| |
| # Unset environment variables that might affect how tools behave. |
| for v in [ |
| 'MAKEFLAGS', |
| 'SDKROOT', |
| 'MACOSX_DEPLOYMENT_TARGET', |
| 'IPHONEOS_DEPLOYMENT_TARGET', |
| 'TVOS_DEPLOYMENT_TARGET', |
| 'WATCHOS_DEPLOYMENT_TARGET']: |
| os.environ.pop(v, None) |
| |
| # Set NINJA_STATUS to format ninja output |
| os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] ' |
| |
| def build_ninja(self): |
| if not os.path.exists(self.workspace.source_dir("ninja")): |
| diagnostics.fatal( |
| "can't find source directory for ninja " |
| "(tried %s)" % (self.workspace.source_dir("ninja"))) |
| |
| ninja_build = products.Ninja( |
| args=self.args, |
| toolchain=self.toolchain, |
| source_dir=self.workspace.source_dir("ninja"), |
| build_dir=self.workspace.build_dir("build", "ninja")) |
| ninja_build.do_build() |
| self.toolchain.ninja = ninja_build.ninja_bin_path |
| |
| def convert_to_impl_arguments(self): |
| """convert_to_impl_arguments() -> (env, args) |
| |
| Convert the invocation to an environment and list of arguments suitable |
| for invoking `build-script-impl`. |
| """ |
| |
| # Create local shadows, for convenience. |
| args = self.args |
| toolchain = self.toolchain |
| |
| cmake = CMake(args=args, |
| toolchain=self.toolchain) |
| |
| impl_args = [ |
| "--workspace", self.workspace.source_root, |
| "--build-dir", self.workspace.build_root, |
| "--install-prefix", args.install_prefix, |
| "--host-target", args.host_target, |
| "--stdlib-deployment-targets", |
| " ".join(args.stdlib_deployment_targets), |
| "--host-cc", toolchain.cc, |
| "--host-cxx", toolchain.cxx, |
| "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain, |
| "--darwin-deployment-version-osx=%s" % ( |
| args.darwin_deployment_version_osx), |
| "--darwin-deployment-version-ios=%s" % ( |
| args.darwin_deployment_version_ios), |
| "--darwin-deployment-version-tvos=%s" % ( |
| args.darwin_deployment_version_tvos), |
| "--darwin-deployment-version-watchos=%s" % ( |
| args.darwin_deployment_version_watchos), |
| "--cmake", toolchain.cmake, |
| "--cmark-build-type", args.cmark_build_variant, |
| "--llvm-build-type", args.llvm_build_variant, |
| "--swift-build-type", args.swift_build_variant, |
| "--swift-stdlib-build-type", args.swift_stdlib_build_variant, |
| "--lldb-build-type", args.lldb_build_variant, |
| "--lldb-build-with-xcode", args.lldb_build_with_xcode, |
| "--foundation-build-type", args.foundation_build_variant, |
| "--libdispatch-build-type", args.libdispatch_build_variant, |
| "--libicu-build-type", args.libicu_build_variant, |
| "--xctest-build-type", args.build_variant, |
| "--swiftpm-build-type", args.build_variant, |
| "--swiftsyntax-build-type", args.build_variant, |
| "--skstresstester-build-type", args.build_variant, |
| "--swiftevolve-build-type", args.build_variant, |
| "--llbuild-build-type", args.build_variant, |
| "--swift-enable-assertions", str(args.swift_assertions).lower(), |
| "--swift-stdlib-enable-assertions", str( |
| args.swift_stdlib_assertions).lower(), |
| "--swift-analyze-code-coverage", str( |
| args.swift_analyze_code_coverage).lower(), |
| "--llbuild-enable-assertions", str( |
| args.llbuild_assertions).lower(), |
| "--lldb-assertions", str( |
| args.lldb_assertions).lower(), |
| "--cmake-generator", args.cmake_generator, |
| "--build-jobs", str(args.build_jobs), |
| "--common-cmake-options=%s" % ' '.join( |
| pipes.quote(opt) for opt in cmake.common_options()), |
| "--build-args=%s" % ' '.join( |
| pipes.quote(arg) for arg in cmake.build_args()), |
| ] |
| |
| # Compute any product specific cmake arguments. |
| for product_class in self.compute_product_classes(): |
| if not product_class.is_build_script_impl_product(): |
| continue |
| |
| product_name = product_class.product_name() |
| product_source_name = product_class.product_source_name() |
| source_dir = self.workspace.source_dir(product_source_name) |
| |
| if not os.path.exists(source_dir): |
| diagnostics.fatal( |
| "can't find source directory for %s " |
| "(tried %s)" % (product_name, source_dir)) |
| |
| product = product_class( |
| args=args, |
| toolchain=self.toolchain, |
| source_dir=source_dir, |
| # FIXME: This is incorrect since it always assumes the host |
| # target I think? |
| build_dir=self.workspace.build_dir( |
| args.host_target, product_name)) |
| cmake_opts = product.cmake_options |
| |
| # FIXME: We should be using pipes.quote here but we run into issues |
| # with build-script-impl/cmake not being happy with all of the |
| # extra "'" in the strings. To fix this easily, we really need to |
| # just invoke cmake from build-script directly rather than futzing |
| # with build-script-impl. This makes even more sense since there |
| # really isn't a security issue here. |
| if cmake_opts: |
| impl_args += [ |
| "--{}-cmake-options={}".format( |
| product_name, ' '.join(cmake_opts)) |
| ] |
| |
| if args.build_stdlib_deployment_targets: |
| impl_args += [ |
| "--build-stdlib-deployment-targets", " ".join( |
| args.build_stdlib_deployment_targets)] |
| if args.cross_compile_hosts: |
| impl_args += [ |
| "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] |
| |
| if args.test_paths: |
| impl_args += ["--test-paths", " ".join(args.test_paths)] |
| |
| if toolchain.ninja: |
| impl_args += ["--ninja-bin=%s" % toolchain.ninja] |
| if args.distcc: |
| impl_args += [ |
| "--distcc", |
| "--distcc-pump=%s" % toolchain.distcc_pump |
| ] |
| |
| # *NOTE* We use normal cmake to pass through tsan/ubsan options. We do |
| # NOT pass them to build-script-impl. |
| if args.enable_asan: |
| impl_args += ["--enable-asan"] |
| # If we are on linux, disable leak detection when running ASAN. We |
| # have a separate bot that checks for leaks. |
| if platform.system() == 'Linux': |
| os.environ['ASAN_OPTIONS'] = 'detect_leaks=0' |
| if args.enable_ubsan: |
| impl_args += ["--enable-ubsan"] |
| |
| # If we have lsan, we need to export our suppression list. The actual |
| # passing in of the LSAN flag is done via the normal cmake method. We |
| # do not pass the flag to build-script-impl. |
| if args.enable_lsan: |
| supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, |
| "utils", |
| "lsan_leaks_suppression_list.txt") |
| os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file) |
| if args.verbose_build: |
| impl_args += ["--verbose-build"] |
| if args.install_symroot: |
| impl_args += [ |
| "--install-symroot", os.path.abspath(args.install_symroot) |
| ] |
| if args.install_destdir: |
| impl_args += [ |
| "--install-destdir", os.path.abspath(args.install_destdir) |
| ] |
| |
| if args.skip_build: |
| impl_args += ["--skip-build-cmark", |
| "--skip-build-llvm", |
| "--skip-build-swift"] |
| if not args.build_benchmarks: |
| impl_args += ["--skip-build-benchmarks"] |
| # Currently we do not build external benchmarks by default. |
| if args.build_external_benchmarks: |
| impl_args += ["--skip-build-external-benchmarks=0"] |
| if not args.build_foundation: |
| impl_args += ["--skip-build-foundation"] |
| if not args.build_xctest: |
| impl_args += ["--skip-build-xctest"] |
| if not args.build_lldb: |
| impl_args += ["--skip-build-lldb"] |
| if not args.build_llbuild: |
| impl_args += ["--skip-build-llbuild"] |
| if not args.build_libcxx: |
| impl_args += ["--skip-build-libcxx"] |
| if not args.build_libdispatch: |
| impl_args += ["--skip-build-libdispatch"] |
| if not args.build_libicu: |
| impl_args += ["--skip-build-libicu"] |
| if not args.build_swiftpm: |
| impl_args += ["--skip-build-swiftpm"] |
| if not args.build_swiftsyntax: |
| impl_args += ["--skip-build-swiftsyntax"] |
| if not args.build_skstresstester: |
| impl_args += ["--skip-build-skstresstester"] |
| if not args.build_swiftevolve: |
| impl_args += ["--skip-build-swiftevolve"] |
| if not args.build_playgroundsupport: |
| impl_args += ["--skip-build-playgroundsupport"] |
| if args.build_swift_dynamic_stdlib: |
| impl_args += ["--build-swift-dynamic-stdlib"] |
| if args.build_swift_static_stdlib: |
| impl_args += ["--build-swift-static-stdlib"] |
| if args.build_swift_stdlib_unittest_extra: |
| impl_args += ["--build-swift-stdlib-unittest-extra"] |
| if args.build_swift_dynamic_sdk_overlay: |
| impl_args += ["--build-swift-dynamic-sdk-overlay"] |
| if args.build_swift_static_sdk_overlay: |
| impl_args += ["--build-swift-static-sdk-overlay"] |
| |
| if not args.build_linux: |
| impl_args += ["--skip-build-linux"] |
| if not args.build_freebsd: |
| impl_args += ["--skip-build-freebsd"] |
| if not args.build_cygwin: |
| impl_args += ["--skip-build-cygwin"] |
| if not args.build_osx: |
| impl_args += ["--skip-build-osx"] |
| if not args.build_ios_device: |
| impl_args += ["--skip-build-ios-device"] |
| if not args.build_ios_simulator: |
| impl_args += ["--skip-build-ios-simulator"] |
| if not args.build_tvos_device: |
| impl_args += ["--skip-build-tvos-device"] |
| if not args.build_tvos_simulator: |
| impl_args += ["--skip-build-tvos-simulator"] |
| if not args.build_watchos_device: |
| impl_args += ["--skip-build-watchos-device"] |
| if not args.build_watchos_simulator: |
| impl_args += ["--skip-build-watchos-simulator"] |
| if not args.build_android: |
| impl_args += ["--skip-build-android"] |
| |
| if not args.test and not args.long_test and not args.stress_test: |
| impl_args += ["--skip-test-swift"] |
| if not args.test: |
| impl_args += ["--skip-test-cmark", |
| "--skip-test-lldb", |
| "--skip-test-llbuild", |
| "--skip-test-swiftpm", |
| "--skip-test-swiftsyntax", |
| "--skip-test-skstresstester", |
| "--skip-test-swiftevolve", |
| "--skip-test-xctest", |
| "--skip-test-foundation", |
| "--skip-test-libdispatch", |
| "--skip-test-libicu", |
| "--skip-test-playgroundsupport"] |
| if not args.test_linux: |
| impl_args += ["--skip-test-linux"] |
| if not args.test_freebsd: |
| impl_args += ["--skip-test-freebsd"] |
| if not args.test_cygwin: |
| impl_args += ["--skip-test-cygwin"] |
| if not args.test_osx: |
| impl_args += ["--skip-test-osx"] |
| if not args.test_ios_host: |
| impl_args += ["--skip-test-ios-host"] |
| if not args.test_ios_simulator: |
| impl_args += ["--skip-test-ios-simulator"] |
| if not args.test_ios_32bit_simulator: |
| impl_args += ["--skip-test-ios-32bit-simulator"] |
| if not args.test_tvos_host: |
| impl_args += ["--skip-test-tvos-host"] |
| if not args.test_tvos_simulator: |
| impl_args += ["--skip-test-tvos-simulator"] |
| if not args.test_watchos_host: |
| impl_args += ["--skip-test-watchos-host"] |
| if not args.test_watchos_simulator: |
| impl_args += ["--skip-test-watchos-simulator"] |
| if not args.test_android: |
| impl_args += ["--skip-test-android"] |
| if not args.test_android_host: |
| impl_args += ["--skip-test-android-host"] |
| if args.build_runtime_with_host_compiler: |
| impl_args += ["--build-runtime-with-host-compiler"] |
| if args.validation_test: |
| impl_args += ["--validation-test"] |
| if args.long_test: |
| impl_args += ["--long-test"] |
| if args.stress_test: |
| impl_args += ["--stress-test"] |
| if not args.benchmark: |
| impl_args += ["--skip-test-benchmarks"] |
| if not args.test_optimized: |
| impl_args += ["--skip-test-optimized"] |
| if not args.test_optimize_for_size: |
| impl_args += ["--skip-test-optimize-for-size"] |
| if not args.test_optimize_none_implicit_dynamic: |
| impl_args += ["--skip-test-optimize-none-implicit-dynamic"] |
| if args.build_libparser_only: |
| impl_args += ["--build-libparser-only"] |
| if args.android: |
| impl_args += [ |
| "--android-arch", args.android_arch, |
| "--android-ndk", args.android_ndk, |
| "--android-api-level", args.android_api_level, |
| "--android-ndk-gcc-version", args.android_ndk_gcc_version, |
| "--android-icu-uc", args.android_icu_uc, |
| "--android-icu-uc-include", args.android_icu_uc_include, |
| "--android-icu-i18n", args.android_icu_i18n, |
| "--android-icu-i18n-include", args.android_icu_i18n_include, |
| "--android-icu-data", args.android_icu_data, |
| ] |
| if args.android_deploy_device_path: |
| impl_args += [ |
| "--android-deploy-device-path", |
| args.android_deploy_device_path, |
| ] |
| |
| if platform.system() == 'Darwin': |
| impl_args += [ |
| "--toolchain-prefix", |
| targets.darwin_toolchain_prefix( |
| args.install_prefix), |
| "--host-lipo", toolchain.lipo, |
| ] |
| |
| if toolchain.libtool is not None: |
| impl_args += [ |
| "--host-libtool", toolchain.libtool, |
| ] |
| |
| # If we have extra_swift_args, combine all of them together and then |
| # add them as one command. |
| if args.extra_swift_args: |
| impl_args += [ |
| "--extra-swift-args=%s" % ';'.join(args.extra_swift_args) |
| ] |
| |
| # If we have extra_cmake_options, combine all of them together and then |
| # add them as one command. |
| if args.extra_cmake_options: |
| impl_args += [ |
| "--extra-cmake-options=%s" % ' '.join( |
| pipes.quote(opt) for opt in args.extra_cmake_options) |
| ] |
| |
| if args.lto_type is not None: |
| impl_args += [ |
| "--llvm-enable-lto=%s" % args.lto_type, |
| "--swift-tools-enable-lto=%s" % args.lto_type |
| ] |
| if args.llvm_max_parallel_lto_link_jobs is not None: |
| impl_args += [ |
| "--llvm-num-parallel-lto-link-jobs=%s" % |
| min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs) |
| ] |
| if args.swift_tools_max_parallel_lto_link_jobs is not None: |
| impl_args += [ |
| "--swift-tools-num-parallel-lto-link-jobs=%s" % |
| min(args.swift_tools_max_parallel_lto_link_jobs, |
| args.build_jobs) |
| ] |
| |
| impl_args += args.build_script_impl_args |
| |
| if args.dry_run: |
| impl_args += ["--dry-run"] |
| |
| if args.clang_profile_instr_use: |
| impl_args += [ |
| "--clang-profile-instr-use=%s" % |
| os.path.abspath(args.clang_profile_instr_use) |
| ] |
| |
| if args.lit_args: |
| impl_args += ["--llvm-lit-args=%s" % args.lit_args] |
| |
| if args.coverage_db: |
| impl_args += [ |
| "--coverage-db=%s" % |
| os.path.abspath(args.coverage_db) |
| ] |
| |
| # Compute the set of host-specific variables, which we pass through to |
| # the build script via environment variables. |
| host_specific_variables = self.compute_host_specific_variables() |
| impl_env = {} |
| for (host_target, options) in host_specific_variables.items(): |
| for (name, value) in options.items(): |
| # We mangle into an environment variable we can easily evaluate |
| # from the `build-script-impl`. |
| impl_env["HOST_VARIABLE_{}__{}".format( |
| host_target.replace("-", "_"), name)] = value |
| |
| return (impl_env, impl_args) |
| |
| def compute_host_specific_variables(self): |
| """compute_host_specific_variables(args) -> dict |
| |
| Compute the host-specific options, organized as a dictionary keyed by |
| host of options. |
| """ |
| |
| args = self.args |
| |
| options = {} |
| for host_target in [args.host_target] + args.cross_compile_hosts: |
| # Compute the host specific configuration. |
| config = HostSpecificConfiguration(host_target, self) |
| |
| # Convert into `build-script-impl` style variables. |
| options[host_target] = { |
| "SWIFT_SDKS": " ".join(sorted( |
| config.sdks_to_configure)), |
| "SWIFT_STDLIB_TARGETS": " ".join( |
| config.swift_stdlib_build_targets), |
| "SWIFT_BENCHMARK_TARGETS": " ".join( |
| config.swift_benchmark_build_targets), |
| "SWIFT_RUN_BENCHMARK_TARGETS": " ".join( |
| config.swift_benchmark_run_targets), |
| "SWIFT_TEST_TARGETS": " ".join( |
| config.swift_test_run_targets), |
| } |
| |
| return options |
| |
| def compute_product_classes(self): |
| """compute_product_classes() -> list |
| |
| Compute the list of all Product classes used in this build. This list |
| is constructed in dependency order. |
| """ |
| |
| # FIXME: This is a weird division (returning a list of class objects), |
| # but it matches the existing structure of the `build-script-impl`. |
| |
| product_classes = [] |
| product_classes.append(products.CMark) |
| product_classes.append(products.LLVM) |
| if self.args.build_libcxx: |
| product_classes.append(products.LibCXX) |
| if self.args.build_libicu: |
| product_classes.append(products.LibICU) |
| product_classes.append(products.Swift) |
| if self.args.build_lldb: |
| product_classes.append(products.LLDB) |
| if self.args.build_libdispatch: |
| product_classes.append(products.LibDispatch) |
| if self.args.build_foundation: |
| product_classes.append(products.Foundation) |
| if self.args.build_xctest: |
| product_classes.append(products.XCTest) |
| if self.args.build_llbuild: |
| product_classes.append(products.LLBuild) |
| if self.args.build_swiftpm: |
| product_classes.append(products.SwiftPM) |
| if self.args.build_swiftsyntax: |
| product_classes.append(products.SwiftSyntax) |
| if self.args.build_skstresstester: |
| product_classes.append(products.SKStressTester) |
| if self.args.build_swiftevolve: |
| product_classes.append(products.SwiftEvolve) |
| if self.args.build_indexstoredb: |
| product_classes.append(products.IndexStoreDB) |
| if self.args.build_sourcekitlsp: |
| product_classes.append(products.SourceKitLSP) |
| return product_classes |
| |
| def execute(self): |
| """Execute the invocation with the configured arguments.""" |
| |
| # Convert to a build-script-impl invocation. |
| (impl_env, impl_args) = self.convert_to_impl_arguments() |
| |
| # If using the legacy implementation, delegate all behavior to |
| # `build-script-impl`. |
| if self.args.legacy_impl: |
| # Execute the underlying build script implementation. |
| call_without_sleeping([build_script_impl] + impl_args, |
| env=impl_env, echo=True) |
| return |
| |
| # Otherwise, we compute and execute the individual actions ourselves. |
| |
| def execute_one_impl_action(host=None, product_class=None, name=None): |
| if host is None: |
| assert (product_class is None and |
| name == "merged-hosts-lipo"), "invalid action" |
| action_name = name |
| elif product_class is None: |
| assert name in ("package", "extractsymbols"), "invalid action" |
| action_name = "{}-{}".format(host.name, name) |
| else: |
| assert name is not None, "invalid action" |
| action_name = "{}-{}-{}".format( |
| host.name, product_class.product_name(), name) |
| call_without_sleeping( |
| [build_script_impl] + impl_args + [ |
| "--only-execute", action_name], |
| env=impl_env, echo=self.args.verbose_build) |
| |
| # Compute the list of hosts to operate on. |
| all_host_names = [ |
| self.args.host_target] + self.args.cross_compile_hosts |
| all_hosts = [StdlibDeploymentTarget.get_target_for_name(name) |
| for name in all_host_names] |
| |
| # Compute the list of product classes to operate on. |
| # |
| # FIXME: This should really be per-host, but the current structure |
| # matches that of `build-script-impl`. |
| product_classes = self.compute_product_classes() |
| |
| impl_product_classes = [cls for cls in product_classes |
| if cls.is_build_script_impl_product()] |
| |
| # Execute each "pass". |
| |
| # Build... |
| for host_target in all_hosts: |
| # FIXME: We should only compute these once. |
| config = HostSpecificConfiguration(host_target.name, self) |
| print("Building the standard library for: {}".format( |
| " ".join(config.swift_stdlib_build_targets))) |
| if config.swift_test_run_targets and ( |
| self.args.test or self.args.long_test): |
| print("Running Swift tests for: {}".format( |
| " ".join(config.swift_test_run_targets))) |
| if config.swift_benchmark_run_targets and self.args.benchmark: |
| print("Running Swift benchmarks for: {}".format( |
| " ".join(config.swift_benchmark_run_targets))) |
| |
| for product_class in impl_product_classes: |
| execute_one_impl_action(host_target, product_class, "build") |
| |
| # Test... |
| for host_target in all_hosts: |
| for product_class in impl_product_classes: |
| execute_one_impl_action(host_target, product_class, "test") |
| |
| # Install... |
| for host_target in all_hosts: |
| for product_class in impl_product_classes: |
| execute_one_impl_action(host_target, product_class, "install") |
| |
| # Non-build-script-impl products... |
| # Note: currently only supports building for the host. |
| for host_target in [self.args.host_target]: |
| for product_class in product_classes: |
| if product_class.is_build_script_impl_product(): |
| continue |
| product_source = product_class.product_source_name() |
| product_name = product_class.product_name() |
| product = product_class( |
| args=self.args, |
| toolchain=self.toolchain, |
| source_dir=self.workspace.source_dir(product_source), |
| build_dir=self.workspace.build_dir( |
| host_target, product_name)) |
| product.do_build(host_target) |
| product.do_test(host_target) |
| |
| # Extract symbols... |
| for host_target in all_hosts: |
| execute_one_impl_action(host_target, name="extractsymbols") |
| |
| # Package... |
| for host_target in all_hosts: |
| execute_one_impl_action(host_target, name="package") |
| |
| # Lipo... |
| execute_one_impl_action(name="merged-hosts-lipo") |
| |
| |
| # Provide a short delay so accidentally invoked clean builds can be canceled. |
| def clean_delay(): |
| sys.stdout.write('Starting clean build in ') |
| for i in range(3, 0, -1): |
| sys.stdout.write('\b%d' % i) |
| sys.stdout.flush() |
| time.sleep(1) |
| print('\b\b\b\bnow.') |
| |
| |
| # Main entry point for the preset mode. |
| def main_preset(): |
| parser = argparse.ArgumentParser( |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| description="""Builds Swift using a preset.""") |
| parser.add_argument( |
| "-n", "--dry-run", |
| help="print the commands that would be executed, but do not execute " |
| "them", |
| action="store_true", |
| default=False) |
| parser.add_argument( |
| "--preset-file", |
| help="load presets from the specified file", |
| metavar="PATH", |
| action="append", |
| dest="preset_file_names", |
| default=[]) |
| parser.add_argument( |
| "--preset", |
| help="use the specified option preset", |
| metavar="NAME") |
| parser.add_argument( |
| "--show-presets", |
| help="list all presets and exit", |
| action=arguments.action.optional_bool) |
| parser.add_argument( |
| "--distcc", |
| help="use distcc", |
| action=arguments.action.optional_bool) |
| parser.add_argument( |
| "--cmake-c-launcher", |
| help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER", |
| metavar="PATH") |
| parser.add_argument( |
| "--cmake-cxx-launcher", |
| help="the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER", |
| metavar="PATH") |
| parser.add_argument( |
| "-j", "--jobs", |
| help="the number of parallel build jobs to use", |
| type=int, |
| dest="build_jobs") |
| parser.add_argument( |
| "preset_substitutions_raw", |
| help="'name=value' pairs that are substituted in the preset", |
| nargs="*", |
| metavar="SUBSTITUTION") |
| parser.add_argument( |
| "--expand-build-script-invocation", |
| help="Print the expanded build-script invocation generated " |
| "by the preset, but do not run the preset", |
| action=arguments.action.optional_bool) |
| parser.add_argument( |
| "--swiftsyntax-install-prefix", |
| help="specify the directory to where SwiftSyntax should be installed") |
| parser.add_argument( |
| "--build-dir", |
| help="specify the directory where build artifact should be stored") |
| args = parser.parse_args() |
| if len(args.preset_file_names) == 0: |
| args.preset_file_names = [ |
| os.path.join( |
| SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", |
| "build-presets.ini") |
| ] |
| |
| user_presets_file = os.path.join(HOME, '.swift-build-presets') |
| if os.path.isfile(user_presets_file): |
| args.preset_file_names.append(user_presets_file) |
| |
| preset_parser = presets.PresetParser() |
| |
| try: |
| preset_parser.read(args.preset_file_names) |
| except presets.Error as e: |
| diagnostics.fatal(e.message) |
| |
| if args.show_presets: |
| for name in sorted(preset_parser.preset_names(), |
| key=lambda name: name.lower()): |
| print(name) |
| return 0 |
| |
| if not args.preset: |
| diagnostics.fatal("missing --preset option") |
| |
| args.preset_substitutions = {} |
| for arg in args.preset_substitutions_raw: |
| name, value = arg.split("=", 1) |
| args.preset_substitutions[name] = value |
| |
| try: |
| preset = preset_parser.get_preset(args.preset, |
| vars=args.preset_substitutions) |
| except presets.Error as e: |
| diagnostics.fatal(e.message) |
| |
| preset_args = migrate_swift_sdks(preset.format_args()) |
| |
| if args.distcc and (args.cmake_c_launcher or args.cmake_cxx_launcher): |
| diagnostics.fatal( |
| '--distcc can not be used with' + |
| ' --cmake-c-launcher or --cmake-cxx-launcher') |
| |
| build_script_args = [sys.argv[0]] |
| if args.dry_run: |
| build_script_args += ["--dry-run"] |
| |
| build_script_args += preset_args |
| if args.distcc: |
| build_script_args += ["--distcc"] |
| if args.build_jobs: |
| build_script_args += ["--jobs", str(args.build_jobs)] |
| if args.swiftsyntax_install_prefix: |
| build_script_args += ["--install-swiftsyntax", |
| "--install-destdir", |
| args.swiftsyntax_install_prefix] |
| if args.build_dir: |
| build_script_args += ["--build-dir", args.build_dir] |
| if args.cmake_c_launcher: |
| build_script_args += ["--cmake-c-launcher", args.cmake_c_launcher] |
| if args.cmake_cxx_launcher: |
| build_script_args += ["--cmake-cxx-launcher", args.cmake_cxx_launcher] |
| |
| diagnostics.note('using preset "{}", which expands to \n\n{}\n'.format( |
| args.preset, shell.quote_command(build_script_args))) |
| |
| if args.expand_build_script_invocation: |
| return 0 |
| |
| call_without_sleeping(build_script_args) |
| return 0 |
| |
| |
| # Main entry point for the normal mode. |
| def main_normal(): |
| parser = driver_arguments.create_argument_parser() |
| |
| args = migration.parse_args(parser, sys.argv[1:]) |
| |
| if args.build_script_impl_args: |
| # If we received any impl args, check if `build-script-impl` would |
| # accept them or not before any further processing. |
| try: |
| migration.check_impl_args(build_script_impl, |
| args.build_script_impl_args) |
| except ValueError as e: |
| exit_rejecting_arguments(e, parser) |
| |
| if '--check-args-only' in args.build_script_impl_args: |
| return 0 |
| |
| shell.dry_run = args.dry_run |
| |
| # Prepare and validate toolchain |
| if args.darwin_xcrun_toolchain is None: |
| xcrun_toolchain = os.environ.get('TOOLCHAINS', |
| defaults.DARWIN_XCRUN_TOOLCHAIN) |
| |
| diagnostics.note('Using toolchain {}'.format(xcrun_toolchain)) |
| args.darwin_xcrun_toolchain = xcrun_toolchain |
| |
| toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain) |
| os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain |
| |
| if args.host_cc is not None: |
| toolchain.cc = args.host_cc |
| if args.host_cxx is not None: |
| toolchain.cxx = args.host_cxx |
| if args.host_lipo is not None: |
| toolchain.lipo = args.host_lipo |
| if args.host_libtool is not None: |
| toolchain.libtool = args.host_libtool |
| if args.cmake is not None: |
| toolchain.cmake = args.cmake |
| |
| # Preprocess the arguments to apply defaults. |
| BuildScriptInvocation.apply_default_arguments(toolchain, args) |
| |
| # Validate the arguments. |
| BuildScriptInvocation.validate_arguments(toolchain, args) |
| |
| # Create the build script invocation. |
| invocation = BuildScriptInvocation(toolchain, args) |
| |
| # Sanitize the runtime environment. |
| invocation.initialize_runtime_environment() |
| |
| # Show SDKs, if requested. |
| if args.show_sdks: |
| debug.print_xcodebuild_versions() |
| |
| # Clean build directory if requested. |
| if args.clean: |
| clean_delay() |
| shell.rmtree(invocation.workspace.build_root) |
| |
| # Create build directory. |
| shell.makedirs(invocation.workspace.build_root) |
| |
| # Build ninja if required, which will update the toolchain. |
| if args.build_ninja: |
| invocation.build_ninja() |
| |
| # Execute the underlying build script implementation. |
| invocation.execute() |
| |
| if args.symbols_package: |
| print('--- Creating symbols package ---') |
| print('-- Package file: {} --'.format(args.symbols_package)) |
| |
| if platform.system() == 'Darwin': |
| prefix = targets.darwin_toolchain_prefix(args.install_prefix) |
| else: |
| prefix = args.install_prefix |
| |
| # As a security measure, `tar` normally strips leading '/' from paths |
| # it is archiving. To stay safe, we change working directories, then |
| # run `tar` without the leading '/' (we remove it ourselves to keep |
| # `tar` from emitting a warning). |
| with shell.pushd(args.install_symroot): |
| tar.tar(source=prefix.lstrip('/'), |
| destination=args.symbols_package) |
| |
| return 0 |
| |
| |
| def main(): |
| if not SWIFT_SOURCE_ROOT: |
| diagnostics.fatal( |
| "could not infer source root directory " + |
| "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)") |
| |
| if not os.path.isdir(SWIFT_SOURCE_ROOT): |
| diagnostics.fatal( |
| "source root directory \'" + SWIFT_SOURCE_ROOT + |
| "\' does not exist " + |
| "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)") |
| |
| # Determine if we are invoked in the preset mode and dispatch accordingly. |
| if any([(opt.startswith("--preset") or opt == "--show-presets") |
| for opt in sys.argv[1:]]): |
| return main_preset() |
| else: |
| return main_normal() |
| |
| |
| if __name__ == "__main__": |
| try: |
| sys.exit(main()) |
| except KeyboardInterrupt: |
| sys.exit(1) |
