recipes/dart_toolchain.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Dart toolchain."""

from recipe_engine.config import Enum
from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/git",
    "fuchsia/goma",
    "fuchsia/macos_sdk",
    "fuchsia/status_check",
    "fuchsia/upload",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

DEPOT_TOOLS_GIT = "https://chromium.googlesource.com/chromium/tools/depot_tools"
DART_SDK_GIT = "https://dart.googlesource.com/sdk.git"

PROPERTIES = {
    "repository": Property(kind=str, help="Git repository URL", default=DART_SDK_GIT),
    "revision": Property(kind=str, help="Revision", default=None),
    "host_cpu": Property(
        kind=Enum("arm64", "x64"),
        help="GN $host_cpu toolchain will run on",
        default=None,
    ),
    "host_os": Property(
        kind=Enum("linux", "mac"),
        help="GN $host_os toolchain will run on",
        default=None,
    ),
}

# Fuchsia targets the toolchain must support.
FUCHSIA_TARGETS = [
    "arm64",
    "x64",
]

GN_CIPD_CPU_MAP = {
    "x64": "amd64",
}


def RunSteps(api, repository, revision, host_cpu, host_os):
    api.goma.ensure()

    gitiles_commit = api.buildbucket.build_input.gitiles_commit
    if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
        repository = "https://%s/%s" % (gitiles_commit.host, gitiles_commit.project)
        revision = gitiles_commit.id

    if not host_cpu:
        # TODO(mcgrathr): Native bots are all x64 and api.platform.arch is useless.
        host_cpu = "x64"
    if not host_os:
        host_os = api.platform.name
    cipd_platform = "%s-%s" % (host_os, GN_CIPD_CPU_MAP.get(host_cpu, host_cpu))

    with api.step.nest("ensure_packages"):
        with api.context(infra_steps=True):
            pkgs = api.cipd.EnsureFile()
            pkgs.add_package("infra/ninja/${platform}", "version:1.8.2")
            if host_os == "linux":
                pkgs.add_package("fuchsia/sysroot/%s" % cipd_platform, "latest")
            cipd_dir = api.path["start_dir"].join("cipd")
            api.cipd.ensure(cipd_dir, pkgs)

    ninja = [cipd_dir.join("ninja"), "-j%d" % api.goma.jobs]

    with api.context(infra_steps=True):
        with api.step.nest("depot_tools"):
            # TODO(phosek): Move this to a named cache so it's shared across builds.
            depot_tools_dir = api.path["start_dir"].join("depot_tools")
            api.git.checkout(DEPOT_TOOLS_GIT, depot_tools_dir, submodules=False)

        with api.step.nest("dart-sdk"):
            sdk_dir = api.path["start_dir"].join("dart", "sdk")
            revision = api.git.checkout(
                repository, sdk_dir, ref=revision, submodules=False
            )

    # Use gclient to fetch the DEPS, but don't let it change dart/sdk itself.
    dart_dir = api.path["start_dir"].join("dart")
    with api.context(
        infra_steps=True, cwd=dart_dir, env_prefixes={"PATH": [depot_tools_dir]}
    ):
        api.step(
            "gclient config",
            [
                "gclient",
                "config",
                "--unmanaged",
                # TODO(mcgrathr): Set --cache-dir?
                "-v",
                DART_SDK_GIT,
            ],
        )
        api.step(
            "gclient sync",
            [
                "gclient",
                "sync",
                "--no-history",
                "-v",
                "--output-json",
                api.json.output(),
            ],
        )

    dart_targets = sorted(
        set(
            [host_cpu]
            + ["sim" + cpu if cpu != host_cpu else cpu for cpu in FUCHSIA_TARGETS]
        )
    )

    def out_dir(cpu):
        return sdk_dir.join(
            "xcodebuild" if api.platform.is_mac else "out", "Release" + cpu.upper()
        )

    with api.macos_sdk(), api.goma.build_with_goma(), api.context(cwd=sdk_dir):
        # TODO(mcgrathr): Move this logic into a host_build recipe module shared
        # with gcc_toolchain.py and others.
        if host_os == "linux":
            host_sysroot = cipd_dir
        elif host_os == "mac":
            with api.context(infra_steps=True):
                step_result = api.step(
                    "xcrun",
                    ["xcrun", "--sdk", "macosx", "--show-sdk-path"],
                    stdout=api.raw_io.output(name="sdk-path", add_output_log=True),
                    step_test_data=lambda: api.raw_io.test_api.stream_output(
                        "/some/xcode/path"
                    ),
                )
                host_sysroot = step_result.stdout.strip()
        else:  # pragma: no cover
            assert False, "what platform?"

        gn_common = [
            "tools/gn.py",
            "-v",
            "--mode=release",
            "--goma",
            # It actually wants a sysroot for the build host regardless of what
            # Dart target CPU the build is configured for, but wants it in the
            # form `%(target_cpu)s=%(host_sysroot)s`.
            "--target-sysroot="
            + ",".join("%s=%s" % (cpu, host_sysroot) for cpu in dart_targets),
        ]

        api.step(
            "gn host (%s)" % host_cpu,
            gn_common + ["--arch=%s" % host_cpu, "--platform-sdk"],
        )
        for cpu in dart_targets:
            if cpu != host_cpu:
                api.step("gn %s" % cpu, gn_common + ["--arch=%s" % cpu])
        api.step(
            "build host (%s) SDK" % host_cpu,
            ninja
            + [
                "-C",
                out_dir(host_cpu),
                "create_sdk",
                "gen_snapshot",
                "gen_snapshot_product",
            ],
        )
        for cpu in dart_targets:
            api.step(
                "build %s" % cpu,
                ninja
                + [
                    "-C",
                    out_dir(cpu),
                    "gen_snapshot_fuchsia",
                    "gen_snapshot_product_fuchsia",
                    "runtime",
                ],
            )
        for cpu in dart_targets:
            remote_config = "dartk-linux-release-%s" % cpu
            local_config = "dartk-%s-release-%s" % (host_os, cpu)
            api.step(
                "run tests",
                [
                    "tools/sdks/dart-sdk/bin/dart",
                    "tools/test.dart",
                    "-n",
                    remote_config,
                    "-N",
                    local_config,
                    "--",
                    "--progress=line",
                    "--report",
                    "--time",
                    "vm",
                    "language",
                ],
            )

    with api.step.nest("install"):
        pkg_dir = api.path["cleanup"].join("dart-sdk")
        api.file.copytree(
            "install dart-sdk",
            out_dir(host_cpu).join("dart-sdk"),
            pkg_dir,
            symlinks=True,
        )
        for cpu, is_host in [(host_cpu, True)] + [(cpu, False) for cpu in dart_targets]:
            exe_dir = out_dir(cpu).join("exe.stripped")
            if not is_host and cpu.startswith("sim"):
                cpu = cpu[3:]
            for tool in ["gen_snapshot", "gen_snapshot_product"]:
                src = exe_dir.join(tool + ("" if is_host else "_fuchsia"))
                dst_name = "%s.%s-%s" % (tool, host_os if is_host else "fuchsia", cpu)
                dst = pkg_dir.join("bin", dst_name)
                api.file.copy("install %s" % dst_name, src, dst)

    dart_sdk_version = api.file.read_text(
        "read dart-sdk version",
        pkg_dir.join("version"),
        test_data="2.0.0-edge." + revision + "\n",
    ).strip()

    api.upload.cipd_package(
        "fuchsia/dart-sdk/" + cipd_platform,
        pkg_dir,
        [api.upload.DirectoryPath(pkg_dir)],
        {"git_revision": revision},
        repository,
        extra_tags={"dart_sdk_version": dart_sdk_version},
    )


def GenTests(api):
    revision = "301b5a1f16414bc031091eb214ddd6c589e6ed9a"
    for platform in ("linux", "mac"):
        yield (
            api.status_check.test(platform)
            + api.platform.name(platform)
            + api.step_data(
                "dart-sdk.git rev-parse", api.raw_io.stream_output(revision)
            )
            + api.buildbucket.ci_build(git_repo=DART_SDK_GIT, revision=revision)
        )
        yield (
            api.status_check.test(platform + "_new")
            + api.platform.name(platform)
            + api.buildbucket.ci_build(git_repo=DART_SDK_GIT, revision=revision)
            + api.step_data(
                "dart-sdk.git rev-parse", api.raw_io.stream_output(revision)
            )
            + api.step_data(
                "cipd.cipd search fuchsia/dart-sdk/"
                + platform
                + "-amd64 "
                + "git_revision:"
                + revision,
                api.json.output({"result": []}),
            )
            + api.step_data("gclient sync", api.json.output({"solutions": {}}))
        )
    for host_os, host_cpu, cipd_host_cpu in [("linux", "arm64", "arm64")]:
        cipd_platform = host_os + "-" + cipd_host_cpu
        yield (
            api.status_check.test(host_os + "_cross")
            + api.platform.name(host_os)
            + api.properties(host_cpu=host_cpu, host_os=host_os)
            + api.buildbucket.ci_build(git_repo=DART_SDK_GIT, revision=revision)
            + api.step_data(
                "dart-sdk.git rev-parse", api.raw_io.stream_output(revision)
            )
            + api.step_data(
                "cipd.cipd search fuchsia/dart-sdk/"
                + cipd_platform
                + " git_revision:"
                + revision,
                api.json.output({"result": []}),
            )
            + api.step_data("gclient sync", api.json.output({"solutions": {}}))
        )