recipes/fuchsia/fuchsia.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
""" fuchsia.py - Builds and optionally tests Fuchsia.

# Execution overview

## Configuration

This recipe uses a protocol buffer message called a spec for most of its
configuration. The only PROPERTIES are those required to acquire the spec.
The recipe fetches the spec from the git repo |spec_remote|. It determines
the correct revision to use from the BuildBucket build input to ensure it
retrieves the correct config for a pending change vs a committed change.

## Checkout + Build

This recipe triggers a child build which runs the fuchsia/build recipe.
That recipe checks out the source code and builds it. This recipe
retrieves the data required to orchestrate tests via Isolate.

## Test

If configured to run tests, this recipe uses the test orchestration data to run tests.
That logic is in the testing recipe module. Under the hood, that module
triggers Swarming tasks that do the actual testing, waits for them, and
reports the results.

## External Tests

If configured to run external tests, this recipe invokes various tools to pass
build artifacts to external infrastructures, trigger tests, waits for them, and
reports the results.
"""

from google.protobuf import json_format
from google.protobuf import text_format

from recipe_engine.recipe_api import Property

from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2
from PB.go.chromium.org.luci.buildbucket.proto import rpc as rpc_pb2
from PB.infra.fuchsia import Fuchsia
from PB.recipe_modules.recipe_engine.led.properties import (
    InputProperties as LedInputProperties,
)

DEPS = [
    "fuchsia/artifacts",
    "fuchsia/build",
    "fuchsia/build_input_resolver",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/fuchsia",
    "fuchsia/fxt",
    "fuchsia/gitiles",
    "fuchsia/spec",
    "fuchsia/subbuild",
    "fuchsia/testing",
    "recipe_engine/buildbucket",
    "recipe_engine/file",
    "recipe_engine/isolated",
    "recipe_engine/json",
    "recipe_engine/led",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/python",
    "recipe_engine/step",
    "recipe_engine/swarming",
]

PROPERTIES = {
    "child_build_id": Property(
        kind=str,
        help=(
            "The buildbucket ID of the child build. If set, "
            "will use this build instead of launching a new one."
        ),
        default=None,
    ),
    "spec_remote": Property(
        kind=str,
        help="URL of the specs git repository",
        default="http://fuchsia.googlesource.com/integration",
    ),
    "perfcompare": Property(
        kind=bool,
        help=(
            "Enable perfcompare mode: Compare performance between the "
            '"without CL" and "with CL" revisions.  This involves '
            "building and testing both of those revisions."
        ),
        default=False,
    ),
    # This property is not intended to be set by a config.  It is just here
    # so that a test case can pass a smaller number in order to make the
    # test expectations simpler.
    "boots_per_revision": Property(
        kind=int,
        help=(
            "Number of boots of Fuchsia to run performance tests on, "
            "in order to deal with cross-boot variation of performance. "
            "Only applies when perfcompare mode is enabled."
        ),
        default=5,
    ),
}


def RunSteps(api, child_build_id, spec_remote, perfcompare, boots_per_revision):
    # At some stage BuildBucket stores properties as google.protobuf.Value,
    # which converts all numbers (including ints) to floats, which is
    # lossy, so we have to use a string properties and convert to int
    # internally.
    child_build_id = int(child_build_id) if child_build_id else None

    spec, spec_revision = api.fuchsia.setup_with_spec(spec_remote)

    orchestrator_id = api.buildbucket_util.id
    if not spec.build.run_tests and not spec.external_tests.tap_projects:
        raise api.step.InfraFailure(
            "if not running tests, use the fuchsia/build recipe directly"
        )

    with api.step.nest("build") as presentation:
        child_build = run_build_steps(
            api, presentation, child_build_id, spec_revision, orchestrator_id
        )
        if spec.build.run_tests:
            child_props = json_format.MessageToDict(child_build.output.properties)
            (
                orchestration_inputs,
                orchestration_inputs_hash,
            ) = collect_test_orchestration_inputs(api, child_props, without_cl=False)
            if perfcompare:
                orchestration_inputs_without_cl, _ = collect_test_orchestration_inputs(
                    api, child_props, without_cl=True
                )

            # Copy to output properties so the coverage recipe can access it.
            presentation.properties[
                api.build.TestOrchestrationInputs.HASH_PROPERTY
            ] = orchestration_inputs_hash
            presentation.properties[api.checkout.ROOT_DIR_PROPERTY] = child_props[
                api.checkout.ROOT_DIR_PROPERTY
            ]
            # Copy to our own properties so the results uploader in google3 can find
            # it without knowing about the child.
            rev_count_prop = api.checkout.REVISION_COUNT_PROPERTY
            if rev_count_prop in child_props:
                presentation.properties[rev_count_prop] = child_props[rev_count_prop]

    # Configure context of uploaded artifacts for test task construction.
    api.artifacts.gcs_bucket = spec.artifact_gcs_bucket
    api.artifacts.uuid = orchestrator_id

    runs_per_shard = boots_per_revision if perfcompare else 1
    if spec.build.run_tests:
        results_with_cl = run_test_steps(
            api, orchestration_inputs, spec, runs_per_shard
        )
    if spec.external_tests.tap_projects:
        with api.step.nest("run external tests") as presentation:
            run_external_test_steps(api, presentation, spec)
    if perfcompare:
        with api.step.nest("test without CL"):
            results_without_cl = run_test_steps(
                api, orchestration_inputs_without_cl, spec, runs_per_shard
            )
        compare_performance_results(
            api, orchestration_inputs.perfcompare, results_without_cl, results_with_cl
        )


def run_build_steps(api, presentation, child_build_id, spec_revision, orchestrator_id):
    builder_name = "{}-subbuild".format(api.buildbucket.build.builder.builder)
    if child_build_id:
        # Text is meant to avoid confusion.
        presentation.step_text = "Reusing child build instead of triggering"
        output_build = api.buildbucket.get(child_build_id)
        build_url = "https://ci.chromium.org/b/%d" % child_build_id
        presentation.links[builder_name] = build_url
    else:
        properties = {
            "spec_revision": spec_revision,
            "parent_id": orchestrator_id,
        }
        builds = api.subbuild.launch(
            [builder_name], presentation, extra_properties=properties
        )
        build_id = builds[builder_name].build_id
        build_url = builds[builder_name].url
        if not api.led.launched_by_led:
            # Consumed by rerun recipe.
            with api.step.nest("child build id") as pres:
                # Passing build IDs directly as properties doesn't work because all
                # numbers get cast to floats, which is lossy, so we convert to str.
                pres.properties["child_build_id"] = str(build_id)
        builds = api.subbuild.collect([build_id], presentation)
        output_build = builds[build_id].build_proto

    if output_build.status != common_pb2.SUCCESS:
        if output_build.status == common_pb2.INFRA_FAILURE:
            exception_type = api.step.InfraFailure
            description = "raised infra failure"
        else:
            exception_type = api.step.StepFailure
            description = "failed"

        # Copy the child summary markdown into the parent summary markdown to
        # better propagate error messages. If the child summary is multiple lines,
        # start it on a new line.
        subbuild_summary = output_build.summary_markdown
        raise exception_type(
            "[build](%s) %s:%s%s"
            % (
                build_url,
                description,
                "\n\n" if "\n" in subbuild_summary else " ",
                subbuild_summary,
            )
        )

    return output_build


def run_test_steps(api, orchestration_inputs, spec, runs_per_shard):
    tryjob = api.buildbucket_util.is_tryjob

    testing_tasks = api.testing.run_test_tasks(
        debug_symbol_url=api.artifacts.debug_symbol_url(),
        orchestration_inputs=orchestration_inputs,
        max_attempts=spec.test.max_attempts,
        rerun_budget_secs=spec.test.rerun_budget_secs,
        runs_per_shard=runs_per_shard,
    )

    all_results = []
    final_results = []
    for t in testing_tasks:
        for attempt in t.attempts:
            if attempt.test_results:
                all_results.append(attempt.test_results)
                if attempt.index == len(t.attempts) - 1:
                    final_results.append(attempt.test_results)

    # Upload test results
    if spec.test.upload_results:
        assert spec.gcs_bucket, "gcs_bucket must be set if test.upload_results is"
        with api.step.nest("upload test results") as presentation:
            link = "go/fuchsia-result-store/bid:%s" % api.buildbucket_util.id
            presentation.links[link] = link.replace("go/", "https://goto.google.com/")
            swarming_task_ids = []
            # All test results, including non-final attempts that were retried.
            # We care so we can analyze flakiness.
            for test_results in all_results:
                test_results.upload_results(
                    gcs_bucket=spec.gcs_bucket,
                    upload_to_catapult=(
                        not tryjob
                        and spec.test.catapult_dashboard_master
                        and spec.test.catapult_dashboard_bot
                    ),
                )
                swarming_task_ids.append(test_results.swarming_task_id)
            # Consumed by the google3 results uploader and the coverage recipe.
            presentation.properties["test-swarming-task-ids"] = swarming_task_ids

    # Raise test failures
    with api.step.defer_results():
        api.testing.raise_failures(testing_tasks)
        # Only for the final attempt of each testing task, otherwise this
        # defeats the point of retries.
        for test_results in final_results:
            test_results.raise_failures()

    return final_results


def run_external_test_steps(api, presentation, spec):
    resp = api.fxt.launch(
        step_name="launch",
        bucket=api.artifacts.gcs_bucket,
        uuid=api.artifacts.uuid,
        name=spec.external_tests.system_image,
        projects=spec.external_tests.tap_projects,
        presentation=presentation,
    )
    # Cleanup workspace regardless of whether the tests pass or fail.
    with api.step.defer_results():
        api.fxt.monitor(step_name="monitor", request_id=resp["request_id"])
        api.fxt.cleanup(step_name="cleanup", workspace=resp["workspace"])


def collect_test_orchestration_inputs(api, build_props, without_cl):
    """Downloads isolated orchestration inputs from a build.

    Args:
      build_props (dict): The properties of the build that produced the test
          orchestration inputs.
      without_cl (bool): Whether to download the "without CL" build.  If false,
          this downloads the "with CL" build.

    Returns:
      FuchsiaBuildApi.TestOrchestrationInputs, hash (str)

    Raises:
      A StepFailure if the required property is not found.
    """
    prop_name = api.build.TestOrchestrationInputs.property_name(without_cl)
    orchestration_inputs_hash = build_props.get(prop_name)
    if not orchestration_inputs_hash:
        raise api.step.StepFailure("no `%s` property found" % prop_name)
    return (
        api.build.TestOrchestrationInputs.download(api, orchestration_inputs_hash),
        orchestration_inputs_hash,
    )


def compare_performance_results(
    api, perfcompare_tool_dir, results_without_cl, results_with_cl
):
    with api.step.nest(
        "convert results to perfcompare directory layout"
    ) as presentation:
        dest_dir = api.path["cleanup"].join("perf_dataset")
        api.file.ensure_directory("make directory", dest_dir)
        dir_args = [
            make_perfcompare_dataset_dir(
                api, dest_dir, results_without_cl, "without_cl"
            ),
            make_perfcompare_dataset_dir(api, dest_dir, results_with_cl, "with_cl"),
        ]
        # Upload the with/without-CL dataset to Isolate so that it can be
        # easily downloaded for further analysis.  The cost of this should be
        # low because Isolate uses share-by-hash (it is content-addressed) and
        # the individual files already came from Isolate.
        isolated = api.isolated.isolated(dest_dir)
        isolated.add_dir(dest_dir)
        dataset_hash = isolated.archive("isolate")
        presentation.properties["perfcompare_dataset_hash"] = dataset_hash
    api.python(
        "compare perf test results without and with CL",
        perfcompare_tool_dir.join("perfcompare.py"),
        ["compare_perf"] + dir_args,
    )


# Convert test results into the directory layout accepted by the
# perfcompare.py tool for a multi-boot dataset.
def make_perfcompare_dataset_dir(api, dest_parent_dir, all_results, with_or_without_cl):
    with api.step.nest('convert results for "%s" revision' % with_or_without_cl):
        dest_dir = dest_parent_dir.join(with_or_without_cl)
        by_boot_dir = dest_dir.join("by_boot")
        api.file.ensure_directory("make results directory", by_boot_dir)
        for boot_idx, test_results in enumerate(all_results):
            api.file.copytree(
                "copy perf test results",
                test_results.output_dir,
                by_boot_dir.join("boot%06d" % boot_idx),
            )
        return dest_dir


def GenTests(api):
    def download_step_data(legacy_qemu):
        task_request_jsonish = api.testing.task_request_jsonish(legacy_qemu=legacy_qemu)
        return api.testing.task_requests_step_data(
            [task_request_jsonish],
            "build.download test orchestration inputs.load task requests",
        )

    def test_step_data(test_in_shards=True, id="610", failure=False):
        shard_name = "QEMU"
        if test_in_shards:
            legacy_qemu = False
            outputs = ["out/path/to/output/file"]
        else:
            legacy_qemu = True
            outputs = ["output.fs"]
        return download_step_data(legacy_qemu) + (
            api.testing.task_retry_step_data(
                [api.swarming.task_result(id=id, name=shard_name, outputs=outputs,),]
            )
            + api.testing.test_step_data(
                shard_name=shard_name, legacy_qemu=legacy_qemu, failure=failure)
        )

    def external_tests_step_data(success=True):
        return api.fxt.launch(
            "run external tests.launch",
            test_data={
                "request_id": "test-id",
                "workspace": "test-ws",
                "change_num": 12345,
            },
        ) + api.fxt.monitor("run external tests.monitor", success=success)

    def spec_data(
        variants=(),
        device_type="QEMU",
        run_tests=True,
        run_external_tests=False,
        test_in_shards=True,
        gcs_bucket=None,
        max_attempts=None,
    ):
        test_spec = None
        if run_tests:
            test_spec = Fuchsia.Test(
                device_type=device_type,
                max_shard_size=0,
                timeout_secs=30 * 60,
                pool="fuchsia.tests",
                test_in_shards=test_in_shards,
                swarming_expiration_timeout_secs=10 * 60,
                swarming_io_timeout_secs=5 * 60,
                upload_results=bool(gcs_bucket),
                use_runtests=True,
                max_attempts=max_attempts,
            )
        external_tests_spec = None
        if run_external_tests:
            external_tests_spec = Fuchsia.ExternalTests(
                system_image="core-x64", tap_projects=["fuchsia.tests"],
            )

        return api.spec.spec_loaded_ok(
            step_name="load spec.build_init",
            message=Fuchsia(
                checkout=Fuchsia.Checkout(
                    manifest="manifest",
                    remote="remote",
                    upload_results=bool(gcs_bucket),
                ),
                build=Fuchsia.Build(
                    variants=variants,
                    build_type="debug",
                    run_tests=run_tests,
                    board="boards/x64.gni",
                    product="products/core.gni",
                    target="x64",
                    upload_results=bool(gcs_bucket),
                ),
                test=test_spec,
                external_tests=external_tests_spec,
                gcs_bucket=gcs_bucket,
                artifact_gcs_bucket=gcs_bucket,
            ),
        )

    child_build = api.subbuild.ci_build_message(
        builder="builder-subbuild",
        output_props={
            "checkout_root": "/checkout/root",
            "integration-revision-count": 1,
            "test_orchestration_inputs_hash": "abc",
        },
        status="SUCCESS",
    )
    failed_child_build = api.subbuild.ci_build_message(
        builder="builder-subbuild",
        output_props={
            "checkout_root": "/checkout/root",
            "integration-revision-count": 1,
            "test_orchestration_inputs_hash": "abc",
        },
        status="FAILURE",
    )
    failed_child_build.summary_markdown = "failed to build fuchsia"
    infra_failure_child_build = api.subbuild.ci_build_message(
        builder="builder-subbuild",
        output_props={
            "checkout_root": "/checkout/root",
            "integration-revision-count": 1,
            "test_orchestration_inputs_hash": "abc",
        },
        status="INFRA_FAILURE",
    )
    infra_failure_child_build.summary_markdown = "checkout timed out\nafter 30m"

    yield api.fuchsia.test(
        "successful_build_and_test",
        steps=[
            api.subbuild.child_build_steps(builds=[child_build]),
            api.override_step_data(
                "launch/collect.0.collect",
                api.swarming.collect(
                    [
                        api.swarming.task_result(
                            id="610", name="QEMU", outputs=["out/path/to/output/file"],
                        ),
                    ]
                ),
            ),
        ],
    ) + spec_data(gcs_bucket="gcs-bucket", variants=("profile",)) + test_step_data()

    # Cover the case where the build succeeds but one or more tests fail.
    yield (
        api.fuchsia.test(
            "failed_tests",
            status="failure",
            steps=[
                api.subbuild.child_build_steps(builds=[child_build]),
                api.override_step_data(
                    "launch/collect.0.collect",
                    api.swarming.collect(
                        [
                            api.swarming.task_result(
                                id="610",
                                name="QEMU",
                                outputs=["out/path/to/output/file"],
                            ),
                        ]
                    ),
                ),
            ],
        )
        # Pass max_attempts=1 because it keeps the test expectations
        # simpler.  Otherwise we would have to generate test expectations
        # for a retry of the failed task.
        + spec_data(gcs_bucket="gcs-bucket", variants=("profile",), max_attempts=1)
        + test_step_data(failure=True)
    )

    # Test expectations for the "without CL" recipe steps of perfcompare mode.
    def perfcompare_test_step_data():
        test = test_step_data(id="611")

        def rename_step(old_name, new_name):
            test.step_data[new_name] = test.step_data[old_name]

        rename_step(
            "build.download test orchestration inputs.load task requests",
            "build.download test orchestration inputs (2).load task requests",
        )
        launch_names = [
            "launch/collect.0.launch.QEMU (attempt 0).trigger",
            "launch/collect.0.process results.QEMU.tefmocheck",
            "launch/collect.0.process results.QEMU.get extracted files",
        ]
        for name in launch_names:
            rename_step(name, "test without CL.%s" % name)
        return test

    yield (
        api.fuchsia.test(
            "successful_build_and_test_perfcompare",
            steps=[
                api.subbuild.child_build_steps(
                    builds=[
                        api.subbuild.ci_build_message(
                            builder="builder-subbuild",
                            output_props={
                                "checkout_root": "/checkout/root",
                                "integration-revision-count": 1,
                                "test_orchestration_inputs_hash": "abc",
                                "test_orchestration_inputs_hash_without_cl": "efg",
                            },
                            status="SUCCESS",
                        )
                    ]
                ),
                api.override_step_data(
                    "launch/collect.0.collect",
                    api.swarming.collect(
                        [
                            api.swarming.task_result(
                                id="610",
                                name="QEMU",
                                outputs=["out/path/to/output/file"],
                            ),
                        ]
                    ),
                ),
                api.override_step_data(
                    "test without CL.launch/collect.0.collect",
                    api.swarming.collect(
                        [
                            api.swarming.task_result(
                                id="611",
                                name="QEMU",
                                outputs=["out/path/to/output/file"],
                            ),
                        ]
                    ),
                ),
                perfcompare_test_step_data(),
            ],
        )
        + spec_data(gcs_bucket="gcs-bucket", variants=("profile",))
        + test_step_data()
        # We test boots_per_revision=1 because generating matching test
        # expectations for a higher number is currently difficult.
        + api.properties(perfcompare=True, boots_per_revision=1)
    )

    yield api.fuchsia.test(
        "child_build_provided__test_not_in_shards",
        steps=[
            api.buildbucket.simulated_get(
                child_build, step_name="build.buildbucket.get"
            ),
        ],
    ) + spec_data(gcs_bucket="gcs-bucket", test_in_shards=False) + test_step_data(
        test_in_shards=False
    ) + api.properties(
        child_build_id=str(child_build.id)
    )

    yield api.fuchsia.test("build_only_failed", status="infra_failure",) + spec_data(
        run_tests=False
    )

    yield api.fuchsia.test(
        "build_failed",
        status="failure",
        steps=[api.subbuild.child_build_steps(builds=[failed_child_build]),],
    ) + spec_data(gcs_bucket="gcs-bucket", variants=("profile",))

    yield api.fuchsia.test(
        "build_infra_failure",
        status="infra_failure",
        steps=[api.subbuild.child_build_steps(builds=[infra_failure_child_build]),],
    ) + spec_data(gcs_bucket="gcs-bucket", variants=("profile",))

    yield api.fuchsia.test(
        "build_with_led",
        status="failure",
        properties={
            "$recipe_engine/led": LedInputProperties(
                led_run_id="led/user_example.com/abc123",
                isolated_input=LedInputProperties.IsolatedInput(
                    hash="abc123",
                    namespace="default-gzip",
                    server="isolateserver.appspot.com",
                ),
            ),
        },
        steps=[
            api.subbuild.child_led_steps(
                api=api,
                builds=[failed_child_build],
                props={
                    "gcs_bucket": api.fuchsia.DEFAULT_GCS_BUCKET,
                    "parent_id": "parentid",
                },
            ),
        ],
    ) + spec_data()

    yield api.fuchsia.test(
        "build_with_led_tryjob",
        status="failure",
        properties={
            "$recipe_engine/led": LedInputProperties(
                led_run_id="led/user_example.com/abc123",
                isolated_input=LedInputProperties.IsolatedInput(
                    hash="abc123",
                    namespace="default-gzip",
                    server="isolateserver.appspot.com",
                ),
            ),
        },
        tryjob=True,
        steps=[
            api.subbuild.child_led_steps(
                api=api,
                tryjob=True,
                builds=[failed_child_build],
                props={
                    "gcs_bucket": api.fuchsia.DEFAULT_GCS_BUCKET,
                    "parent_id": "parentid",
                },
            ),
        ],
    ) + api.build_input_resolver.set_gerrit_branch() + api.gitiles.refs(
        "refs", ["refs/heads/master", "deadbeef",]
    ) + spec_data()

    yield api.fuchsia.test(
        "build_passed_but_hash_is_missing",
        status="failure",
        steps=[
            api.subbuild.child_build_steps(
                builds=[
                    api.subbuild.ci_build_message(
                        builder="builder-subbuild", status="SUCCESS",
                    )
                ]
            )
        ],
    ) + spec_data()

    yield api.fuchsia.test(
        "successful_external_tests",
        steps=[api.subbuild.child_build_steps(builds=[child_build])],
    ) + spec_data(
        gcs_bucket="gcs-bucket", run_tests=False, run_external_tests=True
    ) + external_tests_step_data()

    yield api.fuchsia.test(
        "failed_external_tests",
        status="failure",
        steps=[api.subbuild.child_build_steps(builds=[child_build])],
    ) + spec_data(
        gcs_bucket="gcs-bucket", run_tests=False, run_external_tests=True
    ) + external_tests_step_data(
        success=False
    )