recipes/run_script.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for running a specified script from a given repo."""

from urlparse import urlparse

from PB.recipes.fuchsia.run_script import InputProperties

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/gerrit",
    "fuchsia/git",
    "fuchsia/repo",
    "fuchsia/sso",
    "fuchsia/status_check",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/python",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout_dir = api.path["start_dir"].join("checkout")
    with api.step.nest("checkout"), api.context(infra_steps=True):
        if props.checkout_with_repo:
            # For repo checkouts, Git must be configured to rewrite SSO URLs to
            # HTTPS.
            if props.remote.startswith("sso://"):
                api.sso.configure_insteadof(props.remote)
            api.repo.checkout_from_build_input(
                props.remote,
                path=checkout_dir,
                fallback_ref=props.fallback_ref,
            )
        else:
            api.git.checkout_from_build_input(
                repo=props.remote,
                path=checkout_dir,
                fallback_ref=props.fallback_ref,
            )

    with api.context(cwd=checkout_dir):
        script_args = list(props.script_args)
        if props.upload_to_cipd:
            # Script must emit a newline-separated txt file of CIPD yaml paths,
            # e.g. "/path/to/foo.yaml\n/path/to/bar.yaml" relative to the
            # working dir i.e. the checkout dir.
            assert (
                api.buildbucket.build.input.gitiles_commit.project
            ), "we should only be uploading in CI"
            cipd_yaml_manifest = api.path.mkstemp("cipd_manifest")
            script_args += ["--cipd-yaml-manifest", cipd_yaml_manifest]

        step_name = "run %s" % props.script
        if props.script.endswith(".py"):
            # Run python scripts with vpython, rather than system Python, to
            # enable installation of packages via .vpython files.
            api.python(step_name, props.script, script_args, venv=True)
        else:
            api.step(step_name, [props.script] + script_args)

        if props.attempt_roll:
            change = api.auto_roller.attempt_roll(
                api.gerrit.host_from_remote_url(props.remote),
                gerrit_project=urlparse(props.remote).path.lstrip("/"),
                repo_dir=checkout_dir,
                commit_message='Run "%s"' % props.script,
                commit_untracked=props.roll_props.commit_untracked_files,
                dry_run=props.roll_props.dry_run,
            )
            return api.auto_roller.raw_result(change)

        if props.upload_to_cipd:
            upload_to_cipd(
                api,
                checkout_dir,
                cipd_yaml_manifest,
                set_repo_tags=props.set_repo_tags,
                use_json_cipd_yaml_manifest=props.use_json_cipd_yaml_manifest,
            )


def upload_to_cipd(
    api,
    checkout_dir,
    cipd_yaml_manifest,
    set_repo_tags=False,
    use_json_cipd_yaml_manifest=False,
):
    """Upload package(s) to CIPD from the script-generated CIPD .yaml manifest.

    Args:
        cipd_yaml_manifest (Path): Path to CIPD .yaml manifest.
        set_repo_tags (bool): If True, set CIPD tags based on a repo snapshot.
          Otherwise, set CIPD tags based on the input gitiles commit.
        use_json_cipd_yaml_manifest (bool): If True, read the script-generated
          CIPD .yaml manifest as JSON.
    """
    if use_json_cipd_yaml_manifest:
        # The modern JSON format is an array of objects each describing a path
        # to a CIPD yaml file and optionally, additional refs and/or tags to
        # attach.
        cipd_yaml_metadata = api.file.read_json(
            "read CIPD yaml manifest JSON",
            cipd_yaml_manifest,
            test_data=[
                {
                    "path": "path/to/foo.yaml",
                },
                {
                    "path": "path/to/bar.yaml",
                    "refs": ["stable"],
                    "tags": {"git_revision": "abc"},
                },
            ],
        )
    else:
        # The deprecated format is a newline-separated file of paths to CIPD
        # yaml files. It does not support additional refs and/or tags.
        cipd_yaml_paths = (
            api.file.read_text(
                "read CIPD yaml manifest",
                cipd_yaml_manifest,
                test_data="path/to/foo.yaml\npath/to/bar.yaml\n",
            )
            .rstrip()
            .split("\n")
        )
        # Convert to modern JSON format without refs or tags.
        cipd_yaml_metadata = [{"path": path} for path in cipd_yaml_paths]
    if set_repo_tags:
        tags = {
            # Replace any slash characters in a project's name, as they are not
            # allowed in CIPD tag keys.
            project.replace("/", "_"): revision
            for project, revision in api.repo.snapshot(
                "repo snapshot", path=checkout_dir
            ).iteritems()
        }

    else:
        tags = {"git_revision": api.buildbucket.build.input.gitiles_commit.id}
    for metadata in cipd_yaml_metadata:
        # Attach any additional tags and/or refs specified for each package.
        final_tags = dict(tags)
        final_tags.update(metadata.get("tags", {}))
        api.cipd.create_from_yaml(
            checkout_dir.join(metadata["path"]),
            refs=["latest"] + metadata.get("refs", []),
            tags=final_tags,
        )


def GenTests(api):
    remote = "https://fuchsia.googlesource.com/foo"
    sso_remote = "sso://fuchsia/foo"

    yield (
        api.status_check.test("ci")
        + api.properties(script="run-tests.sh", remote=remote)
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("ci_with_repo")
        + api.properties(
            script="run-tests.sh",
            remote=remote,
            checkout_with_repo=True,
            upload_to_cipd=True,
            set_repo_tags=True,
        )
        + api.buildbucket.ci_build(git_repo=remote)
        + api.repo.snapshot("repo snapshot", ["project/a", "project/b"])
    )

    yield (
        api.status_check.test("sso")
        + api.properties(
            script="run-tests.py",
            remote=sso_remote,
            checkout_with_repo=True,
            script_args=["-flag", "flagval"],
        )
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("ci_upload")
        + api.properties(script="build-pkg.sh", remote=remote, upload_to_cipd=True)
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("use_json_cipd_manifest")
        + api.properties(
            script="build-pkg.sh",
            remote=remote,
            upload_to_cipd=True,
            use_json_cipd_yaml_manifest=True,
        )
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("cq")
        + api.properties(script="run-tests.sh", remote=remote)
        + api.buildbucket.try_build(git_repo=remote)
    )

    yield (
        api.status_check.test("script_failed", status="failure")
        + api.properties(script="run-tests.sh", remote=remote)
        + api.buildbucket.ci_build(git_repo=remote)
        + api.step_data("run run-tests.sh", retcode=1)
    )

    yield (
        api.status_check.test("no_buildbucket_input")
        + api.properties(script="run-tests.sh", remote=remote)
    )

    yield (
        api.status_check.test("attempt_roll")
        + api.properties(script="update.sh", remote=remote, attempt_roll=True)
        + api.auto_roller.success()
    )