recipes/run_script.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for running a specified script from a given repo."""

from future.moves.urllib.parse import urlparse

from PB.recipes.fuchsia.run_script import InputProperties

PYTHON_VERSION_COMPATIBILITY = "PY3"

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/gerrit",
    "fuchsia/git",
    "fuchsia/repo",
    "fuchsia/sso",
    "fuchsia/status_check",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout_dir = api.path["start_dir"].join("checkout")
    if props.checkout_with_repo:
        # For repo checkouts, Git must be configured to rewrite SSO URLs to
        # HTTPS.
        if props.remote.startswith("sso://"):
            api.sso.configure_insteadof(props.remote)
        api.repo.checkout_from_build_input(
            props.remote,
            path=checkout_dir,
            fallback_ref=props.fallback_ref,
        )
    elif props.jiri_manifest:
        api.checkout.with_options(
            path=checkout_dir,
            manifest=props.jiri_manifest,
            remote=props.remote,
            project=props.jiri_project,
        )
    else:
        api.git.checkout_from_build_input(
            repo=props.remote,
            path=checkout_dir,
            fallback_ref=props.fallback_ref,
        )

    with api.context(cwd=checkout_dir):
        script_configs = []
        if props.script_configs:
            script_configs = props.script_configs

        for script_config in script_configs:
            process_script(api, script_config, props, checkout_dir)

        if props.attempt_roll:
            change = api.auto_roller.attempt_roll(
                api.gerrit.host_from_remote_url(props.remote),
                gerrit_project=urlparse(props.remote).path.lstrip("/"),
                repo_dir=checkout_dir,
                commit_message='Run "%s"'
                % ",".join([config.script for config in script_configs]),
                commit_untracked=props.roll_props.commit_untracked_files,
                dry_run=props.roll_props.dry_run,
            )
            return api.auto_roller.raw_result(change)


def process_script(api, script_config, props, checkout_dir):
    script_args = [
        arg.replace("{outdir}", str(api.path["cleanup"].join("outdir"))).replace(
            "{build-id}", str(api.buildbucket_util.id)
        )
        for arg in script_config.script_args
    ]

    if "{version}" in script_args:
        version = "0.0.0.{rev_count}".format(rev_count=api.git.rev_list_count("HEAD"))
        script_args = [arg.replace("{version}", version) for arg in script_args]

    if script_config.upload_to_cipd:
        # Script must emit a newline-separated txt file of CIPD yaml paths,
        # e.g. "/path/to/foo.yaml\n/path/to/bar.yaml" relative to the
        # working dir i.e. the checkout dir.
        assert (
            api.buildbucket.build.input.gitiles_commit.project
        ), "we should only be uploading in CI"
        cipd_yaml_manifest = api.path.mkstemp("cipd_manifest")
        script_args += ["--cipd-yaml-manifest", cipd_yaml_manifest]

    api.step("run %s" % script_config.script, [script_config.script] + script_args)

    if script_config.upload_to_cipd:
        upload_to_cipd(
            api,
            checkout_dir,
            cipd_yaml_manifest,
            set_repo_tags=props.set_repo_tags,
            use_json_cipd_yaml_manifest=props.use_json_cipd_yaml_manifest,
        )


def upload_to_cipd(
    api,
    checkout_dir,
    cipd_yaml_manifest,
    set_repo_tags=False,
    use_json_cipd_yaml_manifest=False,
):
    """Upload package(s) to CIPD from the script-generated CIPD .yaml manifest.

    Args:
        cipd_yaml_manifest (Path): Path to CIPD .yaml manifest.
        set_repo_tags (bool): If True, set CIPD tags based on a repo snapshot.
          Otherwise, set CIPD tags based on the input gitiles commit.
        use_json_cipd_yaml_manifest (bool): If True, read the script-generated
          CIPD .yaml manifest as JSON.
    """
    if use_json_cipd_yaml_manifest:
        # The modern JSON format is an array of objects each describing a path
        # to a CIPD yaml file and optionally, additional refs and/or tags to
        # attach.
        cipd_yaml_metadata = api.file.read_json(
            "read CIPD yaml manifest JSON",
            cipd_yaml_manifest,
            test_data=[
                {
                    "path": "path/to/foo.yaml",
                },
                {
                    "path": "path/to/bar.yaml",
                    "refs": ["stable"],
                    "tags": {"git_revision": "abc"},
                },
            ],
        )
    else:
        # The deprecated format is a newline-separated file of paths to CIPD
        # yaml files. It does not support additional refs and/or tags.
        cipd_yaml_paths = (
            api.file.read_text(
                "read CIPD yaml manifest",
                cipd_yaml_manifest,
                test_data="path/to/foo.yaml\npath/to/bar.yaml\n",
            )
            .rstrip()
            .split("\n")
        )
        # Convert to modern JSON format without refs or tags.
        cipd_yaml_metadata = [{"path": path} for path in cipd_yaml_paths]
    if set_repo_tags:
        tags = {
            # Replace any slash characters in a project's name, as they are not
            # allowed in CIPD tag keys.
            project.replace("/", "_"): revision
            for project, revision in api.repo.snapshot(
                "repo snapshot", path=checkout_dir
            ).items()
        }

    else:
        tags = {"git_revision": api.buildbucket.build.input.gitiles_commit.id}
    for metadata in cipd_yaml_metadata:
        # Attach any additional tags and/or refs specified for each package.
        final_tags = dict(tags)
        final_tags.update(metadata.get("tags", {}))
        api.cipd.create_from_yaml(
            checkout_dir.join(metadata["path"]),
            refs=["latest"] + metadata.get("refs", []),
            tags=final_tags,
        )


def GenTests(api):
    remote = "https://fuchsia.googlesource.com/foo"
    sso_remote = "sso://fuchsia/foo"

    yield (
        api.status_check.test("ci")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.sh",
                }
            ],
            remote=remote,
        )
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("ci_with_jiri")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.sh",
                    "upload_to_cipd": True,
                }
            ],
            remote=remote,
            jiri_manifest="manifest.xml",
            jiri_project="manifest",
        )
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("ci_with_repo")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.sh",
                    "upload_to_cipd": True,
                }
            ],
            remote=remote,
            checkout_with_repo=True,
            set_repo_tags=True,
        )
        + api.buildbucket.ci_build(git_repo=remote)
        + api.repo.snapshot("repo snapshot", ["project/a", "project/b"])
    )

    yield (
        api.status_check.test("sso")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.py",
                    "script_args": ["-flag", "val"],
                    "upload_to_cipd": False,
                },
                {
                    "script": "run-tests2.py",
                    "script_args": [
                        "-flag",
                        "val",
                        "-o",
                        "{outdir}",
                        "-b",
                        "{build-id}",
                        "-v",
                        "{version}",
                    ],
                    "upload_to_cipd": True,
                },
            ],
            remote=sso_remote,
            checkout_with_repo=True,
            attempt_roll=True,
        )
        + api.buildbucket.ci_build(git_repo=remote)
        + api.step_data("git rev-list --count", api.raw_io.stream_output_text("1"))
        + api.auto_roller.success()
    )

    yield (
        api.status_check.test("ci_upload")
        + api.properties(
            remote=remote,
            script_configs=[
                {
                    "script": "build-pkg.sh",
                    "upload_to_cipd": True,
                }
            ],
        )
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("use_json_cipd_manifest")
        + api.properties(
            script_configs=[
                {
                    "script": "build-pkg.sh",
                    "upload_to_cipd": True,
                }
            ],
            remote=remote,
            use_json_cipd_yaml_manifest=True,
        )
        + api.buildbucket.ci_build(git_repo=remote)
    )

    yield (
        api.status_check.test("cq")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.sh",
                }
            ],
            remote=remote,
        )
        + api.buildbucket.try_build(git_repo=remote)
    )

    yield (
        api.status_check.test("script_failed", status="failure")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.sh",
                }
            ],
            remote=remote,
        )
        + api.buildbucket.ci_build(git_repo=remote)
        + api.step_data("run run-tests.sh", retcode=1)
    )

    yield (
        api.status_check.test("no_buildbucket_input")
        + api.properties(
            script_configs=[
                {
                    "script": "run-tests.sh",
                }
            ],
            remote=remote,
        )
    )

    yield (
        api.status_check.test("attempt_roll")
        + api.properties(
            script_configs=[
                {
                    "script": "update.sh",
                }
            ],
            remote=remote,
            attempt_roll=True,
        )
        + api.auto_roller.success()
    )