recipes/recipe_tools_roller.py

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe to autoroll tools into a recipes repository."""

from PB.infra.tool_metadata import ToolMetadata
from PB.recipes.fuchsia.recipe_tools_roller import InputProperties

PYTHON_VERSION_COMPATIBILITY = "PY3"

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/ensure_tool",
    "fuchsia/gerrit",
    "fuchsia/git",
    "recipe_engine/cipd",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    target_ref = props.ref or "latest"

    checkout_root = api.path["start_dir"].join("checkout")
    api.git.checkout_from_build_input(repo=props.remote, path=checkout_root)

    # Find all tool manifests.
    modules_dir = checkout_root.join("recipe_modules")
    tool_manifest_paths = api.file.glob_paths(
        "find tool manifests",
        modules_dir,
        "*/resources/**/tool_manifest.json",
    )
    # Ensure all tool manifests match the target ref.
    for tool_manifest_path in tool_manifest_paths:
        # Get relpath with a stripped extension for a unique and valid step
        # name, which will look like "module/resource/tool_manifest".
        tool_manifest_relpath = api.path.relpath(
            api.path.splitext(tool_manifest_path)[0],
            modules_dir,
        )
        with api.step.nest("read %s" % tool_manifest_relpath) as presentation:
            tool_metadata = api.ensure_tool.get_tool_metadata(
                "tool", tool_manifest_path
            )
            if tool_metadata.do_not_autoroll:
                presentation.step_text = "autoroll disabled by manifest"
                continue
        with api.step.nest("ensure %s is %s" % (tool_manifest_relpath, target_ref)):
            # Use the tag prefix which is currently in the tool manifest.
            tag_prefix = tool_metadata.version.split(":", 1)[0]
            current = api.cipd.describe(tool_metadata.path, tool_metadata.version)
            target = api.cipd.describe(tool_metadata.path, target_ref)
            # Update the tool manifest to the most recent tag which starts with
            # the tag prefix.
            if current.pin.instance_id == target.pin.instance_id:
                continue
            tags = [t for t in target.tags if t.tag.startswith(tag_prefix)]
            most_recent_tag = max(tags, key=lambda t: t.registered_ts)
            api.ensure_tool.write_tool_metadata(
                tool_manifest_path,
                tool_metadata.path,
                most_recent_tag.tag,
                do_not_autoroll=tool_metadata.do_not_autoroll,
            )

    # TODO(fxbug.dev/84472): Parallelize roll attempts per tool manifest. The
    # current behavior combines all tool manifest changes into one CL.
    change = api.auto_roller.attempt_roll(
        api.gerrit.host_from_remote_url(props.remote),
        gerrit_project=api.gerrit.project_from_remote_url(props.remote),
        repo_dir=checkout_root,
        # TODO(fxbug.dev/84472): Construct a meaningful commit message on a
        # per-tool basis once this is parallelized.
        commit_message="[roll] Update tools to %s" % target_ref,
        dry_run=props.dry_run,
    )
    return api.auto_roller.raw_result(change)


def GenTests(api):
    props = api.properties(
        remote="https://fuchsia.googlesource.com/infra/recipes",
    )
    tool_manifests = api.step_data(
        "find tool manifests",
        api.file.glob_paths(
            [
                "module_a/resources/tool_manifest.json",
                "module_b/resources/tool_manifest.json",
                "module_c/resources/tool_manifest.json",
            ]
        ),
    )
    # module_a should not roll.
    describe_current_a = api.step_data(
        "ensure module_a/resources/tool_manifest is latest.cipd describe path/to/tool",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:pinned-version",
            test_data_tags=["version:pinned-version"],
        ),
    )
    describe_target_a = api.step_data(
        "ensure module_a/resources/tool_manifest is latest.cipd describe path/to/tool (2)",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:pinned-version",
            test_data_tags=["version:pinned-version"],
        ),
    )
    # module_b should roll.
    describe_current_b = api.step_data(
        "ensure module_b/resources/tool_manifest is latest.cipd describe path/to/tool",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:pinned-version",
            test_data_tags=["version:pinned-version"],
        ),
    )
    describe_target_b = api.step_data(
        "ensure module_b/resources/tool_manifest is latest.cipd describe path/to/tool (2)",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:new-version-to-pin",
            test_data_tags=["version:new-version-to-pin"],
        ),
    )
    read_manifest_c = api.step_data(
        "read module_c/resources/tool_manifest.read manifest",
        api.file.read_proto(
            ToolMetadata(
                path="path/to/tool",
                version="version:pinned-version",
                do_not_autoroll=True,
            ),
        ),
    )
    yield (
        api.buildbucket_util.test("roll_tool")
        + tool_manifests
        + props
        + describe_current_a
        + describe_target_a
        + describe_current_b
        + describe_target_b
        + read_manifest_c
        + api.auto_roller.success()
    )