recipes/recipe_tools_roller.py

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe to autoroll tools into a recipes repository."""

from PB.infra.tool_metadata import ToolMetadata
from PB.recipes.fuchsia.recipe_tools_roller import InputProperties

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/ensure_tool",
    "fuchsia/git_checkout",
    "recipe_engine/cipd",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    target_ref = props.ref or "latest"

    checkout_root, _ = api.git_checkout(repo=props.remote)

    # Find all tool manifests.
    tool_manifest_paths = []
    tool_manifest_paths += api.file.glob_paths(
        "find module tool manifests",
        checkout_root.join("recipe_modules"),
        "*/*resources/**/tool_manifest.json",
    )
    tool_manifest_paths += api.file.glob_paths(
        "find recipe tool manifests",
        checkout_root.join("recipes"),
        "**/*.resources/**/tool_manifest.json",
    )

    updated_tools = []

    # Ensure all tool manifests match the target ref.
    for tool_manifest_path in tool_manifest_paths:
        # Get relpath with a stripped extension for a unique and valid step
        # name, which will look like
        # "recipe_modules/module/resource/tool_manifest".
        tool_manifest_relpath = api.path.relpath(
            api.path.splitext(tool_manifest_path)[0],
            checkout_root,
        )
        with api.step.nest("read %s" % tool_manifest_relpath) as presentation:
            tool_metadata = api.ensure_tool.get_tool_metadata(
                "tool", tool_manifest_path
            )
            if tool_metadata.do_not_autoroll:
                presentation.step_text = "autoroll disabled by manifest"
                continue
        with api.step.nest("ensure %s is %s" % (tool_manifest_relpath, target_ref)):
            # Use the tag prefix which is currently in the tool manifest.
            tag_prefix = tool_metadata.version.split(":", 1)[0]
            current = api.cipd.describe(tool_metadata.path, tool_metadata.version)
            target = api.cipd.describe(tool_metadata.path, target_ref)
            if current.pin.instance_id == target.pin.instance_id:
                # This tool is already pinned to the latest available instance,
                # so no need to update it.
                continue
            # Update the tool manifest to the oldest tag which starts with the
            # tag prefix. Using the oldest tag ensures that consecutive attempts
            # to roll a tool to a given package instance will yield the same
            # diff even if newer tags are added to the instance between
            # attempts.
            tags = [t for t in target.tags if t.tag.startswith(tag_prefix)]
            oldest_tag = min(tags, key=lambda t: t.registered_ts)
            api.ensure_tool.write_tool_metadata(
                tool_manifest_path,
                tool_metadata.path,
                oldest_tag.tag,
                do_not_autoroll=tool_metadata.do_not_autoroll,
            )
            updated_tools.append(tool_metadata.path)

    # TODO(fxbug.dev/84472): Construct a meaningful commit message on a per-tool
    # basis once this is parallelized.
    commit_message = "[roll] Update tools to %s" % target_ref
    if updated_tools:
        commit_message += "\n\n" + "\n".join("- %s" % t for t in updated_tools) + "\n"

    # TODO(fxbug.dev/84472): Parallelize roll attempts per tool manifest. The
    # current behavior combines all tool manifest changes into one CL.
    change = api.auto_roller.attempt_roll(
        api.auto_roller.Options(
            remote=props.remote,
            dry_run=props.dry_run,
        ),
        repo_dir=checkout_root,
        commit_message=commit_message,
    )
    return api.auto_roller.raw_result(change)


def GenTests(api):
    props = api.properties(
        remote="https://fuchsia.googlesource.com/infra/recipes",
    )
    tool_manifests = api.step_data(
        "find module tool manifests",
        api.file.glob_paths(
            [
                "module_a/resources/tool_manifest.json",
                "module_b/resources/tool_manifest.json",
            ]
        ),
    ) + api.step_data(
        "find recipe tool manifests",
        api.file.glob_paths(
            [
                "foo/resources/tool_manifest.json",
            ]
        ),
    )
    # module_a should not roll.
    describe_current_a = api.step_data(
        "ensure recipe_modules/module_a/resources/tool_manifest is latest.cipd describe path/to/tool",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:pinned-version",
            test_data_tags=["version:pinned-version"],
        ),
    )
    describe_target_a = api.step_data(
        "ensure recipe_modules/module_a/resources/tool_manifest is latest.cipd describe path/to/tool (2)",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:pinned-version",
            test_data_tags=["version:pinned-version"],
        ),
    )
    # module_b should roll.
    describe_current_b = api.step_data(
        "ensure recipe_modules/module_b/resources/tool_manifest is latest.cipd describe path/to/tool",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:pinned-version",
            test_data_tags=["version:pinned-version"],
        ),
    )
    describe_target_b = api.step_data(
        "ensure recipe_modules/module_b/resources/tool_manifest is latest.cipd describe path/to/tool (2)",
        api.cipd.example_describe(
            package_name="path/to/tool",
            version="version:new-version-to-pin",
            test_data_tags=["version:new-version-to-pin"],
        ),
    )
    read_manifest_foo = api.step_data(
        "read recipes/foo/resources/tool_manifest.read manifest",
        api.file.read_proto(
            ToolMetadata(
                path="path/to/tool",
                version="version:pinned-version",
                do_not_autoroll=True,
            ),
        ),
    )
    yield (
        api.buildbucket_util.test("roll_tool")
        + tool_manifests
        + props
        + describe_current_a
        + describe_target_a
        + describe_current_b
        + describe_target_b
        + read_manifest_foo
        + api.auto_roller.success()
    )