recipes/cipd_json_roller.py

# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe to autoroll CIPD packages pinned in JSON files."""

from PB.infra.tool_metadata import ToolMetadata
from PB.recipes.fuchsia.cipd_json_roller import InputProperties

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/ensure_tool",
    "fuchsia/git_checkout",
    "recipe_engine/cipd",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout_root, _ = api.git_checkout(repo=props.remote)

    manifest_paths = []
    with api.step.nest("search for manifests"):
        for glob in props.manifest_globs:
            manifest_paths += api.file.glob_paths(f"glob `{glob}`", checkout_root, glob)

    updated_packages = []
    # Ensure all manifests match their target ref.
    for manifest_path in manifest_paths:
        # Get relpath with a stripped extension for a unique and valid step
        # name.
        manifest_relpath = api.path.relpath(
            api.path.splitext(manifest_path)[0],
            checkout_root,
        )
        with api.step.nest(f"read {manifest_relpath}") as presentation:
            metadata = api.ensure_tool.get_tool_metadata("tool", manifest_path)
            if metadata.do_not_autoroll:
                presentation.step_text = "autoroll disabled by manifest"
                continue
        target_ref = metadata.target_ref or "latest"
        with api.step.nest(f"ensure {manifest_relpath} is {target_ref}"):
            # Use the tag prefix which is currently in the manifest.
            tag_prefix = metadata.version.split(":", 1)[0]
            current = api.cipd.describe(metadata.path, metadata.version)
            target = api.cipd.describe(metadata.path, target_ref)
            if current.pin.instance_id == target.pin.instance_id:
                # This package is already pinned to the latest available instance,
                # so no need to update it.
                continue
            # Update the manifest to the oldest tag which starts with the tag
            # prefix. Using the oldest tag ensures that consecutive attempts to
            # roll a package to a given instance will yield the same diff even
            # if newer tags are added to the instance between attempts.
            tags = [t for t in target.tags if t.tag.startswith(tag_prefix)]
            oldest_tag = min(tags, key=lambda t: t.registered_ts)

            metadata.version = oldest_tag.tag
            api.ensure_tool.write_tool_metadata(manifest_path, metadata)

            updated_packages.append(metadata.path)

    for cmd in props.postprocess_commands:
        api.step(
            f"run {cmd.path}",
            [checkout_root.join(cmd.path)] + list(cmd.args),
        )

    commit_message = props.commit_message
    assert commit_message, "commit_message property is required"

    if updated_packages:
        commit_message += "\n\n" + "\n".join(f"- {p}" for p in updated_packages) + "\n"

    # TODO(andresvi): Once builders pass the roll_options props, stop rewriting it here.
    props.roll_options.remote = props.remote
    props.roll_options.dry_run = props.dry_run

    change = api.auto_roller.attempt_roll(
        props.roll_options,
        repo_dir=checkout_root,
        commit_message=commit_message,
    )
    return api.auto_roller.raw_result(change)


def GenTests(api):
    yield (
        api.buildbucket_util.test("successful_roll")
        + api.properties(
            remote="https://example.googlesource.com/foo",
            commit_message="[roll] Update pinned packages",
            manifest_globs=[
                "subdir1/**/manifest.json",
                "subdir2/**/manifest.json",
            ],
            postprocess_commands=[
                {
                    "path": "scripts/foo.sh",
                    "args": ["-a", "bar"],
                }
            ],
        )
        + api.step_data(
            "search for manifests.glob `subdir1/**/manifest.json`",
            api.file.glob_paths(
                [
                    "subdir1/a/manifest.json",
                    "subdir1/b/manifest.json",
                ]
            ),
        )
        + api.step_data(
            "search for manifests.glob `subdir2/**/manifest.json`",
            api.file.glob_paths(
                [
                    "subdir2/c/manifest.json",
                ]
            ),
        )
        # Packages in subdir1/a/manifest should *not* roll.
        + api.step_data(
            "ensure subdir1/a/manifest is latest.cipd describe path/to/tool",
            api.cipd.example_describe(
                package_name="path/to/tool",
                version="version:pinned-version",
                test_data_tags=["version:pinned-version"],
            ),
        )
        + api.step_data(
            "ensure subdir1/a/manifest is latest.cipd describe path/to/tool (2)",
            api.cipd.example_describe(
                package_name="path/to/tool",
                version="version:pinned-version",
                test_data_tags=["version:pinned-version"],
            ),
        )
        # Packages in subdir1/b/manifest *should* roll.
        + api.step_data(
            "ensure subdir1/b/manifest is latest.cipd describe path/to/tool",
            api.cipd.example_describe(
                package_name="path/to/tool",
                version="version:pinned-version",
                test_data_tags=["version:pinned-version"],
            ),
        )
        + api.step_data(
            "ensure subdir1/b/manifest is latest.cipd describe path/to/tool (2)",
            api.cipd.example_describe(
                package_name="path/to/tool",
                version="version:new-version-to-pin",
                test_data_tags=["version:new-version-to-pin"],
            ),
        )
        # Rolling is disabled for the package in subdir2/c/manifest.
        + api.step_data(
            "read subdir2/c/manifest.read manifest",
            api.file.read_proto(
                ToolMetadata(
                    path="path/to/foo",
                    version="version:pinned-version",
                    do_not_autoroll=True,
                ),
            ),
        )
        + api.auto_roller.success()
    )