recipes/cipd_json_roller.py

# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Recipe to autoroll CIPD packages pinned in CIPD ensure files.
TODO(atyfto): Merge into cipd_ensure_file_roller.py.
"""

from PB.infra.roller_metadata import RollerMetadata
from PB.recipes.fuchsia.cipd_json_roller import InputProperties

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/cipd_ensure",
    "fuchsia/cipd_resolver",
    "fuchsia/git_checkout",
    "recipe_engine/cipd",
    "recipe_engine/file",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout_root, _ = api.git_checkout(repo=props.remote)
    updated_packages = roll_ensure_files(api, checkout_root, props)

    for cmd in props.postprocess_commands:
        api.step(
            f"run {cmd.path}",
            [checkout_root / cmd.path] + list(cmd.args),
        )

    commit_message = props.commit_message
    assert commit_message, "commit_message property is required"

    if updated_packages:
        commit_message += "\n\n" + "\n".join(f"- {p}" for p in updated_packages) + "\n"

    change = api.auto_roller.attempt_roll(
        props.roll_options,
        repo_dir=checkout_root,
        commit_message=commit_message,
    )
    return api.auto_roller.raw_result(change)


def roll_ensure_files(api, checkout_root, props):
    ensure_files = []
    with api.step.nest("search for ensure files"):
        for glob in props.ensure_file_globs:
            ensure_files += api.file.glob_paths(f"glob `{glob}`", checkout_root, glob)

    updated_packages = []
    # Ensure all ensure files match their target ref.
    for ensure_file in ensure_files:
        # Get relpath with a stripped extension for a unique and valid step
        # name.
        ensure_file_relpath = api.path.relpath(
            api.path.splitext(ensure_file)[0],
            checkout_root,
        )
        with api.step.nest(f"read {ensure_file_relpath}") as presentation:
            # Roller metadata must have a specific filename within the ensure
            # file's directory. This can be changed if we decide there is a
            # better naming convention.
            metadata_path = ensure_file.parent / "roller_metadata.json"
            # Do not attempt to roll an ensure file that doesn't specify roller
            # metadata. We may decide at a future point that this should be an
            # opt-out instead of opt-in.
            api.path.mock_add_paths(metadata_path)
            if not api.path.exists(metadata_path):  # pragma: no cover
                continue
            metadata = api.cipd_ensure.get_roller_metadata(metadata_path)
            if metadata.do_not_autoroll:
                presentation.step_text = "autoroll disabled by roller metadata"
                continue
        with api.step.nest(f"ensure {ensure_file_relpath} is {metadata.ref}"):
            packages = api.cipd_ensure.get_packages(
                "get packages",
                ensure_file,
            )
            platforms = api.cipd_ensure.get_platforms(
                "get platforms",
                ensure_file,
            )
            expanded_packages = api.cipd_ensure.expand_packages_by_platforms(
                packages=packages,
                platforms=platforms,
            )
            candidate_versions = api.cipd_resolver.resolve_common_tags(
                ref=metadata.ref,
                # TODO(fxbug.dev/130779): Add a utility to resolve the tag
                # prefix of the packages in the ensure file, and remove `tag`
                # from the roller metadata.
                tag_name=metadata.tag,
                packages=expanded_packages,
            )
            updated_packages += api.cipd_ensure.update_packages(
                "update packages",
                ensure_file=ensure_file,
                packages=packages,
                version=candidate_versions[0],
            )
            api.cipd.ensure_file_resolve(ensure_file)

    return updated_packages


def GenTests(api):
    def resolved_tags(step_name, tags):
        return api.step_data(
            f"{step_name}.resolve common tags",
            api.json.output(tags),
        )

    yield (
        api.buildbucket_util.test("successful_roll")
        + api.properties(
            remote="https://example.googlesource.com/foo",
            commit_message="[roll] Update pinned packages",
            ensure_file_globs=[
                "subdir1/**/cipd.ensure",
                "subdir2/**/cipd.ensure",
            ],
            postprocess_commands=[
                {
                    "path": "scripts/foo.sh",
                    "args": ["-a", "bar"],
                }
            ],
            roll_options=api.auto_roller.Options(
                remote="https://example.googlesource.com/foo",
            ),
        )
        + api.step_data(
            "search for ensure files.glob `subdir1/**/cipd.ensure`",
            api.file.glob_paths(
                [
                    "subdir1/a/cipd.ensure",
                    "subdir1/b/cipd.ensure",
                ]
            ),
        )
        + api.step_data(
            "search for ensure files.glob `subdir2/**/cipd.ensure`",
            api.file.glob_paths(
                [
                    "subdir2/c/cipd.ensure",
                ]
            ),
        )
        + api.cipd_ensure.get_packages(
            "ensure subdir1/a/cipd is latest.get packages",
            {"path/to/tool": "version:pinned-version"},
        )
        + resolved_tags(
            "ensure subdir1/a/cipd is latest",
            ["version:pinned-version"],
        )
        + api.cipd_ensure.get_packages(
            "ensure subdir1/b/cipd is latest.get packages",
            {"path/to/tool": "version:old"},
        )
        + resolved_tags(
            "ensure subdir1/b/cipd is latest",
            ["version:new-version-to-pin"],
        )
        + api.cipd_ensure.update_packages(
            "ensure subdir1/b/cipd is latest.update packages",
            ["path/to/tool"],
        )
        + api.step_data(
            "read subdir2/c/cipd.read roller metadata",
            api.file.read_proto(
                RollerMetadata(
                    ref="latest",
                    do_not_autoroll=True,
                    tag="version",
                ),
            ),
        )
        + api.auto_roller.success()
    )