| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Recipe to autoroll tools into a recipes repository.""" |
| |
| from PB.infra.tool_metadata import ToolMetadata |
| from PB.recipes.fuchsia.recipe_tools_roller import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/ensure_tool", |
| "fuchsia/git_checkout", |
| "recipe_engine/cipd", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps(api, props): |
| target_ref = props.ref or "latest" |
| |
| checkout_root, _ = api.git_checkout(repo=props.remote) |
| |
| # Find all tool manifests. |
| tool_manifest_paths = [] |
| tool_manifest_paths += api.file.glob_paths( |
| "find module tool manifests", |
| checkout_root.join("recipe_modules"), |
| "*/*resources/**/tool_manifest.json", |
| ) |
| tool_manifest_paths += api.file.glob_paths( |
| "find recipe tool manifests", |
| checkout_root.join("recipes"), |
| "**/*.resources/**/tool_manifest.json", |
| ) |
| |
| updated_tools = [] |
| |
| # Ensure all tool manifests match the target ref. |
| for tool_manifest_path in tool_manifest_paths: |
| # Get relpath with a stripped extension for a unique and valid step |
| # name, which will look like |
| # "recipe_modules/module/resource/tool_manifest". |
| tool_manifest_relpath = api.path.relpath( |
| api.path.splitext(tool_manifest_path)[0], |
| checkout_root, |
| ) |
| with api.step.nest("read %s" % tool_manifest_relpath) as presentation: |
| tool_metadata = api.ensure_tool.get_tool_metadata( |
| "tool", tool_manifest_path |
| ) |
| if tool_metadata.do_not_autoroll: |
| presentation.step_text = "autoroll disabled by manifest" |
| continue |
| with api.step.nest("ensure %s is %s" % (tool_manifest_relpath, target_ref)): |
| # Use the tag prefix which is currently in the tool manifest. |
| tag_prefix = tool_metadata.version.split(":", 1)[0] |
| current = api.cipd.describe(tool_metadata.path, tool_metadata.version) |
| target = api.cipd.describe(tool_metadata.path, target_ref) |
| if current.pin.instance_id == target.pin.instance_id: |
| # This tool is already pinned to the latest available instance, |
| # so no need to update it. |
| continue |
| # Update the tool manifest to the oldest tag which starts with the |
| # tag prefix. Using the oldest tag ensures that consecutive attempts |
| # to roll a tool to a given package instance will yield the same |
| # diff even if newer tags are added to the instance between |
| # attempts. |
| tags = [t for t in target.tags if t.tag.startswith(tag_prefix)] |
| oldest_tag = min(tags, key=lambda t: t.registered_ts) |
| api.ensure_tool.write_tool_metadata( |
| tool_manifest_path, |
| tool_metadata.path, |
| oldest_tag.tag, |
| do_not_autoroll=tool_metadata.do_not_autoroll, |
| ) |
| updated_tools.append(tool_metadata.path) |
| |
| # TODO(fxbug.dev/84472): Construct a meaningful commit message on a per-tool |
| # basis once this is parallelized. |
| commit_message = "[roll] Update tools to %s" % target_ref |
| if updated_tools: |
| commit_message += "\n\n" + "\n".join("- %s" % t for t in updated_tools) + "\n" |
| |
| # TODO(fxbug.dev/84472): Parallelize roll attempts per tool manifest. The |
| # current behavior combines all tool manifest changes into one CL. |
| change = api.auto_roller.attempt_roll( |
| api.auto_roller.Options( |
| remote=props.remote, |
| dry_run=props.dry_run, |
| ), |
| repo_dir=checkout_root, |
| commit_message=commit_message, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| |
| def GenTests(api): |
| props = api.properties( |
| remote="https://fuchsia.googlesource.com/infra/recipes", |
| ) |
| tool_manifests = api.step_data( |
| "find module tool manifests", |
| api.file.glob_paths( |
| [ |
| "module_a/resources/tool_manifest.json", |
| "module_b/resources/tool_manifest.json", |
| ] |
| ), |
| ) + api.step_data( |
| "find recipe tool manifests", |
| api.file.glob_paths( |
| [ |
| "foo/resources/tool_manifest.json", |
| ] |
| ), |
| ) |
| # module_a should not roll. |
| describe_current_a = api.step_data( |
| "ensure recipe_modules/module_a/resources/tool_manifest is latest.cipd describe path/to/tool", |
| api.cipd.example_describe( |
| package_name="path/to/tool", |
| version="version:pinned-version", |
| test_data_tags=["version:pinned-version"], |
| ), |
| ) |
| describe_target_a = api.step_data( |
| "ensure recipe_modules/module_a/resources/tool_manifest is latest.cipd describe path/to/tool (2)", |
| api.cipd.example_describe( |
| package_name="path/to/tool", |
| version="version:pinned-version", |
| test_data_tags=["version:pinned-version"], |
| ), |
| ) |
| # module_b should roll. |
| describe_current_b = api.step_data( |
| "ensure recipe_modules/module_b/resources/tool_manifest is latest.cipd describe path/to/tool", |
| api.cipd.example_describe( |
| package_name="path/to/tool", |
| version="version:pinned-version", |
| test_data_tags=["version:pinned-version"], |
| ), |
| ) |
| describe_target_b = api.step_data( |
| "ensure recipe_modules/module_b/resources/tool_manifest is latest.cipd describe path/to/tool (2)", |
| api.cipd.example_describe( |
| package_name="path/to/tool", |
| version="version:new-version-to-pin", |
| test_data_tags=["version:new-version-to-pin"], |
| ), |
| ) |
| read_manifest_foo = api.step_data( |
| "read recipes/foo/resources/tool_manifest.read manifest", |
| api.file.read_proto( |
| ToolMetadata( |
| path="path/to/tool", |
| version="version:pinned-version", |
| do_not_autoroll=True, |
| ), |
| ), |
| ) |
| yield ( |
| api.buildbucket_util.test("roll_tool") |
| + tool_manifests |
| + props |
| + describe_current_a |
| + describe_target_a |
| + describe_current_b |
| + describe_target_b |
| + read_manifest_foo |
| + api.auto_roller.success() |
| ) |