| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling CIPD prebuilts which depend on other prebuilts. |
| |
| Version dependencies are defined in a file called versions.json on the |
| root folder of the CIPD package. This recipe reads the dependencies and |
| validate the correct versions are already in the tree. If all the |
| dependencies are satisfied then the new CIPD package is rolled if not it |
| will just present a message notifying is waiting for a given version. |
| """ |
| |
| import re |
| |
| from PB.recipes.fuchsia.contrib.cipd_with_dependencies_roller import InputProperties |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/cipd_dependencies", |
| "fuchsia/gerrit", |
| "fuchsia/jiri", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| COMMIT_MESSAGE_TITLE = """{prepend}[{type}] {type_descr} {roller} CIPD packages:""" |
| |
| COMMIT_MESSAGE_DO_NOT_SUBMIT = "DO NOT SUBMIT " |
| |
| COMMIT_MESSAGE = """ |
| |
| {packages} |
| |
| From: {old_version} |
| To: {version} |
| |
| Test: CQ |
| """ |
| |
| CIPD_URL = "https://chrome-infra-packages.appspot.com/p/{package}/+/{version}" |
| |
| |
| def RunSteps(api, props): # pylint: disable=inconsistent-return-statements |
| ref = props.ref or "latest" |
| with api.context(infra_steps=True): |
| gerrit_host = api.gerrit.host_from_remote_url(props.manifests[0]["remote"]) |
| if props.owners: |
| with api.step.nest("owners") as owners_presentation: |
| owners_presentation.step_summary_text = ", ".join(props.owners) |
| |
| api.jiri.init(use_lock_file=True) |
| for manifest_config in props.manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config["manifest"], |
| remote=manifest_config["remote"], |
| name=manifest_config["project"], |
| ) |
| api.jiri.update(run_hooks=False) |
| with api.context(cwd=api.path["start_dir"]): |
| api.jiri.run_hooks() |
| deps_info = api.jiri.package([props.package]) |
| cipd_description = api.cipd.describe(props.package, ref) |
| |
| # Is there a new version to roll? |
| if cipd_description.tags[0].tag == deps_info.json.output[0]["version"]: |
| api.step.empty("manifest up-to-date; nothing to roll") |
| return |
| |
| version = cipd_description.tags[0].tag |
| # We need to roll the new version. |
| versions_json = api.cipd_dependencies.get_dependencies( |
| "read versions", |
| props.package, |
| cipd_description.pin.instance_id, |
| props.version_file, |
| ) |
| summary = api.cipd_dependencies.validate_against_tree( |
| package=props.package, |
| dependencies=props.package_dependencies, |
| versions_dict=versions_json, |
| ) |
| if summary: |
| api.step.empty("package dependencies not satisfied: {}".format(summary)) |
| return |
| # Verify source dependencies are satisfied. |
| summary = api.cipd_dependencies.validate_against_tree( |
| props.package, props.project_dependencies, versions_json, is_package=False |
| ) |
| if summary: |
| api.step.empty("project dependencies not satisfied: {}".format(summary)) |
| return |
| |
| changes = api.jiri.edit_manifest( |
| props.import_in, packages=[(props.package, version)] |
| ) |
| old_version = changes["packages"][0]["old_version"] |
| exact_packages = set() |
| |
| # Update the lockfiles. |
| for lock_entry in props.lockfiles: |
| fields = lock_entry.split("=") |
| manifest = fields[0] |
| lock = fields[1] |
| resolve_output = api.jiri.resolve( |
| local_manifest=True, output=lock, manifests=[manifest] |
| ).stdout |
| platform_pkgs = get_platform_specific_packages( |
| props.package, resolve_output |
| ) |
| exact_packages = exact_packages.union(platform_pkgs) |
| exact_packages.add(props.package) |
| |
| exact_packages = sorted(exact_packages) |
| packages_with_urls = append_urls(exact_packages, old_version, version) |
| |
| message = generate_message( |
| builder_name=api.buildbucket.builder_name, |
| packages="\n".join(packages_with_urls), |
| old_version=old_version, |
| version=version, |
| build_id=api.buildbucket_util.id, |
| dry_run=props.dry_run, |
| ) |
| |
| # Land the changes. |
| project_dir = api.path["start_dir"].join(*props.project.split("/")) |
| with api.context(cwd=project_dir): |
| change = api.auto_roller.attempt_roll( |
| gerrit_host, |
| gerrit_project=props.project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=props.dry_run, |
| roller_owners=props.owners, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| |
| def get_platform_specific_packages(package, output): |
| platform_regex = "(?<=" + package.replace("${platform}", r"\${platform=).*(?=})") |
| if not package.endswith("${platform}"): |
| return [package] |
| pattern = re.compile(platform_regex) |
| match = pattern.search(output) |
| if match: |
| platforms = match.group(0).split(",") |
| return [package.replace("${platform}", platform) for platform in platforms] |
| |
| |
| def append_urls(packages, old_version, new_version): |
| package_line = "{package} old:{old} new:{new}" |
| packages_with_urls = [] |
| for package in packages: |
| if "${platform}" in package: |
| packages_with_urls.append(package) |
| else: |
| packages_with_urls.append( |
| package_line.format( |
| old=CIPD_URL.format(package=package, version=old_version), |
| new=CIPD_URL.format(package=package, version=new_version), |
| package=package, |
| ) |
| ) |
| return packages_with_urls |
| |
| |
| def generate_message(builder_name, packages, old_version, version, build_id, dry_run): |
| roller_string = builder_name.replace("-roller", "").replace("-dryrun", "") |
| |
| if dry_run: |
| message_title = COMMIT_MESSAGE_TITLE.format( |
| prepend=COMMIT_MESSAGE_DO_NOT_SUBMIT, |
| type="dryrun", |
| type_descr="Dry run", |
| roller=roller_string, |
| ) |
| else: |
| message_title = COMMIT_MESSAGE_TITLE.format( |
| prepend="", |
| type="roll", |
| type_descr="Roll", |
| roller=roller_string, |
| ) |
| |
| message_body = COMMIT_MESSAGE.format( |
| roller=roller_string, |
| packages=packages, |
| old_version=old_version, |
| version=version, |
| builder=builder_name, |
| build_id=build_id, |
| ) |
| |
| return "".join([message_title, message_body]) |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "project": "integration", |
| "manifests": [ |
| { |
| "project": "integration", |
| "manifest": "other/dependency", |
| "remote": "sso://fuchsia/integration", |
| }, |
| { |
| "project": "integration", |
| "manifest": "fuchsia/flower", |
| "remote": "sso://fuchsia/integration", |
| }, |
| ], |
| "import_in": "fuchsia/prebuilts", |
| "package": "flutter/dependent/${platform}", |
| "lockfiles": ["integration/flower=integration/jiri.lock"], |
| "version_file": "flutter/versions.json", |
| "package_dependencies": ["flutter/fuchsia=engine_version"], |
| "project_dependencies": ["dart/sdk=dart_version"], |
| "owners": ["abc@google.com"], |
| "dry_run": False, |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| flutter_dependent_no_match_test_data = api.step_data( |
| "cipd describe flutter/dependent/${platform}", |
| api.cipd.example_describe( |
| package_name="flutter/dependent", |
| version="dependent_version_abc", |
| test_data_tags=["git_revision:dependent_revision_jkl"], |
| ), |
| ) |
| package_test_data = [ |
| { |
| "name": "flutter/dependent", |
| "path": "path/flutter/dependent", |
| "version": "git_revision:dependent_revision_jkl", |
| "manifest": "manifest1", |
| }, |
| ] |
| |
| yield ( |
| api.buildbucket_util.test("nothing_to_roll") |
| + properties() |
| + flutter_dependent_no_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("nothing_to_roll_dryrun") |
| + properties(dry_run=True) |
| + flutter_dependent_no_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| ) |
| |
| flutter_dependent_match_test_data = api.step_data( |
| "cipd describe flutter/dependent/${platform}", |
| api.cipd.example_describe( |
| package_name="flutter/dependent", |
| version="dependent_version_def", |
| test_data_tags=["git_revision:dependent_revision_lmn"], |
| ), |
| ) |
| versions_json_deps_not_satisfied = { |
| "engine_version": "engine_version_xyz", |
| } |
| dependencies_package_test_data = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "path/flutter/fuchsia", |
| "version": "git_revision:engine_revision_abc", |
| "manifest": "manifest1", |
| }, |
| ] |
| yield ( |
| api.buildbucket_util.test("package_dependencies_not_satisfied") |
| + properties() |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "read versions", |
| api.file.read_json(json_content=versions_json_deps_not_satisfied), |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("package_dependencies_not_satisfied_dryrun") |
| + properties(dry_run=True) |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "read versions", |
| api.file.read_json(json_content=versions_json_deps_not_satisfied), |
| ) |
| ) |
| |
| deps_satisfied = { |
| "engine_version": "engine_revision_abc", |
| "dart_version": "dart_version_abc", |
| } |
| project_test_data = [ |
| { |
| "name": "dart/sdk", |
| "path": "path/fuchsia/dart/sdk", |
| "revision": "dart_version_abc", |
| "manifest": "manifest1", |
| } |
| ] |
| |
| flutter_dependent_match_test_data_no_platform = api.step_data( |
| "cipd describe flutter/dependent", |
| api.cipd.example_describe( |
| package_name="flutter/dependent", |
| version="dependent_version_def", |
| test_data_tags=["git_revision:dependent_revision_lmn"], |
| ), |
| ) |
| yield ( |
| api.buildbucket_util.test( |
| "dependencies_satisfied_package_with_platform", |
| builder="flutter-dependent-roller", |
| ) |
| + properties(package="flutter/dependent") |
| + flutter_dependent_match_test_data_no_platform |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri resolve", stdout=api.jiri.example_resolve_data("flutter/dependent") |
| ) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data("jiri project", api.jiri.project(project_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "dependencies_satisfied", builder="flutter-dependent-roller" |
| ) |
| + properties(dry_run=True) |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri resolve", |
| stdout=api.jiri.example_resolve_data("flutter/dependent/${platform}"), |
| ) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data("jiri project", api.jiri.project(project_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "dependencies_satisfied_dryrun", builder="flutter-dependent-roller" |
| ) |
| + properties(dry_run=True) |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri resolve", |
| stdout=api.jiri.example_resolve_data("flutter/dependent/${platform}"), |
| ) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data("jiri project", api.jiri.project(project_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| project_test_data_not_satisfied = [ |
| { |
| "name": "dart/sdk", |
| "path": "path/fuchsia/dart/sdk", |
| "revision": "dart_version_xyz", |
| "manifest": "manifest1", |
| } |
| ] |
| yield ( |
| api.buildbucket_util.test("project_dependencies_not_satisfied") |
| + properties() |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "jiri project", api.jiri.project(project_test_data_not_satisfied) |
| ) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("project_dependencies_not_satisfied_dryrun") |
| + properties(dry_run=True) |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "jiri project", api.jiri.project(project_test_data_not_satisfied) |
| ) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| ) |
