| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling CIPD prebuilts which depend on other prebuilts. |
| |
| Version dependencies are defined in a file called versions.json on the |
| root folder of the CIPD package. This recipe reads the dependencies and |
| validate the correct versions are already in the tree. If all the |
| dependencies are satisfied then the new CIPD package is rolled if not it |
| will just present a message notifying is waiting for a given version. |
| """ |
| import copy |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/cipd_dependencies", |
| "fuchsia/gerrit", |
| "fuchsia/jiri", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Jiri remote manifest project", default=None), |
| "manifests": Property( |
| kind=List(dict), |
| help=("A list of dictionaries with project, manifest and remote key."), |
| ), |
| "import_in": Property( |
| kind=str, help="Path to the manifest to edit relative to $project" |
| ), |
| "package": Property( |
| kind=str, help="The list of CIPD packages to update in $import_in" |
| ), |
| "lockfiles": Property( |
| kind=List(str), |
| default=(), |
| help='The list of lockfiles to update in "${manifest}=${lockfile}" format', |
| ), |
| "dry_run": Property( |
| kind=bool, |
| default=False, |
| help="Whether to dry-run the auto-roller (CQ+1 and abandon the change)", |
| ), |
| "ref": Property( |
| kind=str, |
| default="latest", |
| help="A common CIPD ref to resolve when rolling a set of packages", |
| ), |
| "owners": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "The owners responsible for watching this roller " |
| '(example: "username@google.com").' |
| ), |
| ), |
| "package_dependencies": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "List of package dependencies in the form of " |
| "package_name=package_key where package_name is the " |
| "cipd package name and package_key is the key in " |
| "versions file with the expected version for the " |
| "dependency cipd package." |
| ), |
| ), |
| "project_dependencies": Property( |
| kind=List(str), |
| default=(), |
| help="List of project dependencies in the form of project_name=package_key", |
| ), |
| "version_file": Property( |
| kind=str, |
| help=( |
| "A string with the path to the versions file inside " "the CIPD package." |
| ), |
| ), |
| } |
| |
| COMMIT_MESSAGE_TITLE = """{prepend}[{type}] {type_descr} {roller} CIPD packages:""" |
| |
| COMMIT_MESSAGE_DO_NOT_SUBMIT = "DO NOT SUBMIT " |
| |
| COMMIT_MESSAGE = """ |
| |
| {packages} |
| |
| From: {old_version} |
| To: {version} |
| |
| Test: CQ |
| """ |
| |
| CIPD_URL = "https://chrome-infra-packages.appspot.com/p/{package}/+/{version}" |
| |
| |
| def RunSteps( |
| api, |
| manifests, |
| import_in, |
| package, |
| lockfiles, |
| dry_run, |
| ref, |
| owners, |
| package_dependencies, |
| version_file, |
| project_dependencies, |
| project, |
| ): |
| with api.context(infra_steps=True): |
| gerrit_host = api.gerrit.host_from_remote_url(manifests[0]["remote"]) |
| if owners: |
| with api.step.nest("owners") as owners_presentation: |
| owners_presentation.step_summary_text = ", ".join(owners) |
| |
| api.jiri.init(use_lock_file=True) |
| for manifest_config in manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config["manifest"], |
| remote=manifest_config["remote"], |
| name=manifest_config["project"], |
| ) |
| api.jiri.update(run_hooks=False) |
| with api.context(cwd=api.path["start_dir"]): |
| api.jiri.run_hooks() |
| deps_info = api.jiri.package([package]) |
| cipd_description = api.cipd.describe(package, ref) |
| |
| # Is there a new version to roll? |
| if cipd_description.tags[0].tag == deps_info.json.output[0]["version"]: |
| api.step("manifest up-to-date; nothing to roll", None) |
| return |
| |
| version = cipd_description.tags[0].tag |
| # We need to roll the new version. |
| versions_json = api.cipd_dependencies.get_dependencies( |
| "read versions", package, cipd_description.pin.instance_id, version_file |
| ) |
| summary = api.cipd_dependencies.validate_against_tree( |
| package=package, |
| dependencies=package_dependencies, |
| versions_dict=versions_json, |
| ) |
| if summary: |
| api.step("package dependencies not satisfied: {}".format(summary), None) |
| return |
| # Verify source dependencies are satisfied. |
| summary = api.cipd_dependencies.validate_against_tree( |
| package, project_dependencies, versions_json, is_package=False |
| ) |
| if summary: |
| api.step("project dependencies not satisfied: {}".format(summary), None) |
| return |
| |
| changes = api.jiri.edit_manifest(import_in, packages=[(package, version)]) |
| old_version = changes["packages"][0]["old_version"] |
| exact_packages = set() |
| |
| # Update the lockfiles. |
| for lock_entry in lockfiles: |
| fields = lock_entry.split("=") |
| manifest = fields[0] |
| lock = fields[1] |
| resolve_output = api.jiri.resolve( |
| local_manifest=True, output=lock, manifests=[manifest] |
| ).stdout |
| platform_pkgs = get_platform_specific_packages(package, resolve_output) |
| exact_packages = exact_packages.union(platform_pkgs) |
| exact_packages.add(package) |
| |
| exact_packages = sorted(exact_packages) |
| packages_with_urls = append_urls(exact_packages, old_version, version) |
| |
| message = generate_message( |
| builder_name=api.buildbucket.builder_name, |
| packages="\n".join(packages_with_urls), |
| old_version=old_version, |
| version=version, |
| build_id=api.buildbucket_util.id, |
| dry_run=dry_run, |
| ) |
| |
| # Land the changes. |
| project_dir = api.path["start_dir"].join(*project.split("/")) |
| with api.context(cwd=project_dir): |
| change = api.auto_roller.attempt_roll( |
| gerrit_host, |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| |
| def get_platform_specific_packages(package, output): |
| platform_regex = "(?<=" + package.replace("${platform}", r"\${platform=).*(?=})") |
| if not package.endswith("${platform}"): |
| return [package] |
| pattern = re.compile(platform_regex) |
| match = pattern.search(output) |
| if match: |
| platforms = match.group(0).split(",") |
| return [package.replace("${platform}", platform) for platform in platforms] |
| |
| |
| def append_urls(packages, old_version, new_version): |
| package_line = "{package} old:{old} new:{new}" |
| packages_with_urls = [] |
| for package in packages: |
| if "${platform}" in package: |
| packages_with_urls.append(package) |
| else: |
| packages_with_urls.append( |
| package_line.format( |
| old=CIPD_URL.format(package=package, version=old_version), |
| new=CIPD_URL.format(package=package, version=new_version), |
| package=package, |
| ) |
| ) |
| return packages_with_urls |
| |
| |
| def generate_message(builder_name, packages, old_version, version, build_id, dry_run): |
| roller_string = builder_name.replace("-roller", "").replace("-dryrun", "") |
| |
| if dry_run: |
| message_title = COMMIT_MESSAGE_TITLE.format( |
| prepend=COMMIT_MESSAGE_DO_NOT_SUBMIT, |
| type="dryrun", |
| type_descr="Dry run", |
| roller=roller_string, |
| ) |
| else: |
| message_title = COMMIT_MESSAGE_TITLE.format( |
| prepend="", |
| type="roll", |
| type_descr="Roll", |
| roller=roller_string, |
| ) |
| |
| message_body = COMMIT_MESSAGE.format( |
| roller=roller_string, |
| packages=packages, |
| old_version=old_version, |
| version=version, |
| builder=builder_name, |
| build_id=build_id, |
| ) |
| |
| return "".join([message_title, message_body]) |
| |
| |
| def GenTests(api): |
| properties = { |
| "project": "integration", |
| "manifests": [ |
| { |
| "project": "integration", |
| "manifest": "other/dependency", |
| "remote": "sso://fuchsia/integration", |
| }, |
| { |
| "project": "integration", |
| "manifest": "fuchsia/flower", |
| "remote": "sso://fuchsia/integration", |
| }, |
| ], |
| "import_in": "fuchsia/prebuilts", |
| "package": "flutter/dependent/${platform}", |
| "lockfiles": ["integration/flower=integration/jiri.lock"], |
| "version_file": "flutter/versions.json", |
| "package_dependencies": ["flutter/fuchsia=engine_version"], |
| "project_dependencies": ["dart/sdk=dart_version"], |
| "owners": ["abc@google.com"], |
| "dry_run": False, |
| } |
| flutter_dependent_no_match_test_data = api.step_data( |
| "cipd describe flutter/dependent/${platform}", |
| api.cipd.example_describe( |
| package_name="flutter/dependent", |
| version="dependent_version_abc", |
| test_data_tags=["git_revision:dependent_revision_jkl"], |
| ), |
| ) |
| package_test_data = [ |
| { |
| "name": "flutter/dependent", |
| "path": "path/flutter/dependent", |
| "version": "git_revision:dependent_revision_jkl", |
| "manifest": "manifest1", |
| }, |
| ] |
| default_properties = api.properties(**properties) |
| |
| # Dry run specific properties |
| properties_no_dryrun = copy.deepcopy(properties) |
| properties_no_dryrun["dry_run"] = True |
| default_properties_dryrun = api.properties(**properties_no_dryrun) |
| |
| yield ( |
| api.status_check.test("nothing_to_roll") |
| + default_properties |
| + flutter_dependent_no_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| ) |
| |
| yield ( |
| api.status_check.test("nothing_to_roll_dryrun") |
| + default_properties_dryrun |
| + flutter_dependent_no_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| ) |
| |
| flutter_dependent_match_test_data = api.step_data( |
| "cipd describe flutter/dependent/${platform}", |
| api.cipd.example_describe( |
| package_name="flutter/dependent", |
| version="dependent_version_def", |
| test_data_tags=["git_revision:dependent_revision_lmn"], |
| ), |
| ) |
| versions_json_deps_not_satisfied = { |
| "engine_version": "engine_version_xyz", |
| } |
| dependencies_package_test_data = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "path/flutter/fuchsia", |
| "version": "git_revision:engine_revision_abc", |
| "manifest": "manifest1", |
| }, |
| ] |
| yield ( |
| api.status_check.test("package_dependencies_not_satisfied") |
| + default_properties |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "read versions", |
| api.file.read_json(json_content=versions_json_deps_not_satisfied), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("package_dependencies_not_satisfied_dryrun") |
| + default_properties_dryrun |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "read versions", |
| api.file.read_json(json_content=versions_json_deps_not_satisfied), |
| ) |
| ) |
| |
| deps_satisfied = { |
| "engine_version": "engine_revision_abc", |
| "dart_version": "dart_version_abc", |
| } |
| project_test_data = [ |
| { |
| "name": "dart/sdk", |
| "path": "path/fuchsia/dart/sdk", |
| "revision": "dart_version_abc", |
| "manifest": "manifest1", |
| } |
| ] |
| |
| # Updating properties |
| properties_no_platform = copy.deepcopy(properties) |
| properties_no_platform["package"] = "flutter/dependent" |
| default_properties = api.properties(**properties_no_platform) |
| |
| flutter_dependent_match_test_data_no_platform = api.step_data( |
| "cipd describe flutter/dependent", |
| api.cipd.example_describe( |
| package_name="flutter/dependent", |
| version="dependent_version_def", |
| test_data_tags=["git_revision:dependent_revision_lmn"], |
| ), |
| ) |
| yield ( |
| api.status_check.test("dependencies_satisfied_package_with_platform") |
| + default_properties |
| + flutter_dependent_match_test_data_no_platform |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri resolve", stdout=api.jiri.example_resolve_data("flutter/dependent") |
| ) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data("jiri project", api.jiri.project(project_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| + api.auto_roller.success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependent-roller") |
| ) |
| ) |
| |
| # Setting properties back to default |
| default_properties = api.properties(**properties) |
| |
| yield ( |
| api.status_check.test("dependencies_satisfied") |
| + default_properties_dryrun |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri resolve", |
| stdout=api.jiri.example_resolve_data("flutter/dependent/${platform}"), |
| ) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data("jiri project", api.jiri.project(project_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| + api.auto_roller.dry_run_success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependent-roller") |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("dependencies_satisfied_dryrun") |
| + default_properties_dryrun |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri resolve", |
| stdout=api.jiri.example_resolve_data("flutter/dependent/${platform}"), |
| ) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data("jiri project", api.jiri.project(project_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| + api.auto_roller.dry_run_success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependent-roller") |
| ) |
| ) |
| |
| project_test_data_not_satisfied = [ |
| { |
| "name": "dart/sdk", |
| "path": "path/fuchsia/dart/sdk", |
| "revision": "dart_version_xyz", |
| "manifest": "manifest1", |
| } |
| ] |
| yield ( |
| api.status_check.test("project_dependencies_not_satisfied") |
| + default_properties |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "jiri project", api.jiri.project(project_test_data_not_satisfied) |
| ) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("project_dependencies_not_satisfied_dryrun") |
| + default_properties_dryrun |
| + flutter_dependent_match_test_data |
| + api.step_data("jiri package", api.jiri.package(package_test_data)) |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(dependencies_package_test_data) |
| ) |
| + api.step_data( |
| "jiri project", api.jiri.project(project_test_data_not_satisfied) |
| ) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=deps_satisfied) |
| ) |
| ) |