| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling flutter and dependencies. |
| |
| Flutter and dependencies need to get rolled based on the version |
| dependencies of another fuchsia prebuilt. In this recipe we are |
| calling the fuchsia prebuilt dictating the dependencies "orchestrator". |
| |
| The orchestrator cipd package contains a json file with the dependencies |
| it was built with. The following is an example of the json file: |
| |
| { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz" |
| } |
| |
| This recipe checks if there is a new version of the orchestrator and |
| if it requires different versions of the dependencies from the ones in |
| the source tree. If both conditions are true then it proceeds to roll |
| new versions of projects and packages as described in the |
| rolling_packages_projects parameter and using the versions from the |
| orchestrator dependencies json file. |
| |
| Warning: If this recipe runs in dry mode it will still roll changes into |
| fuchsia dart 3p repository but the fuchsia tree won't pin to the latest |
| version. This is safe as the fuchsia dart 3p hash will be bumped only on a |
| successful roll of all the other dependencies. |
| """ |
| |
| import copy |
| |
| from recipe_engine.config import List |
| from recipe_engine.post_process import StepSuccess |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/cipd_dependencies", |
| "fuchsia/dart_util", |
| "fuchsia/debug_symbols", |
| "fuchsia/gerrit", |
| "fuchsia/jiri", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Jiri remote manifest project", default=None), |
| "manifests": Property( |
| kind=List(dict), |
| help=( |
| "A list of dictionaries with project, manifest and remote keys " |
| "used to signal jiri wich manifests to synchronize. " |
| 'E.g. [{"project": "integration", "manifest": "other/dependency", ' |
| ' "remote": "sso://fuchsia/integration", ' |
| ' "lock_file": "integration/jiri.lock"}]' |
| ), |
| ), |
| "locks": Property( |
| kind=List(dict), |
| help=( |
| "A list of dictionaries with manifest and files keys used to " |
| "signal jiri which lock files to update. " |
| ' E.g. [{"manifest": "integration/fuchsia/flower", ' |
| ' "file": "integration/fuchsia/jiri.lock"}]' |
| ), |
| ), |
| "orchestrator_package": Property( |
| kind=str, |
| help=( |
| "A string with the CIPD package used to validate " |
| "flutter and dependencies versions. This is a special " |
| "cipd package that contains a json file describing " |
| "the dependencies it was built with." |
| ), |
| ), |
| "orchestrator_ref": Property( |
| kind=str, |
| default="latest", |
| help="A cipd ref used to get a given version of the orchestrator package", |
| ), |
| "orchestrator_import_in": Property( |
| kind=str, |
| default="", |
| help=( |
| "A string with the location where the orchestrator package " |
| "should be imported into the source tree." |
| ), |
| ), |
| "owners": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "The owners responsible for watching this roller " |
| '(example: "username@google.com").' |
| ), |
| ), |
| "rolling_packages_projects": Property( |
| kind=List(dict), |
| default=(), |
| help=( |
| "A list of dictionaries with rolling package/project, type," |
| " and location to import_in. This is used to list the " |
| "dependencies that should be updated: " |
| 'E.g. [{"name": "dart", "type": "package", ' |
| ' "import_in": "fuchsia/prebuilts", ' |
| ' "version_tag": "dart_version"}]' |
| ), |
| ), |
| "versions_file": Property( |
| kind=str, |
| help=( |
| "A string with the path to the versions file inside" |
| "the orchestrator CIPD package." |
| ), |
| ), |
| "validate_against_package": Property( |
| kind=dict, |
| help=( |
| "A dictionary with the package and tag for the version to use" |
| " for validation. A single dependency is used to validate if" |
| ' a roll is needed. E.g. {"package": "flutter/fuchsia", ' |
| ' "version_tag": "flutter_version"} ' |
| "will validate the source tree version of flutter/fuchsia " |
| "with the value of flutter_version from the orchestrator " |
| "json file." |
| ), |
| ), |
| "dry_run": Property( |
| kind=bool, |
| default=False, |
| help="Whether to dry-run the auto-roller (CQ+1 and abandon the change)", |
| ), |
| "debug_symbol_packages": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "A list of strings with the cipd packages containing debug " |
| "symbols and their associated packages." |
| ), |
| ), |
| "debug_symbol_gcs_paths": Property( |
| kind=List(list), |
| default=(), |
| help="GCS bucket and namespace pairs to upload debug symbols to", |
| ), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {roller} packages and projects: |
| |
| {packages} |
| |
| Test: CQ""" |
| |
| |
| def RunSteps( |
| api, |
| project, |
| manifests, |
| locks, |
| orchestrator_ref, |
| owners, |
| versions_file, |
| rolling_packages_projects, |
| orchestrator_package, |
| orchestrator_import_in, |
| validate_against_package, |
| dry_run, |
| debug_symbol_packages, |
| debug_symbol_gcs_paths, |
| ): |
| """Run the recipe steps.""" |
| with api.context(infra_steps=True): |
| if owners: |
| with api.step.nest("owners") as presentation: |
| presentation.step_summary_text = ", ".join(owners) |
| |
| # Import manifests |
| with api.step.nest("jiri-manifests") as presentation: |
| api.jiri.init(use_lock_file=True) |
| presentation.logs["manifests"] = str(manifests) |
| for manifest_config in manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config["manifest"], |
| remote=manifest_config["remote"], |
| name=manifest_config["project"], |
| ) |
| api.jiri.update(run_hooks=False) |
| with api.context(cwd=api.path["start_dir"]): |
| api.jiri.run_hooks() |
| |
| deps_info = api.jiri.package([orchestrator_package]) |
| cipd_description = api.cipd.describe(orchestrator_package, orchestrator_ref) |
| orchestrator_current_version = deps_info.json.output[0]["version"] |
| # Is there a new version to roll? |
| orchestrator_expected_version = cipd_description.tags[0].tag |
| if orchestrator_expected_version == orchestrator_current_version: |
| msg = "%s in tree version is [%s] and latest from cipd is [%s]" % ( |
| orchestrator_package, |
| orchestrator_current_version, |
| orchestrator_expected_version, |
| ) |
| api.step( |
| "manifest up-to-date; nothing to roll", None |
| ).presentation.step_text = msg |
| return api.auto_roller.nothing_to_roll() |
| |
| # Read main package versions file. |
| orchestrator_version_dict = api.cipd_dependencies.get_dependencies( |
| "read versions", |
| orchestrator_package, |
| cipd_description.pin.instance_id, |
| versions_file, |
| ) |
| |
| # Use version of the validation package to decide whether a roll is needed or |
| # not. |
| validation_info = api.jiri.package([validate_against_package["package"]]) |
| local_version = validation_info.json.output[0]["version"] |
| expected_version = ( |
| "git_revision:%s" |
| % orchestrator_version_dict[validate_against_package["version_tag"]] |
| ) |
| if local_version == expected_version: |
| msg = "Expected version [%s] == local fuchsia tree version [%s]" % ( |
| expected_version, |
| local_version, |
| ) |
| api.step("validation_summary", cmd=None).presentation.step_text = msg |
| return api.auto_roller.nothing_to_roll() |
| # Start rolling orchestrator |
| rolling_packages_projects.append( |
| { |
| "type": "package", |
| "name": orchestrator_package, |
| "version_tag": orchestrator_package, |
| "import_in": orchestrator_import_in, |
| } |
| ) |
| orchestrator_version_dict[orchestrator_package] = orchestrator_expected_version |
| # Roll all the dependencies |
| package_msgs = [] |
| package_msg_template = "%s %s rolled from %s to %s" |
| with api.step.nest("edit jiri manifests"): |
| for package_project in rolling_packages_projects: |
| package_msg = "" |
| if package_project["type"] == "package": |
| if package_project["name"] == orchestrator_package: |
| expected_version = orchestrator_version_dict[ |
| package_project["version_tag"] |
| ] |
| else: |
| expected_version = ( |
| "git_revision:%s" |
| % orchestrator_version_dict[package_project["version_tag"]] |
| ) |
| # Save flutter/fuchsia version, as it is the coordinating version |
| # for debug symbol uploads. |
| if package_project["name"] == "flutter/fuchsia": |
| flutter_fuchsia_version = str(expected_version) |
| changes = api.jiri.edit_manifest( |
| package_project["import_in"], |
| packages=[(package_project["name"], expected_version)], |
| name=package_project["name"], |
| ) |
| # Sometimes only a subset of packages are updated. In those cases |
| # the changes list is empty. |
| if changes["packages"]: |
| old_version = changes["packages"][0]["old_version"] |
| package_msg = package_msg_template % ( |
| "package", |
| package_project["name"], |
| old_version, |
| expected_version, |
| ) |
| else: |
| expected_version = orchestrator_version_dict[ |
| package_project["version_tag"] |
| ] |
| changes = api.jiri.edit_manifest( |
| package_project["import_in"], |
| name=package_project["name"], |
| projects=[(package_project["name"], expected_version)], |
| ) |
| # Sometimes only a subset of projects are updated. In those cases |
| # the changes list is empty. |
| if changes["projects"]: |
| old_version = changes["projects"][0]["old_revision"] |
| package_msg = package_msg_template % ( |
| "project", |
| package_project["name"], |
| old_version, |
| expected_version, |
| ) |
| if package_msg: |
| package_msgs.append(package_msg) |
| |
| # Update fuchsia's third-party dart packages (including flutter dependencies), |
| with api.step.nest("third-party dart packages"): |
| checkout_root = api.path["start_dir"] |
| rolled_hash = api.dart_util.update_3p_packages( |
| checkout_root=checkout_root, |
| flutter_revision=orchestrator_version_dict["flutter_version"], |
| ) |
| changes = api.jiri.edit_manifest( |
| "integration/fuchsia/topaz/dart", |
| name="third_party/dart-pkg", |
| projects=[("third_party/dart-pkg", rolled_hash)], |
| ) |
| |
| # Update lock files |
| with api.step.nest("update jiri lock files"): |
| for lock in locks: |
| api.jiri.resolve( |
| local_manifest=True, output=lock["file"], manifests=[lock["manifest"]] |
| ) |
| |
| roller_name = api.buildbucket.builder_name |
| if roller_name.endswith("-roller"): |
| roller_name = api.buildbucket.builder_name[:7] |
| message = COMMIT_MESSAGE.format( |
| roller=roller_name, |
| packages="\n\n".join(package_msgs), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # All the changes are happening inside one of the git cloned projects. |
| # The "project" property points to the location of the project containing |
| # the changes and we need to move to that directory for the roller to pick |
| # up the changes. |
| project_dir = api.path["start_dir"].join(*project.split("/")) |
| with api.context(cwd=project_dir): |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(manifests[0]["remote"]), |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| rolled = change and change.success |
| |
| # If running on dry_run mode, skip building and pushing debug symbols |
| # because the correct dependencies won't be available in the tree. |
| if rolled and debug_symbol_gcs_paths and not dry_run: |
| api.debug_symbols.fetch_and_upload( |
| packages=debug_symbol_packages, |
| version=flutter_fuchsia_version, |
| gcs_paths=debug_symbol_gcs_paths, |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def GenTests(api): |
| """Tests for cipd with dependents roller.""" |
| properties = { |
| "project": "integration", |
| "manifests": [ |
| { |
| "project": "integration", |
| "manifest": "other/dependency", |
| "remote": "sso://fuchsia/integration", |
| "lock_file": "integration/jiri.lock", |
| }, |
| { |
| "project": "integration", |
| "manifest": "fuchsia/flower", |
| "remote": "sso://fuchsia/integration", |
| "lock_file": "integration/fuchsia/jiri.lock", |
| }, |
| ], |
| "locks": [{"manifest": "a/b", "file": "a/b/jiri.lock"}], |
| "orchestrator_package": "fuchsia/orchestrator", |
| "orchestrator_import_in": "fuchsia/prebuilts", |
| "validate_against_package": { |
| "package": "flutter/fuchsia", |
| "version_tag": "flutter_version", |
| }, |
| "rolling_packages_projects": [ |
| { |
| "name": "dart/sdk", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "dart_version", |
| }, |
| { |
| "name": "skia", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "skia_version", |
| }, |
| { |
| "name": "flutter/fuchsia", |
| "type": "package", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "engine_version", |
| }, |
| ], |
| "versions_file": "flutter/versions.json", |
| "owners": ["abc@gmail.com"], |
| "dry_run": True, |
| "debug_symbol_packages": ["flutter/fuchsia-debug-symbols-x64"], |
| "debug_symbol_gcs_paths": [["fuchsia-debug-symbols-shortlived", ""]], |
| } |
| |
| package_no_match_test_data = api.step_data( |
| "cipd describe fuchsia/orchestrator", |
| api.cipd.example_describe( |
| package_name="fuchsia/orchestrator", |
| version="version_abc", |
| test_data_tags=["git_revision:revision_abc"], |
| ), |
| ) |
| jiri_package_test_data = [ |
| { |
| "name": "fuchsia/orchestrator", |
| "path": "fuchsia/orchestrator", |
| "version": "git_revision:revision_abc", |
| "manifest": "manifest1", |
| }, |
| ] |
| default_properties = api.properties(**properties) |
| # Version of the cipd orchestrator package is the same as the one |
| # in the tree. |
| yield ( |
| api.test("nothing_to_roll") |
| + default_properties |
| + package_no_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| ) |
| |
| # Version of the cipd orchestrator package is different from the one |
| # in the tree and ready to get rolled but flutter version is the same as the |
| # one in the tree. |
| package_match_test_data = api.step_data( |
| "cipd describe fuchsia/orchestrator", |
| api.cipd.example_describe( |
| package_name="fuchsia/orchestrator", |
| version="version_abc", |
| test_data_tags=["git_revision:revision_jkl"], |
| ), |
| ) |
| package_versions = { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz", |
| } |
| jiri_flutter_package_test_data = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "flutter/fuchsia", |
| "version": "git_revision:flutter_version_xyz", |
| "manifest": "manifest1", |
| }, |
| ] |
| yield ( |
| api.test("orchestrator_dep_version_same_as_tree") |
| + default_properties |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + package_match_test_data |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(jiri_flutter_package_test_data) |
| ) |
| + api.post_process(StepSuccess, "validation_summary") |
| ) |
| |
| # Version of the validation cipd package is different from the one in the tree |
| # and ready to get rolled along its dependencies. |
| jiri_flutter_package_test_data_no_match = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "flutter/fuchsia", |
| "version": "git_revision:flutter_version_xyz1", |
| "manifest": "manifest1", |
| }, |
| ] |
| yield ( |
| api.test("orchestrator_dep_version_different_from_tree_dry_run") |
| + default_properties |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data( |
| "jiri package (2)", |
| api.jiri.package(jiri_flutter_package_test_data_no_match), |
| ) |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| + api.auto_roller.dry_run_success( |
| name="third-party dart packages.check for completion.check if done ({})" |
| ) |
| ) |
| |
| # Validate the build and upload debug symbols path. |
| props = copy.deepcopy(properties) |
| props["dry_run"] = False |
| no_dry_run_props = api.properties(**props) |
| yield ( |
| api.test("orchestrator_dep_version_different_from_tree") |
| + no_dry_run_props |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data( |
| "jiri package (2)", |
| api.jiri.package(jiri_flutter_package_test_data_no_match), |
| ) |
| + api.auto_roller.success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| + api.auto_roller.success( |
| name="third-party dart packages.check for completion.check if done (0)" |
| ) |
| ) |