| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling flutter and dependencies. |
| |
| Flutter and dependencies need to get rolled based on the version |
| dependencies of another fuchsia prebuilt. In this recipe we are |
| calling the fuchsia prebuilt dictating the dependencies "orchestrator". |
| |
| The orchestrator cipd package contains a json file with the dependencies |
| it was built with. The following is an example of the json file: |
| |
| { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz" |
| } |
| |
| This recipe checks if there is a new version of the orchestrator and |
| if it requires different versions of the dependencies from the ones in |
| the source tree. If both conditions are true then it proceeds to roll |
| new versions of projects and packages as described in the |
| rolling_packages_projects parameter and using the versions from the |
| orchestrator dependencies json file. |
| |
| Warning: If this recipe runs in dry mode it will still roll changes into |
| fuchsia dart 3p repository but the fuchsia tree won't pin to the latest |
| version. This is safe as the fuchsia dart 3p hash will be bumped only on a |
| successful roll of all the other dependencies. |
| """ |
| |
| import copy |
| |
| from recipe_engine.config import List |
| from recipe_engine.post_process import StepSuccess |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/cipd_dependencies", |
| "fuchsia/debug_symbols", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "fuchsia/jiri", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/python", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Jiri remote manifest project", default=None), |
| "manifests": Property( |
| kind=List(dict), |
| help=( |
| "A list of dictionaries with project, manifest and remote keys " |
| "used to signal jiri wich manifests to synchronize. " |
| 'E.g. [{"project": "integration", "manifest": "other/dependency", ' |
| ' "remote": "sso://fuchsia/integration", ' |
| ' "lock_file": "integration/jiri.lock"}]' |
| ), |
| ), |
| "locks": Property( |
| kind=List(dict), |
| help=( |
| "A list of dictionaries with manifest and files keys used to " |
| "signal jiri which lock files to update. " |
| ' E.g. [{"manifest": "integration/fuchsia/flower", ' |
| ' "file": "integration/fuchsia/jiri.lock"}]' |
| ), |
| ), |
| "orchestrator_package": Property( |
| kind=str, |
| help=( |
| "A string with the CIPD package used to validate " |
| "flutter and dependencies versions. This is a special " |
| "cipd package that contains a json file describing " |
| "the dependencies it was built with." |
| ), |
| ), |
| "orchestrator_ref": Property( |
| kind=str, |
| default="latest", |
| help="A cipd ref used to get a given version of the orchestrator package", |
| ), |
| "orchestrator_import_in": Property( |
| kind=str, |
| default="", |
| help=( |
| "A string with the location where the orchestrator package " |
| "should be imported into the source tree." |
| ), |
| ), |
| "owners": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "The owners responsible for watching this roller " |
| '(example: "username@google.com").' |
| ), |
| ), |
| "rolling_packages_projects": Property( |
| kind=List(dict), |
| default=(), |
| help=( |
| "A list of dictionaries with rolling package/project, type," |
| " and location to import_in. This is used to list the " |
| "dependencies that should be updated: " |
| 'E.g. [{"name": "dart", "type": "package", ' |
| ' "import_in": "fuchsia/prebuilts", ' |
| ' "version_tag": "dart_version"}]' |
| ), |
| ), |
| "versions_file": Property( |
| kind=str, |
| help=( |
| "A string with the path to the versions file inside" |
| "the orchestrator CIPD package." |
| ), |
| ), |
| "validate_against_package": Property( |
| kind=dict, |
| help=( |
| "A dictionary with the package and tag for the version to use" |
| " for validation. A single dependency is used to validate if" |
| ' a roll is needed. E.g. {"package": "flutter/fuchsia", ' |
| ' "version_tag": "flutter_version"} ' |
| "will validate the source tree version of flutter/fuchsia " |
| "with the value of flutter_version from the orchestrator " |
| "json file." |
| ), |
| ), |
| "dry_run": Property( |
| kind=bool, |
| default=False, |
| help="Whether to dry-run the auto-roller (CQ+1 and abandon the change)", |
| ), |
| "debug_symbol_packages": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "A list of strings with the cipd packages containing debug " |
| "symbols and their associated packages." |
| ), |
| ), |
| "debug_symbol_gcs_buckets": Property( |
| kind=List(str), default=(), help="GCS buckets to upload debug symbols to", |
| ), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {roller} packages and projects: |
| |
| {packages} |
| |
| Test: CQ""" |
| |
| |
| def RunSteps( |
| api, |
| project, |
| manifests, |
| locks, |
| orchestrator_ref, |
| owners, |
| versions_file, |
| rolling_packages_projects, |
| orchestrator_package, |
| orchestrator_import_in, |
| validate_against_package, |
| dry_run, |
| debug_symbol_packages, |
| debug_symbol_gcs_buckets, |
| ): |
| with api.context(infra_steps=True): |
| if owners: |
| with api.step.nest("owners") as presentation: |
| presentation.step_summary_text = ", ".join(owners) |
| |
| # Import manifests |
| with api.step.nest("jiri-manifests") as presentation: |
| api.jiri.init(use_lock_file=True) |
| presentation.logs["manifests"] = str(manifests) |
| for manifest_config in manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config["manifest"], |
| remote=manifest_config["remote"], |
| name=manifest_config["project"], |
| ) |
| api.jiri.update(run_hooks=False) |
| with api.context(cwd=api.path["start_dir"]): |
| api.jiri.run_hooks() |
| |
| deps_info = api.jiri.package([orchestrator_package]) |
| cipd_description = api.cipd.describe(orchestrator_package, orchestrator_ref) |
| orchestrator_current_version = deps_info.json.output[0]["version"] |
| # Is there a new version to roll? |
| orchestrator_expected_version = cipd_description.tags[0].tag |
| if orchestrator_expected_version == orchestrator_current_version: |
| msg = "%s in tree version is [%s] and latest from cipd is [%s]" % ( |
| orchestrator_package, |
| orchestrator_current_version, |
| orchestrator_expected_version, |
| ) |
| api.step( |
| "manifest up-to-date; nothing to roll", None |
| ).presentation.step_text = msg |
| return api.auto_roller.nothing_to_roll() |
| |
| # Read main package versions file. |
| orchestrator_version_dict = api.cipd_dependencies.get_dependencies( |
| "read versions", |
| orchestrator_package, |
| cipd_description.pin.instance_id, |
| versions_file, |
| ) |
| |
| # Use version of the validation package to decide whether a roll is needed or |
| # not. |
| validation_info = api.jiri.package([validate_against_package["package"]]) |
| local_version = validation_info.json.output[0]["version"] |
| expected_version = ( |
| "git_revision:%s" |
| % orchestrator_version_dict[validate_against_package["version_tag"]] |
| ) |
| if local_version == expected_version: |
| msg = "Expected version [%s] == local fuchsia tree version [%s]" % ( |
| expected_version, |
| local_version, |
| ) |
| api.step("validation summary", cmd=None).presentation.step_text = msg |
| return api.auto_roller.nothing_to_roll() |
| # Start rolling orchestrator |
| rolling_packages_projects.append( |
| { |
| "type": "package", |
| "name": orchestrator_package, |
| "version_tag": orchestrator_package, |
| "import_in": orchestrator_import_in, |
| } |
| ) |
| orchestrator_version_dict[orchestrator_package] = orchestrator_expected_version |
| # Roll all the dependencies |
| package_msgs = [] |
| flutter_fuchsia_version = None |
| package_msg_template = "%s %s rolled from %s to %s" |
| with api.step.nest("edit jiri manifests"): |
| for package_project in rolling_packages_projects: |
| package_msg = "" |
| if package_project["type"] == "package": |
| if package_project["name"] == orchestrator_package: |
| expected_version = orchestrator_version_dict[ |
| package_project["version_tag"] |
| ] |
| else: |
| expected_version = ( |
| "git_revision:%s" |
| % orchestrator_version_dict[package_project["version_tag"]] |
| ) |
| # Save flutter/fuchsia version, as it is the coordinating version |
| # for debug symbol uploads. |
| if package_project["name"] == "flutter/fuchsia": |
| flutter_fuchsia_version = str(expected_version) |
| changes = api.jiri.edit_manifest( |
| package_project["import_in"], |
| packages=[(package_project["name"], expected_version)], |
| name=package_project["name"], |
| ) |
| # Sometimes only a subset of packages are updated. In those cases |
| # the changes list is empty. |
| if changes["packages"]: |
| old_version = changes["packages"][0]["old_version"] |
| package_msg = package_msg_template % ( |
| "package", |
| package_project["name"], |
| old_version, |
| expected_version, |
| ) |
| else: |
| expected_version = orchestrator_version_dict[ |
| package_project["version_tag"] |
| ] |
| changes = api.jiri.edit_manifest( |
| package_project["import_in"], |
| name=package_project["name"], |
| projects=[(package_project["name"], expected_version)], |
| ) |
| # Sometimes only a subset of projects are updated. In those cases |
| # the changes list is empty. |
| if changes["projects"]: |
| old_version = changes["projects"][0]["old_revision"] |
| package_msg = package_msg_template % ( |
| "project", |
| package_project["name"], |
| old_version, |
| expected_version, |
| ) |
| if package_msg: |
| package_msgs.append(package_msg) |
| |
| # Update fuchsia's third-party dart packages (including flutter dependencies), |
| # only if we are rolling a new version of dart-sdk. |
| rolling_package_names = [p["name"] for p in rolling_packages_projects] |
| if "fuchsia/dart-sdk/${platform}" in rolling_package_names: |
| with api.step.nest("third-party dart packages") as presentation: |
| checkout_root = api.path["start_dir"] |
| rolled_hash = update_3p_packages( |
| api=api, |
| presentation=presentation, |
| checkout_root=checkout_root, |
| flutter_revision=orchestrator_version_dict["flutter_version"], |
| ) |
| changes = api.jiri.edit_manifest( |
| "integration/fuchsia/topaz/dart", |
| name="third_party/dart-pkg", |
| projects=[("third_party/dart-pkg", rolled_hash)], |
| ) |
| |
| # Update lock files |
| with api.step.nest("update jiri lock files"): |
| for lock in locks: |
| api.jiri.resolve( |
| local_manifest=True, output=lock["file"], manifests=[lock["manifest"]] |
| ) |
| |
| roller_name = api.buildbucket.builder_name |
| if roller_name.endswith("-roller"): |
| roller_name = api.buildbucket.builder_name[:7] |
| message = COMMIT_MESSAGE.format( |
| roller=roller_name, |
| packages="\n\n".join(package_msgs), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # All the changes are happening inside one of the git cloned projects. |
| # The "project" property points to the location of the project containing |
| # the changes and we need to move to that directory for the roller to pick |
| # up the changes. |
| project_dir = api.path["start_dir"].join(*project.split("/")) |
| with api.context(cwd=project_dir): |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(manifests[0]["remote"]), |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| rolled = change and change.success |
| |
| # Skip building and pushing Flutter debug symbols if a new version is not |
| # provided or running in dry run mode. For dry run mode, dependencies won't |
| # be available in the tree. |
| if ( |
| rolled |
| and debug_symbol_gcs_buckets |
| and flutter_fuchsia_version is not None |
| and debug_symbol_packages |
| and not dry_run |
| ): |
| api.debug_symbols.fetch_and_upload( |
| packages=debug_symbol_packages, |
| version=flutter_fuchsia_version, |
| buckets=debug_symbol_gcs_buckets, |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def update_3p_packages( |
| api, checkout_root, presentation, flutter_revision=None, dry_run=False |
| ): |
| """Updates fuchsia's third-party dart packages. |
| |
| Args: |
| presentation (StepPresentation): A presentation to attach logs, etc. to. |
| checkout_root (Path): Root path to checkouts. |
| flutter_revision (Optional): A git hash within the flutter repo. |
| dry_run (bool): Whether the roll will be allowed to complete or not. |
| |
| Returns: |
| A string with third_party/dart-pkg/pub latest commit hash. |
| """ |
| commit_msg = "[roll] Update third-party dart packages\n" |
| packages_root = checkout_root.join("third_party", "dart-pkg", "pub") |
| with api.context(cwd=packages_root): |
| # Make sure third_party/dart-pkg is at origin/master before running the |
| # update script to catch any manual commits that extend past the revision at |
| # integration's HEAD. |
| api.git("git fetch", "fetch", "origin") |
| api.git("git checkout", "checkout", "origin/master") |
| flutter_flags = ( |
| ["--flutter-revision", flutter_revision] if flutter_revision else [] |
| ) |
| api.python( |
| "update dart 3p packages", |
| checkout_root.join("scripts", "dart", "update_3p_packages.py"), |
| args=["--debug"] + flutter_flags, |
| ) |
| change = api.auto_roller.attempt_roll( |
| "fuchsia-review.googlesource.com", |
| gerrit_project="third_party/dart-pkg", |
| repo_dir=packages_root, |
| commit_message=commit_msg, |
| commit_untracked=True, |
| dry_run=dry_run, |
| ) |
| # If we rolled changes, the final hash of the merged CL should |
| # correspond to the remote HEAD. Otherwise, we can assume that local |
| # HEAD is up to date with remote HEAD. |
| current_hash = change.revision if change else api.git.get_hash() |
| presentation.step_text = current_hash |
| return current_hash |
| |
| |
| def GenTests(api): |
| """Tests for cipd with dependents roller.""" |
| properties = { |
| "project": "integration", |
| "manifests": [ |
| { |
| "project": "integration", |
| "manifest": "other/dependency", |
| "remote": "sso://fuchsia/integration", |
| "lock_file": "integration/jiri.lock", |
| }, |
| { |
| "project": "integration", |
| "manifest": "fuchsia/flower", |
| "remote": "sso://fuchsia/integration", |
| "lock_file": "integration/fuchsia/jiri.lock", |
| }, |
| ], |
| "locks": [{"manifest": "a/b", "file": "a/b/jiri.lock"}], |
| "orchestrator_package": "fuchsia/orchestrator", |
| "orchestrator_import_in": "fuchsia/prebuilts", |
| "validate_against_package": { |
| "package": "flutter/fuchsia", |
| "version_tag": "flutter_version", |
| }, |
| "rolling_packages_projects": [ |
| { |
| "name": "dart/sdk", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "dart_version", |
| }, |
| { |
| "name": "skia", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "skia_version", |
| }, |
| { |
| "name": "flutter/fuchsia", |
| "type": "package", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "engine_version", |
| }, |
| { |
| "import_in": "integration/fuchsia/prebuilts", |
| "name": "fuchsia/dart-sdk/${platform}", |
| "type": "package", |
| "version_tag": "dart_version", |
| }, |
| ], |
| "versions_file": "flutter/versions.json", |
| "owners": ["abc@gmail.com"], |
| "dry_run": True, |
| "debug_symbol_packages": ["flutter/fuchsia-debug-symbols-x64"], |
| "debug_symbol_gcs_buckets": ["fuchsia-debug-symbols-shortlived"], |
| } |
| |
| package_no_match_test_data = api.step_data( |
| "cipd describe fuchsia/orchestrator", |
| api.cipd.example_describe( |
| package_name="fuchsia/orchestrator", |
| version="version_abc", |
| test_data_tags=["git_revision:revision_abc"], |
| ), |
| ) |
| jiri_package_test_data = [ |
| { |
| "name": "fuchsia/orchestrator", |
| "path": "fuchsia/orchestrator", |
| "version": "git_revision:revision_abc", |
| "manifest": "manifest1", |
| }, |
| ] |
| default_properties = api.properties(**properties) |
| # Version of the cipd orchestrator package is the same as the one |
| # in the tree. |
| yield ( |
| api.status_check.test("nothing_to_roll") |
| + default_properties |
| + package_no_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| ) |
| |
| # Version of the cipd orchestrator package is different from the one |
| # in the tree and ready to get rolled but flutter version is the same as the |
| # one in the tree. |
| package_match_test_data = api.step_data( |
| "cipd describe fuchsia/orchestrator", |
| api.cipd.example_describe( |
| package_name="fuchsia/orchestrator", |
| version="version_abc", |
| test_data_tags=["git_revision:revision_jkl"], |
| ), |
| ) |
| package_versions = { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz", |
| } |
| jiri_flutter_package_test_data = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "flutter/fuchsia", |
| "version": "git_revision:flutter_version_xyz", |
| "manifest": "manifest1", |
| }, |
| ] |
| yield ( |
| api.status_check.test("orchestrator_dep_version_same_as_tree") |
| + default_properties |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + package_match_test_data |
| + api.step_data( |
| "jiri package (2)", api.jiri.package(jiri_flutter_package_test_data) |
| ) |
| ) |
| |
| # Version of the validation cipd package is different from the one in the tree |
| # and ready to get rolled along its dependencies. |
| jiri_flutter_package_test_data_no_match = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "flutter/fuchsia", |
| "version": "git_revision:flutter_version_xyz1", |
| "manifest": "manifest1", |
| }, |
| ] |
| yield ( |
| api.status_check.test( |
| "orchestrator_dep_version_different_from_tree_dry_run", status="failure" |
| ) |
| + default_properties |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data( |
| "jiri package (2)", |
| api.jiri.package(jiri_flutter_package_test_data_no_match), |
| ) |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| + api.auto_roller.dry_run_success( |
| name="third-party dart packages.check for completion.check if done ({})" |
| ) |
| ) |
| |
| # Validate the build and upload debug symbols path. |
| props = copy.deepcopy(properties) |
| props["dry_run"] = False |
| no_dry_run_props = api.properties(**props) |
| yield ( |
| api.status_check.test("orchestrator_dep_version_different_from_tree") |
| + no_dry_run_props |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data( |
| "jiri package (2)", |
| api.jiri.package(jiri_flutter_package_test_data_no_match), |
| ) |
| + api.auto_roller.success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| + api.auto_roller.success( |
| name="third-party dart packages.check for completion.check if done (0)" |
| ) |
| ) |
| # Validate the dar-sdk plugins steps is not executed when dart-sdk is not |
| # being rolled. |
| rolling_packages = [ |
| { |
| "name": "dart/sdk", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "dart_version", |
| }, |
| { |
| "name": "skia", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "skia_version", |
| }, |
| { |
| "name": "flutter/fuchsia", |
| "type": "package", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "engine_version", |
| }, |
| ] |
| props = copy.deepcopy(properties) |
| props["dry_run"] = False |
| props["rolling_packages_projects"] = rolling_packages |
| no_dry_run_props = api.properties(**props) |
| yield ( |
| api.status_check.test("do_not_roll_plugins_if_dart_not_rolled") |
| + no_dry_run_props |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data( |
| "jiri package (2)", |
| api.jiri.package(jiri_flutter_package_test_data_no_match), |
| ) |
| + api.auto_roller.success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| ) |
| |
| # Validate that uploading debug symbols is skipped when the flutter/fuchsia |
| # package is not rolled. |
| rolling_packages = [ |
| { |
| "name": "dart/sdk", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "dart_version", |
| }, |
| { |
| "name": "skia", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "skia_version", |
| }, |
| ] |
| props = copy.deepcopy(properties) |
| props["dry_run"] = False |
| props["rolling_packages_projects"] = rolling_packages |
| no_dry_run_props = api.properties(**props) |
| yield ( |
| api.status_check.test( |
| "do_not_upload_debug_symbols_if_flutter_fuchsia_is_not_rolled" |
| ) |
| + no_dry_run_props |
| + package_match_test_data |
| + api.step_data("jiri package", api.jiri.package(jiri_package_test_data)) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data( |
| "jiri package (2)", |
| api.jiri.package(jiri_flutter_package_test_data_no_match), |
| ) |
| + api.auto_roller.success() |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| ) |
