| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling flutter and dependencies. |
| |
| This recipe rolls Flutter and related packages not to HEAD, but to a version |
| specified by a file that appears in a Fuchsia checkout. Essentially, it looks up |
| the version of Flutter used by a particular prebuilt package and rolls to that. |
| We use the fact that the prebuilt package successfully released and rolled into |
| fuchisa.git as a signal that the version of Flutter it uses is good. |
| |
| In this recipe we are calling the file that indicates the desired versions of |
| Flutter (and dependencies) the "orchestrator file". |
| |
| The orchestrator file is a json file of the following form: |
| |
| { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz" |
| } |
| |
| For each package and project listed in the `rolling_packages_projects` |
| parameter, this recipe updates its jiri manifest to match the correct version |
| from the orchestrator file. If everything is already at the correct version, it |
| simply bails out with "nothing to roll". |
| |
| Warning: If this recipe runs in dry mode it will still roll changes into |
| fuchsia dart 3p repository but the fuchsia tree won't pin to the latest |
| version. This is safe as the fuchsia dart 3p hash will be bumped only on a |
| successful roll of all the other dependencies. |
| """ |
| |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/debug_symbols", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "fuchsia/jiri", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/python", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Jiri remote manifest project", default=None), |
| "manifests": Property( |
| kind=List(dict), |
| help=( |
| "A list of dictionaries with project, manifest and remote keys " |
| "used to signal jiri wich manifests to synchronize. " |
| 'E.g. [{"project": "integration", "manifest": "other/dependency", ' |
| ' "remote": "sso://fuchsia/integration", ' |
| ' "lock_file": "integration/jiri.lock"}]' |
| ), |
| ), |
| "locks": Property( |
| kind=List(dict), |
| help=( |
| "A list of dictionaries with manifest and files keys used to " |
| "signal jiri which lock files to update. " |
| ' E.g. [{"manifest": "integration/fuchsia/flower", ' |
| ' "file": "integration/fuchsia/jiri.lock"}]' |
| ), |
| ), |
| "owners": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "The owners responsible for watching this roller " |
| '(example: "username@google.com").' |
| ), |
| ), |
| "rolling_packages_projects": Property( |
| kind=List(dict), |
| default=(), |
| help=( |
| "A list of dictionaries with rolling package/project, type," |
| " and location to import_in. This is used to list the " |
| "dependencies that should be updated: " |
| 'E.g. [{"name": "dart", "type": "package", ' |
| ' "import_in": "fuchsia/prebuilts", ' |
| ' "version_tag": "dart_version"}]' |
| ), |
| ), |
| "orchestrator_file_path": Property( |
| kind=str, |
| help=( |
| "A string with the path to the orchestrator file inside a " |
| "checked-out fuchsia build." |
| ), |
| ), |
| "dry_run": Property( |
| kind=bool, |
| default=False, |
| help="Whether to dry-run the auto-roller (CQ+1 and abandon the change)", |
| ), |
| "debug_symbol_packages": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "A list of strings with the cipd packages containing debug " |
| "symbols and their associated packages." |
| ), |
| ), |
| "debug_symbol_gcs_buckets": Property( |
| kind=List(str), |
| default=(), |
| help="GCS buckets to upload debug symbols to", |
| ), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {roller} packages and projects: |
| |
| {packages} |
| |
| Test: CQ""" |
| |
| |
| def RunSteps( |
| api, |
| project, |
| manifests, |
| locks, |
| owners, |
| orchestrator_file_path, |
| rolling_packages_projects, |
| dry_run, |
| debug_symbol_packages, |
| debug_symbol_gcs_buckets, |
| ): |
| with api.context(infra_steps=True): |
| if owners: |
| with api.step.nest("owners") as presentation: |
| presentation.step_summary_text = ", ".join(owners) |
| |
| # Import manifests |
| with api.step.nest("jiri-manifests") as presentation: |
| api.jiri.init(use_lock_file=True) |
| presentation.logs["manifests"] = str(manifests) |
| for manifest_config in manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config["manifest"], |
| remote=manifest_config["remote"], |
| name=manifest_config["project"], |
| ) |
| api.jiri.update(run_hooks=False) |
| with api.context(cwd=api.path["start_dir"]): |
| api.jiri.run_hooks() |
| |
| version_dict = api.file.read_json( |
| "read versions", api.path["start_dir"].join(orchestrator_file_path) |
| ) |
| |
| # Roll all the dependencies |
| package_msgs = [] |
| package_msg_template = "%s %s rolled from %s to %s" |
| updated_packages_projects = set() |
| with api.step.nest("edit jiri manifests"): |
| for package_project in rolling_packages_projects: |
| if package_project["type"] == "package": |
| expected_version = ( |
| "git_revision:%s" % version_dict[package_project["version_tag"]] |
| ) |
| changes = api.jiri.edit_manifest( |
| package_project["import_in"], |
| packages=[(package_project["name"], expected_version)], |
| name=package_project["name"], |
| ) |
| # Sometimes only a subset of packages are updated. In those cases |
| # the changes list is empty. |
| if changes["packages"]: |
| updated_packages_projects.add(package_project["name"]) |
| old_version = changes["packages"][0]["old_version"] |
| package_msgs.append( |
| package_msg_template |
| % ( |
| "package", |
| package_project["name"], |
| old_version, |
| expected_version, |
| ) |
| ) |
| else: |
| expected_version = version_dict[package_project["version_tag"]] |
| changes = api.jiri.edit_manifest( |
| package_project["import_in"], |
| name=package_project["name"], |
| projects=[(package_project["name"], expected_version)], |
| ) |
| # Sometimes only a subset of projects are updated. In those cases |
| # the changes list is empty. |
| if changes["projects"]: |
| updated_packages_projects.add(package_project["name"]) |
| old_version = changes["projects"][0]["old_revision"] |
| package_msgs.append( |
| package_msg_template |
| % ( |
| "project", |
| package_project["name"], |
| old_version, |
| expected_version, |
| ) |
| ) |
| |
| if not updated_packages_projects: |
| return api.auto_roller.nothing_to_roll() |
| |
| # Update fuchsia's third-party dart packages (including flutter dependencies), |
| # only if we are rolling a new version of dart-sdk. |
| if "fuchsia/dart-sdk/${platform}" in updated_packages_projects: |
| with api.step.nest("third-party dart packages") as presentation: |
| checkout_root = api.path["start_dir"] |
| rolled_hash = update_3p_packages( |
| api=api, |
| presentation=presentation, |
| checkout_root=checkout_root, |
| flutter_revision=version_dict["flutter_version"], |
| ) |
| changes = api.jiri.edit_manifest( |
| "integration/fuchsia/topaz/dart", |
| name="third_party/dart-pkg", |
| projects=[("third_party/dart-pkg", rolled_hash)], |
| ) |
| |
| # Update lock files |
| with api.step.nest("update jiri lock files"): |
| for lock in locks: |
| api.jiri.resolve( |
| local_manifest=True, output=lock["file"], manifests=[lock["manifest"]] |
| ) |
| |
| message = COMMIT_MESSAGE.format( |
| roller=api.buildbucket.builder_name, |
| packages="\n\n".join(package_msgs), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # All the changes are happening inside one of the git cloned projects. |
| # The "project" property points to the location of the project containing |
| # the changes and we need to move to that directory for the roller to pick |
| # up the changes. |
| project_dir = api.path["start_dir"].join(*project.split("/")) |
| with api.context(cwd=project_dir): |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(manifests[0]["remote"]), |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| rolled = change and change.success |
| |
| # Skip building and pushing Flutter debug symbols if a new version is not |
| # provided or running in dry run mode. For dry run mode, dependencies won't |
| # be available in the tree. |
| if ( |
| rolled |
| and debug_symbol_gcs_buckets |
| and "flutter/fuchsia" in updated_packages_projects |
| and debug_symbol_packages |
| and not dry_run |
| ): |
| api.debug_symbols.fetch_and_upload( |
| packages=debug_symbol_packages, |
| version=("git_revision:%s" % version_dict["engine_version"]), |
| buckets=debug_symbol_gcs_buckets, |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def update_3p_packages( |
| api, checkout_root, presentation, flutter_revision=None, dry_run=False |
| ): |
| """Updates fuchsia's third-party dart packages. |
| |
| Args: |
| presentation (StepPresentation): A presentation to attach logs, etc. to. |
| checkout_root (Path): Root path to checkouts. |
| flutter_revision (Optional): A git hash within the flutter repo. |
| dry_run (bool): Whether the roll will be allowed to complete or not. |
| |
| Returns: |
| A string with third_party/dart-pkg/pub latest commit hash. |
| """ |
| commit_msg = "[roll] Update third-party dart packages\n" |
| packages_root = checkout_root.join("third_party", "dart-pkg", "pub") |
| with api.context(cwd=packages_root): |
| # Make sure third_party/dart-pkg is at origin/main before running the |
| # update script to catch any manual commits that extend past the revision at |
| # integration's HEAD. |
| api.git("git fetch", "fetch", "origin") |
| api.git("git checkout", "checkout", "origin/main") |
| flutter_flags = ( |
| ["--flutter-revision", flutter_revision] if flutter_revision else [] |
| ) |
| api.python( |
| "update dart 3p packages", |
| checkout_root.join("scripts", "dart", "update_3p_packages.py"), |
| args=["--debug"] + flutter_flags, |
| ) |
| change = api.auto_roller.attempt_roll( |
| "fuchsia-review.googlesource.com", |
| gerrit_project="third_party/dart-pkg", |
| repo_dir=packages_root, |
| commit_message=commit_msg, |
| commit_untracked=True, |
| dry_run=dry_run, |
| ) |
| # If we rolled changes, the final hash of the merged CL should |
| # correspond to the remote HEAD. Otherwise, we can assume that local |
| # HEAD is up to date with remote HEAD. |
| current_hash = change.revision if change else api.git.get_hash() |
| presentation.step_text = current_hash |
| return current_hash |
| |
| |
| def GenTests(api): |
| """Tests for cipd with dependents roller.""" |
| properties = { |
| "project": "integration", |
| "manifests": [ |
| { |
| "project": "integration", |
| "manifest": "other/dependency", |
| "remote": "sso://fuchsia/integration", |
| "lock_file": "integration/jiri.lock", |
| }, |
| { |
| "project": "integration", |
| "manifest": "fuchsia/flower", |
| "remote": "sso://fuchsia/integration", |
| "lock_file": "integration/fuchsia/jiri.lock", |
| }, |
| ], |
| "locks": [{"manifest": "a/b", "file": "a/b/jiri.lock"}], |
| "validate_against_package": { |
| "package": "flutter/fuchsia", |
| "version_tag": "flutter_version", |
| }, |
| "rolling_packages_projects": [ |
| { |
| "name": "dart/sdk", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "dart_version", |
| }, |
| { |
| "name": "skia", |
| "type": "project", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "skia_version", |
| }, |
| { |
| "name": "flutter/fuchsia", |
| "type": "package", |
| "import_in": "fuchsia/prebuilts", |
| "version_tag": "engine_version", |
| }, |
| { |
| "import_in": "integration/fuchsia/prebuilts", |
| "name": "fuchsia/dart-sdk/${platform}", |
| "type": "package", |
| "version_tag": "dart_version", |
| }, |
| ], |
| "orchestrator_file_path": "some_prebuilt/flutter/versions.json", |
| "owners": ["abc@gmail.com"], |
| "dry_run": False, |
| "debug_symbol_packages": ["flutter/fuchsia-debug-symbols-x64"], |
| "debug_symbol_gcs_buckets": ["fuchsia-debug-symbols-shortlived"], |
| } |
| |
| jiri_package_test_data = [ |
| { |
| "name": "fuchsia/orchestrator", |
| "path": "fuchsia/orchestrator", |
| "version": "git_revision:revision_abc", |
| "manifest": "manifest1", |
| }, |
| ] |
| default_properties = api.properties(**properties) |
| |
| # Flutter version is the same as the one in the tree. |
| package_versions = { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz", |
| } |
| jiri_flutter_package_test_data = [ |
| { |
| "name": "flutter/fuchsia", |
| "path": "flutter/fuchsia", |
| "version": "git_revision:flutter_version_xyz", |
| "manifest": "manifest1", |
| }, |
| ] |
| |
| jiri_no_changes_output = api.json.output(api.jiri.example_edit([], [], [])) |
| |
| yield ( |
| api.status_check.test("nothing_to_roll") |
| + default_properties |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data("edit jiri manifests.dart/sdk", jiri_no_changes_output) |
| + api.step_data("edit jiri manifests.skia", jiri_no_changes_output) |
| + api.step_data("edit jiri manifests.flutter/fuchsia", jiri_no_changes_output) |
| + api.step_data( |
| "edit jiri manifests.fuchsia/dart-sdk/${platform}", jiri_no_changes_output |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("everything_to_roll") |
| + default_properties |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder="flutter-dependents-roller") |
| ) |
| + api.auto_roller.success() |
| + api.auto_roller.success( |
| name="third-party dart packages.check for completion.check if done (0)" |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("roll_neither_flutter_nor_dart") |
| + default_properties |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data("edit jiri manifests.dart/sdk", jiri_no_changes_output) |
| # Default response (an update happened) for skia. |
| + api.step_data("edit jiri manifests.flutter/fuchsia", jiri_no_changes_output) |
| + api.step_data( |
| "edit jiri manifests.fuchsia/dart-sdk/${platform}", jiri_no_changes_output |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.status_check.test("roll_flutter_package_alone") |
| + default_properties |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data("edit jiri manifests.dart/sdk", jiri_no_changes_output) |
| + api.step_data("edit jiri manifests.skia", jiri_no_changes_output) |
| # Default response (an update happened) for flutter/fuchsia. |
| + api.step_data( |
| "edit jiri manifests.fuchsia/dart-sdk/${platform}", jiri_no_changes_output |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.status_check.test("roll_dart_sdk_package_alone") |
| + default_properties |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.step_data("edit jiri manifests.dart/sdk", jiri_no_changes_output) |
| + api.step_data("edit jiri manifests.skia", jiri_no_changes_output) |
| + api.step_data("edit jiri manifests.flutter/fuchsia", jiri_no_changes_output) |
| # Default response (an update happened) for fuchsia/dart-sdk/${platform}. |
| + api.auto_roller.success() |
| + api.auto_roller.success( |
| name="third-party dart packages.check for completion.check if done (0)" |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("dry_run") |
| + api.properties(**dict(properties.items() + [("dry_run", True)])) |
| + api.step_data( |
| "read versions", api.file.read_json(json_content=package_versions) |
| ) |
| + api.auto_roller.success() |
| + api.auto_roller.success( |
| name="third-party dart packages.check for completion.check if done (0)" |
| ) |
| ) |
