| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for automatically updating Crasphad and its sub-dependency - mini_chromium.""" |
| |
| from collections import OrderedDict |
| |
| import re |
| |
| from PB.recipes.fuchsia.contrib.crashpad_roller import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/git", |
| "fuchsia/git_checkout", |
| "fuchsia/gitiles", |
| "fuchsia/jiri", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| CRASHPAD_PROJECT_NAME = "third_party/crashpad" |
| MINI_CHROMIUM_PROJECT_NAME = "chromium/mini_chromium" |
| CRASHPAD_REMOTE = "https://fuchsia.googlesource.com/third_party/crashpad" |
| |
| COMMIT_MESSAGE = """\ |
| [roll] Update {deps} |
| |
| {logs} |
| """ |
| |
| PROJECT_LOG_FORMAT = """\ |
| {project} {old}..{new} ({count} commits) |
| {commits} |
| """ |
| |
| TEST_DEPS_DATA_TEMPLATE = """\ |
| '%s': |
| Var('chromium_git') + '/linux-syscall-support.git@' + |
| '8048ece6c16c91acfe0d36d1d3cc0890ab6e945c', |
| """ |
| |
| |
| def RunSteps(api, props): |
| with api.context(infra_steps=True): |
| api.jiri.init(use_lock_file=True) |
| api.jiri.import_manifest(props.manifest, props.remote, props.project) |
| api.jiri.update(run_hooks=False) |
| |
| manifest_repo = api.path.start_dir.join(*props.project.split("/")) |
| manifest_path = manifest_repo.join(*props.import_in.split("/")) |
| |
| revision = props.revision |
| if not revision: |
| revision = api.git.get_remote_branch_head(CRASHPAD_REMOTE, "refs/heads/master") |
| |
| updated_deps = update_crashpad_deps(api, revision, manifest_path) |
| |
| if not updated_deps: |
| api.step.empty("manifest up-to-date; nothing to roll") |
| return api.auto_roller.nothing_to_roll() |
| |
| # Update lockfiles |
| with api.context(cwd=manifest_repo): |
| for lock_entry in props.lockfiles: |
| fields = lock_entry.split("=") |
| manifest = fields[0] |
| lock = fields[1] |
| api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest]) |
| |
| # Land the changes |
| commit_message = COMMIT_MESSAGE.format( |
| deps=", ".join(updated_deps.keys()), |
| logs="\n".join(updated_deps.values()), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| change = api.auto_roller.attempt_roll( |
| api.auto_roller.Options( |
| remote=props.remote, |
| ), |
| repo_dir=manifest_repo, |
| commit_message=commit_message, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| |
| def extract_revision_from_deps(api, deps_path, dep_name): |
| """Extracts the dependency version from Crashpad's DEPS file. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| deps_path (Path): A path to a DEPS file. |
| dep_name (str): The name of a dependency to extract from the DEPS file. |
| |
| Returns: |
| A String containing the revision. |
| """ |
| dep_id = f"crashpad/third_party/{dep_name}/{dep_name}" |
| contents = api.file.read_text( |
| name=f"read DEPS file for {dep_name}", |
| source=deps_path, |
| test_data=TEST_DEPS_DATA_TEMPLATE % dep_id, |
| ) |
| m = re.search(r"\'%s\':[\s\S]*?\'(?P<revision>[0-9a-f]{40})\'," % dep_id, contents) |
| if m: |
| return m.group("revision") |
| raise api.step.InfraFailure(f"failed to find {dep_id} in DEPS") |
| |
| |
| def update_crashpad_deps(api, crashpad_revision, manifest): |
| """Update manifest for all Crashpad dependencies. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| crashpad_revision (str): SHA-1 hash representing the Crashpad revision |
| to roll dependencies to. |
| manifest (str): Path to the manifest to edit relative to project. |
| |
| Returns: |
| A dictionary mapping of updated dependencies to their corresponding |
| update logs. |
| """ |
| updated_deps = OrderedDict() |
| with api.step.nest("crashpad"): |
| update_manifest_project( |
| api, |
| manifest=manifest, |
| project_name=CRASHPAD_PROJECT_NAME, |
| revision=crashpad_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| # Only check for sub-dependency changes if Crashpad's revision has been updated |
| if updated_deps: |
| # Checkout the Crashpad revision that we are rolling to in order to view |
| # its sub-dependencies which are defined in the Crashpad repo's DEPS file |
| crashpad_dir, _ = api.git_checkout(CRASHPAD_REMOTE, revision=crashpad_revision) |
| crashpad_deps_path = crashpad_dir.join("DEPS") |
| mini_chromium_revision = extract_revision_from_deps( |
| api, crashpad_deps_path, "mini_chromium" |
| ) |
| |
| with api.step.nest("mini_chromium"): |
| update_manifest_project( |
| api, |
| manifest=manifest, |
| project_name=MINI_CHROMIUM_PROJECT_NAME, |
| revision=mini_chromium_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| return updated_deps |
| |
| |
| def update_manifest_project(api, manifest, project_name, revision, updated_deps): |
| """Updates the revision for a project in a manifest. |
| |
| Adds an entry to "updated_deps" if manifest is successfully updated; |
| otherwise, no new entries are added. |
| |
| Args: |
| manifest (Path): Path to the Jiri manifest to update. |
| project_name (str): Name of the project in the Jiri manifest to update. |
| revision (str): SHA-1 hash representing the updated revision for |
| project_name in the manifest. |
| updated_deps (OrderedDict): A dictionary mapping project name to a formatted |
| log of the corresponding changes. |
| |
| Returns: |
| The project's remote property. |
| """ |
| remote = api.jiri.read_manifest_element( |
| manifest=manifest, |
| element_type="project", |
| element_name=project_name, |
| ).get("remote") |
| changes = api.jiri.edit_manifest( |
| manifest=manifest, |
| projects=[(project_name, revision)], |
| test_data={"projects": [{"old_revision": "abc123", "new_revision": "def456"}]}, |
| name=f"jiri edit {project_name}", |
| ) |
| if not changes["projects"]: |
| api.step.active_result.presentation.step_text = ( |
| "manifest up-to-date, nothing to roll" |
| ) |
| return remote |
| |
| old_rev = changes["projects"][0]["old_revision"] |
| new_rev = changes["projects"][0]["new_revision"] |
| log = api.gitiles.log( |
| remote, f"{old_rev}..{new_rev}", step_name=f"log {project_name}" |
| ) |
| formatted_log = PROJECT_LOG_FORMAT.format( |
| project=project_name, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| commits="\n".join( |
| [ |
| f"{commit['id'][:7]} {commit['message'].splitlines()[0]}" |
| for commit in log |
| ] |
| ), |
| ) |
| |
| # Add dependency update entry |
| updated_deps[project_name] = formatted_log |
| return remote |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "revision": "abc123", |
| "project": "integration", |
| "manifest": "fuchsia/minimal", |
| "remote": "sso://fuchsia/integration", |
| "import_in": "fuchsia/third_party/flower", |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| project_names = { |
| "crashpad": CRASHPAD_PROJECT_NAME, |
| "mini_chromium": MINI_CHROMIUM_PROJECT_NAME, |
| } |
| |
| def noop_edit(name): |
| return api.step_data( |
| f"{name}.jiri edit {project_names[name]}", |
| api.json.output({"projects": []}), |
| ) |
| |
| def log_data(name): |
| return api.gitiles.log(f"{name}.log {project_names[name]}", "A") |
| |
| crashpad_check_data = api.jiri.read_manifest_element( |
| element_name=project_names["crashpad"], |
| test_output={"remote": "https://fuchsia.googlesource.com/third_party/crashpad"}, |
| nesting="crashpad", |
| ) |
| crashpad_log_data = log_data("crashpad") |
| |
| mini_chromium_check_data = api.jiri.read_manifest_element( |
| element_name=project_names["mini_chromium"], |
| test_output={"remote": "https://fuchsia.googlesource.com/third_party/crashpad"}, |
| nesting="mini_chromium", |
| ) |
| mini_chromium_log_data = log_data("mini_chromium") |
| |
| yield ( |
| api.test("noop_roll") |
| + properties() |
| + crashpad_check_data |
| + noop_edit("crashpad") |
| ) |
| |
| yield ( |
| api.test("DEPS_parsing_error", status="INFRA_FAILURE") |
| + properties() |
| + crashpad_check_data |
| + crashpad_log_data |
| + api.step_data( |
| "read DEPS file for mini_chromium", |
| api.raw_io.output_text("invalid content"), |
| ) |
| ) |
| |
| yield ( |
| api.test("missing_revision") |
| + properties(revision="") |
| + api.git.get_remote_branch_head( |
| "git ls-remote", "fc4dc762688d2263b254208f444f5c0a4b91bc07" |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + noop_edit("mini_chromium") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.test("crashpad_roll_with_lockfile") |
| + properties(lockfiles=["manifest/foo=jiri.lock"]) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + noop_edit("mini_chromium") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.test("crashpad") |
| + properties() |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + noop_edit("mini_chromium") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.test("crashpad_and_mini_chromium") |
| + properties() |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + mini_chromium_log_data |
| + api.auto_roller.success() |
| ) |
