| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for automatically updating Crasphad and its sub-dependencies - mini_chromium & lss.""" |
| |
| from collections import OrderedDict |
| |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "fuchsia/gitiles", |
| "fuchsia/jiri", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Jiri remote manifest project", default=None), |
| "manifest": Property(kind=str, help="Jiri manifest to use"), |
| "remote": Property(kind=str, help="Remote manifest repository"), |
| "import_in": Property( |
| kind=str, |
| help="Path to the manifest to edit relative to $project", |
| default="fuchsia/third_party/flower", |
| ), |
| "revision": Property(kind=str, help="Crashpad revision to roll to", default=""), |
| "lockfiles": Property( |
| kind=List(str), |
| default=[], |
| help='The list of lockfiles to update in "${manifest}=${lockfile}" format', |
| ), |
| } |
| |
| CRASHPAD_PROJECT_NAME = "third_party/crashpad" |
| MINI_CHROMIUM_PROJECT_NAME = "third_party/mini_chromium" |
| LSS_PROJECT_NAME = "third_party/lss" |
| CRASHPAD_REMOTE = "https://chromium.googlesource.com/crashpad/crashpad" |
| |
| COMMIT_MESSAGE = """\ |
| [roll] Update {deps} |
| |
| {logs} |
| """ |
| |
| PROJECT_LOG_FORMAT = """\ |
| {project} {old}..{new} ({count} commits) |
| {commits} |
| """ |
| |
| TEST_DEPS_DATA_TEMPLATE = """\ |
| '%s': |
| Var('chromium_git') + '/linux-syscall-support.git@' + |
| '8048ece6c16c91acfe0d36d1d3cc0890ab6e945c', |
| """ |
| |
| |
| def RunSteps(api, manifest, remote, project, revision, import_in, lockfiles): |
| with api.context(infra_steps=True): |
| api.jiri.init(use_lock_file=True) |
| api.jiri.import_manifest(manifest, remote, project) |
| api.jiri.update(run_hooks=False) |
| |
| manifest_repo = api.path["start_dir"].join(*project.split("/")) |
| manifest_path = manifest_repo.join(*import_in.split("/")) |
| |
| if not revision: |
| revision = api.git.get_remote_branch_head(CRASHPAD_REMOTE, "refs/heads/master") |
| |
| updated_deps = update_crashpad_deps(api, revision, manifest_path) |
| |
| if not updated_deps: |
| api.step("manifest up-to-date; nothing to roll", None) |
| return api.auto_roller.nothing_to_roll() |
| |
| # Update lockfiles |
| with api.context(cwd=manifest_repo): |
| for lock_entry in lockfiles: |
| fields = lock_entry.split("=") |
| manifest = fields[0] |
| lock = fields[1] |
| api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest]) |
| |
| # Land the changes |
| commit_message = COMMIT_MESSAGE.format( |
| deps=", ".join(updated_deps.keys()), |
| logs="\n".join(updated_deps.itervalues()), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(remote), |
| gerrit_project=project, |
| repo_dir=manifest_repo, |
| commit_message=commit_message, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| |
| def extract_revision_from_deps(api, deps_path, dep_name): |
| """Extracts the dependency version from Crashpad's DEPS file. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| deps_path (Path): A path to a DEPS file. |
| dep_name (str): The name of a dependency to extract from the DEPS file. |
| |
| Returns: |
| A String containing the revision. |
| """ |
| dep_id = "crashpad/third_party/%s/%s" % (dep_name, dep_name) |
| contents = api.file.read_text( |
| name="read DEPS file for %s" % dep_name, |
| source=deps_path, |
| test_data=TEST_DEPS_DATA_TEMPLATE % dep_id, |
| ) |
| m = re.search(r"\'%s\':[\s\S]*?\'(?P<revision>[0-9a-f]{40})\'," % dep_id, contents) |
| if m: |
| return m.group("revision") |
| raise api.step.InfraFailure("failed to find %s in DEPS" % dep_id) |
| |
| |
| def update_crashpad_deps(api, crashpad_revision, manifest): |
| """Update manifest for all Crashpad dependencies. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| crashpad_revision (str): SHA-1 hash representing the Crashpad revision |
| to roll dependencies to. |
| manifest (str): Path to the manifest to edit relative to project. |
| |
| Returns: |
| A dictionary mapping of updated dependencies to their corresponding |
| update logs. |
| """ |
| updated_deps = OrderedDict() |
| with api.step.nest("crashpad"): |
| update_manifest_project( |
| api, |
| manifest=manifest, |
| project_name=CRASHPAD_PROJECT_NAME, |
| revision=crashpad_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| # Only check for sub-dependency changes if Crashpad's revision has been updated |
| if updated_deps: |
| tmp_dir = api.path.mkdtemp("git_checkout") |
| crashpad_dir = tmp_dir.join("crashpad") |
| crashpad_deps_path = crashpad_dir.join("DEPS") |
| # Checkout the Crashpad revision that we are rolling to in order to view |
| # its sub-dependencies which are defined in the Crashpad repo's DEPS file |
| api.git.checkout( |
| url=CRASHPAD_REMOTE, |
| path=crashpad_dir, |
| ref=crashpad_revision, |
| ) |
| mini_chromium_revision = extract_revision_from_deps( |
| api, crashpad_deps_path, "mini_chromium" |
| ) |
| lss_revision = extract_revision_from_deps(api, crashpad_deps_path, "lss") |
| |
| with api.step.nest("mini_chromium"): |
| update_manifest_project( |
| api, |
| manifest=manifest, |
| project_name=MINI_CHROMIUM_PROJECT_NAME, |
| revision=mini_chromium_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| with api.step.nest("lss"): |
| update_manifest_project( |
| api, |
| manifest=manifest, |
| project_name=LSS_PROJECT_NAME, |
| revision=lss_revision, |
| updated_deps=updated_deps, |
| ) |
| return updated_deps |
| |
| |
| def update_manifest_project(api, manifest, project_name, revision, updated_deps): |
| """Updates the revision for a project in a manifest. |
| |
| Adds an entry to "updated_deps" if manifest is successfully updated; |
| otherwise, no new entries are added. |
| |
| Args: |
| manifest (Path): Path to the Jiri manifest to update. |
| project_name (str): Name of the project in the Jiri manifest to update. |
| revision (str): SHA-1 hash representing the updated revision for |
| project_name in the manifest. |
| updated_deps (OrderedDict): A dictionary mapping project name to a formatted |
| log of the corresponding changes. |
| |
| Returns: |
| The project's remote property. |
| """ |
| remote = api.jiri.read_manifest_element( |
| manifest=manifest, |
| element_type="project", |
| element_name=project_name, |
| ).get("remote") |
| changes = api.jiri.edit_manifest( |
| manifest=manifest, |
| projects=[(project_name, revision)], |
| test_data={"projects": [{"old_revision": "abc123", "new_revision": "def456"}]}, |
| name="jiri edit %s" % project_name, |
| ) |
| if not changes["projects"]: |
| api.step.active_result.presentation.step_text = ( |
| "manifest up-to-date, nothing to roll" |
| ) |
| return remote |
| |
| old_rev = changes["projects"][0]["old_revision"] |
| new_rev = changes["projects"][0]["new_revision"] |
| log = api.gitiles.log( |
| remote, "%s..%s" % (old_rev, new_rev), step_name="log %s" % project_name |
| ) |
| formatted_log = PROJECT_LOG_FORMAT.format( |
| project=project_name, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| commits="\n".join( |
| [ |
| "{commit} {subject}".format( |
| commit=commit["id"][:7], |
| subject=commit["message"].splitlines()[0], |
| ) |
| for commit in log |
| ] |
| ), |
| ) |
| |
| # Add dependency update entry |
| updated_deps[project_name] = formatted_log |
| return remote |
| |
| |
| def GenTests(api): |
| project_names = { |
| "crashpad": CRASHPAD_PROJECT_NAME, |
| "mini_chromium": MINI_CHROMIUM_PROJECT_NAME, |
| "lss": LSS_PROJECT_NAME, |
| } |
| |
| noop_edit = lambda name: api.step_data( |
| "%s.jiri edit %s" % (name, project_names[name]), |
| api.json.output({"projects": []}), |
| ) |
| |
| log_data = lambda name: api.gitiles.log( |
| "%s.log %s" % (name, project_names[name]), "A" |
| ) |
| |
| crashpad_check_data = api.jiri.read_manifest_element( |
| manifest="fuchsia/third_party/flower", |
| element_name=project_names["crashpad"], |
| element_type="project", |
| test_output={"remote": "https://chromium.googlesource.com/crashpad/crashpad"}, |
| nesting="crashpad", |
| ) |
| crashpad_log_data = log_data("crashpad") |
| |
| mini_chromium_check_data = api.jiri.read_manifest_element( |
| manifest="fuchsia/third_party/flower", |
| element_name=project_names["mini_chromium"], |
| element_type="project", |
| test_output={"remote": "https://chromium.googlesource.com/crashpad/crashpad"}, |
| nesting="mini_chromium", |
| ) |
| mini_chromium_log_data = log_data("mini_chromium") |
| |
| lss_check_data = api.jiri.read_manifest_element( |
| manifest="fuchsia/third_party/flower", |
| element_name=project_names["lss"], |
| element_type="project", |
| test_output={"remote": "https://chromium.googlesource.com/crashpad/crashpad"}, |
| nesting="lss", |
| ) |
| lss_log_data = log_data("lss") |
| yield ( |
| api.status_check.test("noop_roll") |
| + api.properties( |
| revision="abc123", |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| ) |
| + crashpad_check_data |
| + noop_edit("crashpad") |
| ) |
| |
| yield ( |
| api.status_check.test("DEPS_parsing_error", status="infra_failure") |
| + api.properties( |
| revision="abc123", |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + api.step_data( |
| "read DEPS file for mini_chromium", |
| api.raw_io.output_text("invalid content"), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("missing_revision") |
| + api.properties( |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| ) |
| + api.git.get_remote_branch_head( |
| "git ls-remote", "fc4dc762688d2263b254208f444f5c0a4b91bc07" |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + noop_edit("mini_chromium") |
| + lss_check_data |
| + noop_edit("lss") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.status_check.test("crashpad_roll_with_lockfile") |
| + api.properties( |
| revision="abc123", |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| lockfiles=["manifest/topaz=jiri.lock"], |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + noop_edit("mini_chromium") |
| + lss_check_data |
| + noop_edit("lss") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.status_check.test("crashpad") |
| + api.properties( |
| revision="abc123", |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + noop_edit("mini_chromium") |
| + lss_check_data |
| + noop_edit("lss") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.status_check.test("crashpad_and_mini_chromium") |
| + api.properties( |
| revision="abc123", |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + mini_chromium_log_data |
| + lss_check_data |
| + noop_edit("lss") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.status_check.test("crashpad_mini_chromium_and_lss") |
| + api.properties( |
| revision="abc123", |
| project="integration", |
| manifest="fuchsia/minimal", |
| remote="sso://integration", |
| ) |
| + crashpad_check_data |
| + crashpad_log_data |
| + mini_chromium_check_data |
| + mini_chromium_log_data |
| + lss_check_data |
| + lss_log_data |
| + api.auto_roller.success() |
| ) |
