| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for updating the code search super manifest project.""" |
| |
| from PB.recipes.fuchsia.code_search import InputProperties |
| |
| DEPS = [ |
| "fuchsia/checkout", |
| "fuchsia/git", |
| "fuchsia/git_checkout", |
| "fuchsia/jiri", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| GENERATE_SCRIPT = "setup.sh" |
| |
| LOCAL_BRANCH_NAME = "latest" |
| |
| |
| def RunSteps(api, props): |
| checkout = api.checkout.fuchsia_with_options( |
| manifest=props.manifest, |
| remote=props.remote, |
| fetch_packages=False, |
| ) |
| |
| if props.preserve_root: |
| with api.context(cwd=checkout.root_dir): |
| api.git.raw_checkout(branch=LOCAL_BRANCH_NAME) |
| |
| with api.step.nest("checkout super manifest project"): |
| super_manifest_project_path = checkout.root_dir / "super" |
| api.file.ensure_directory("make dirs", super_manifest_project_path) |
| |
| _, base_revision = api.git_checkout( |
| props.super_manifest_project, |
| path=super_manifest_project_path, |
| submodules=False, |
| ignore_build_input=True, |
| ) |
| |
| with api.context(cwd=super_manifest_project_path): |
| with api.step.nest("reset submodule state"): |
| submodule_configs = ( |
| api.git.config( |
| "--name-only", |
| "--null", |
| "--file", |
| ".gitmodules", |
| "--get-regexp", |
| "submodule.*path", |
| step_name="get submodule configs", |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "submodule.a.path\x00submodule.b.path" |
| ), |
| ) |
| .stdout.rstrip("\x00") |
| .split("\x00") |
| ) |
| for submodule_config in submodule_configs: |
| submodule_path = api.git.config( |
| "--null", |
| "--file", |
| ".gitmodules", |
| "--get", |
| submodule_config, |
| step_name="get submodule path", |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "path/to/submodule\x00" |
| ), |
| ).stdout.rstrip("\x00") |
| api.git( |
| f"rm {submodule_path}", |
| "rm", |
| "--cached", |
| "--", |
| submodule_path, |
| ok_ret="any", |
| ) |
| submodule_section_name = submodule_config.removesuffix(".path") |
| api.git.config( |
| "--file", |
| ".gitmodules", |
| "--remove-section", |
| submodule_section_name, |
| step_name=f"remove section {submodule_section_name}", |
| ) |
| api.git.add(add_all=True, force=True) |
| |
| # For details, see: |
| # https://fuchsia.googlesource.com/jiri/+/main/cmd/jiri/generate-gitmodules.go |
| generate_script_path = super_manifest_project_path / GENERATE_SCRIPT |
| api.jiri.generate_gitmodules( |
| ".gitmodules", |
| generate_script=generate_script_path, |
| redirect_root=not props.preserve_root, |
| ) |
| |
| api.step(f"run {GENERATE_SCRIPT}", [generate_script_path]) |
| api.file.remove(f"remove {GENERATE_SCRIPT}", generate_script_path) |
| |
| api.git.add(add_all=True, force=True) |
| # Only commit if we have a diff, otherwise commit fails. |
| if api.git.diff( |
| ref_base=base_revision, cached=True, exit_code=True, ok_ret="any" |
| ).retcode: |
| api.git.commit( |
| message=f"Update submodules to {checkout.integration_revision}" |
| ) |
| |
| if props.preserve_root: |
| # Add our local fuchsia.git checkout as a new remote of the super |
| # manifest project's Git repo, and merge in the history. |
| source_remote = "local-upstream" |
| api.git.remote_add(source_remote, checkout.root_dir) |
| api.git.fetch(source_remote) |
| api.git( |
| "git merge", |
| "merge", |
| "-m", |
| "merge upstream changes", |
| "-X", |
| "theirs", |
| "--allow-unrelated-histories", |
| f"{source_remote}/{LOCAL_BRANCH_NAME}", |
| ) |
| # The preceding merge can lead to duplicate entries in the |
| # .gitmodules file when the submodule updater has updated the same |
| # lines. This cleans up the merge by removing the duplicates. |
| with api.step.nest("dedupe gitmodules entries"): |
| deduped = dedupe_gitmodules(api) |
| |
| # Once the deduplication is done, we have to add the .gitmodules |
| # file changes to the merge commit. |
| if deduped: |
| api.git.commit(no_edit=True, amend=True, all_tracked=True) |
| |
| # Only push if we have a diff. |
| if api.git.diff( |
| step_name="check for push", |
| ref_base=base_revision, |
| cached=True, |
| exit_code=True, |
| ok_ret="any", |
| ).retcode: |
| # Skip pushing entirely instead of setting --dry-run because a dry |
| # run push may fail if it races with a production build. |
| if not props.dry_run: |
| api.git.push( |
| "HEAD:main", |
| options=["nokeycheck", "skip-validation"], |
| ) |
| |
| |
| def dedupe_gitmodules(api): |
| seen = {} |
| deduped_something = False |
| submodule_configs = ( |
| api.git.config( |
| "--name-only", |
| "--null", |
| "--file", |
| ".gitmodules", |
| "--get-regexp", |
| "submodule.*path", |
| step_name="get submodule configs", |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "\x00".join( |
| [ |
| "submodule.a.path", |
| "submodule.b.path", |
| "submodule.c.path", |
| "submodule.a.path", |
| ] |
| ) |
| ), |
| ) |
| .stdout.rstrip("\x00") |
| .split("\x00") |
| ) |
| for submodule_config in submodule_configs: |
| if seen.get(submodule_config, False): |
| deduped_something = True |
| # We can't just remove the submodule config because the remove-section |
| # command removes _all_ copies of the section. Therefore, we need to |
| # store one copy of the variables here, delete all copies, then add a |
| # single copy. |
| submodule_section_name = submodule_config.removesuffix(".path") |
| |
| # Retrieve the existing configuration values. |
| raw_section_values = ( |
| api.git.config( |
| "--null", |
| "--file", |
| ".gitmodules", |
| "--get-regexp", |
| f"{submodule_section_name}.*", |
| step_name=f"get section config for {submodule_section_name}", |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "\x00".join( |
| [ |
| "submodule.a.path path", |
| "submodule.a.url url", |
| "submodule.a.name name", |
| "submodule.a.ignore all", |
| ] |
| ) |
| ), |
| ) |
| .stdout.rstrip("\x00") |
| .split("\x00") |
| ) |
| section_vars = {} |
| for section_val in raw_section_values: |
| vals = section_val.split() |
| section_vars[vals[0]] = vals[1] |
| |
| # Remove all copies of the section from the file. |
| api.git.config( |
| "--file", |
| ".gitmodules", |
| "--remove-section", |
| submodule_section_name, |
| step_name=f"remove section {submodule_section_name}", |
| ) |
| |
| # Add a single copy of the variables back. |
| for k, v in section_vars.items(): |
| api.git.config( |
| "--file", |
| ".gitmodules", |
| "--add", |
| k, |
| v, |
| step_name=f"adding variable {k} {v}", |
| ) |
| seen[submodule_config] = True |
| |
| return deduped_something |
| |
| |
| def GenTests(api): |
| source_info = [ |
| { |
| "name": "integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": "integration", |
| }, |
| ] |
| |
| def props(**kwargs): |
| return api.properties( |
| remote="https://fuchsia.googlesource.com/integration", |
| manifest="flower", |
| super_manifest_project="https://fuchsia.googlesource.com/super", |
| **kwargs, |
| ) |
| |
| yield api.test("no_diff") + props() + api.checkout.source_info(source_info) |
| |
| yield api.test("yes_diff") + props() + api.step_data( |
| "git diff", retcode=1 |
| ) + api.checkout.source_info(source_info) + api.step_data( |
| "check for push", retcode=1 |
| ) |
| |
| yield api.test("dry_run") + props(dry_run=True) + api.step_data( |
| "git diff", retcode=1 |
| ) + api.checkout.source_info(source_info) + api.step_data( |
| "check for push", retcode=1 |
| ) |
| |
| yield api.test("preserve_root") + props( |
| preserve_root=True |
| ) + api.checkout.source_info(source_info) |
