| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for running a specified script from a given repo.""" |
| |
| from PB.recipes.fuchsia.run_script import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/build", |
| "fuchsia/checkout", |
| "fuchsia/git_checkout", |
| "fuchsia/repo", |
| "fuchsia/sso", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/resultdb", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| # The env var where scripts can write a file containing the |
| # commit message they'd like to use with the roll. |
| COMMIT_MESSAGE_FILE_ENV = "COMMIT_MESSAGE_FILE" |
| |
| |
| def RunSteps(api, props): |
| checkout_dir = api.path.start_dir / "checkout" |
| if props.checkout_with_repo: |
| assert ( |
| not props.shallow_checkout |
| ), "shallow checkouts are not supported for repo" |
| # For repo checkouts, Git must be configured to rewrite SSO URLs to |
| # HTTPS. |
| if props.remote.startswith("sso://"): |
| api.sso.configure_insteadof(props.remote) |
| api.repo.checkout_from_build_input( |
| props.remote, |
| path=checkout_dir, |
| fallback_ref=props.fallback_ref, |
| ) |
| elif props.jiri_manifest: |
| assert ( |
| not props.shallow_checkout |
| ), "shallow checkouts are not supported for Jiri" |
| if props.fint_params_path: |
| checkout = api.checkout.fuchsia_with_options( |
| path=checkout_dir, |
| manifest=props.jiri_manifest, |
| remote=props.remote, |
| project=props.jiri_project, |
| ) |
| api.build.with_options( |
| checkout=checkout, fint_params_path=props.fint_params_path |
| ) |
| else: |
| api.checkout.with_options( |
| path=checkout_dir, |
| manifest=props.jiri_manifest, |
| remote=props.remote, |
| project=props.jiri_project, |
| ) |
| else: |
| api.git_checkout( |
| repo=props.remote, |
| path=checkout_dir, |
| fallback_ref=props.fallback_ref, |
| depth=1 if props.shallow_checkout else None, |
| ) |
| |
| tempdir = api.path.mkdtemp() |
| commit_msg_file = tempdir / "commit_message.txt" |
| env = {COMMIT_MESSAGE_FILE_ENV: commit_msg_file} |
| with api.context(cwd=checkout_dir, env=env): |
| script_configs = [] |
| if props.script_configs: |
| script_configs = props.script_configs |
| |
| for script_config in script_configs: |
| process_script(api, script_config, props, checkout_dir) |
| |
| if props.attempt_roll: |
| commit_message = "" |
| api.path.mock_add_paths(commit_msg_file) |
| if api.path.exists(commit_msg_file): |
| commit_message = api.file.read_text( |
| "read commit message file", commit_msg_file |
| ) |
| # If the commit message file does not exist or is empty, |
| # use default commit message. |
| if not commit_message: |
| commit_message = ( |
| f"Run \"{','.join([config.script for config in script_configs])}\"" |
| ) |
| repo_dir = checkout_dir |
| if props.roll_dir: |
| repo_dir = repo_dir / props.roll_dir |
| change = api.auto_roller.attempt_roll( |
| props.roll_options, |
| repo_dir=repo_dir, |
| commit_message=commit_message, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| |
| def process_script(api, script_config, props, checkout_dir): |
| cmd = [checkout_dir / script_config.script] |
| |
| cmd.extend(script_config.script_args) |
| |
| if script_config.upload_to_cipd: |
| # Script must emit a newline-separated txt file of CIPD yaml paths, |
| # e.g. "/path/to/foo.yaml\n/path/to/bar.yaml" relative to the |
| # working dir i.e. the checkout dir. |
| assert ( |
| api.buildbucket.build.input.gitiles_commit.project |
| ), "we should only be uploading in CI" |
| cipd_yaml_manifest = api.path.mkstemp("cipd_manifest") |
| cmd += ["--cipd-yaml-manifest", cipd_yaml_manifest] |
| |
| api.step( |
| f"run {script_config.script}", api.resultdb.wrap(cmd, require_build_inv=False) |
| ) |
| |
| if script_config.upload_to_cipd: |
| upload_to_cipd( |
| api, |
| checkout_dir, |
| cipd_yaml_manifest, |
| set_repo_tags=props.set_repo_tags, |
| use_json_cipd_yaml_manifest=props.use_json_cipd_yaml_manifest, |
| ) |
| |
| |
| def upload_to_cipd( |
| api, |
| checkout_dir, |
| cipd_yaml_manifest, |
| set_repo_tags=False, |
| use_json_cipd_yaml_manifest=False, |
| ): |
| """Upload package(s) to CIPD from the script-generated CIPD .yaml manifest. |
| |
| Args: |
| cipd_yaml_manifest (Path): Path to CIPD .yaml manifest. |
| set_repo_tags (bool): If True, set CIPD tags based on a repo snapshot. |
| Otherwise, set CIPD tags based on the input gitiles commit. |
| use_json_cipd_yaml_manifest (bool): If True, read the script-generated |
| CIPD .yaml manifest as JSON. |
| """ |
| if use_json_cipd_yaml_manifest: |
| # The modern JSON format is an array of objects each describing a path |
| # to a CIPD yaml file and optionally, additional refs and/or tags to |
| # attach. |
| cipd_yaml_metadata = api.file.read_json( |
| "read CIPD yaml manifest JSON", |
| cipd_yaml_manifest, |
| test_data=[ |
| { |
| "path": "path/to/foo.yaml", |
| }, |
| { |
| "path": "path/to/bar.yaml", |
| "refs": ["stable"], |
| "tags": {"git_revision": "abc"}, |
| }, |
| ], |
| ) |
| else: |
| # The deprecated format is a newline-separated file of paths to CIPD |
| # yaml files. It does not support additional refs and/or tags. |
| cipd_yaml_paths = ( |
| api.file.read_text( |
| "read CIPD yaml manifest", |
| cipd_yaml_manifest, |
| test_data="path/to/foo.yaml\npath/to/bar.yaml\n", |
| ) |
| .rstrip() |
| .split("\n") |
| ) |
| # Convert to modern JSON format without refs or tags. |
| cipd_yaml_metadata = [{"path": path} for path in cipd_yaml_paths] |
| if set_repo_tags: |
| tags = { |
| # Replace any slash characters in a project's name, as they are not |
| # allowed in CIPD tag keys. |
| project.replace("/", "_"): revision |
| for project, revision in api.repo.snapshot( |
| "repo snapshot", path=checkout_dir |
| ).items() |
| } |
| else: |
| tags = {"git_revision": api.buildbucket.build.input.gitiles_commit.id} |
| for metadata in cipd_yaml_metadata: |
| # Attach any additional tags and/or refs specified for each package. |
| final_tags = dict(tags) |
| final_tags.update(metadata.get("tags", {})) |
| api.cipd.create_from_yaml( |
| checkout_dir / metadata["path"], |
| refs=["latest"] + metadata.get("refs", []), |
| tags=final_tags, |
| ) |
| |
| |
| def GenTests(api): |
| remote = "https://fuchsia.googlesource.com/foo" |
| sso_remote = "sso://fuchsia/foo" |
| |
| yield ( |
| api.test("ci") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| } |
| ], |
| remote=remote, |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.test("ci_with_jiri") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| "upload_to_cipd": True, |
| } |
| ], |
| remote=remote, |
| jiri_manifest="manifest.xml", |
| jiri_project="manifest", |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.test("ci_with_repo") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| "upload_to_cipd": True, |
| } |
| ], |
| remote=remote, |
| checkout_with_repo=True, |
| set_repo_tags=True, |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| + api.repo.snapshot("repo snapshot", ["project/a", "project/b"]) |
| ) |
| |
| yield ( |
| api.test("sso") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.py", |
| "script_args": ["-flag", "val"], |
| "upload_to_cipd": False, |
| }, |
| { |
| "script": "run-tests2.py", |
| "script_args": ["-flag", "val"], |
| "upload_to_cipd": True, |
| }, |
| ], |
| remote=sso_remote, |
| checkout_with_repo=True, |
| attempt_roll=True, |
| roll_options=api.auto_roller.Options(remote=sso_remote), |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.test("ci_upload") |
| + api.properties( |
| remote=remote, |
| shallow_checkout=True, |
| script_configs=[ |
| { |
| "script": "build-pkg.sh", |
| "upload_to_cipd": True, |
| } |
| ], |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.test("use_json_cipd_manifest") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "build-pkg.sh", |
| "upload_to_cipd": True, |
| } |
| ], |
| remote=remote, |
| use_json_cipd_yaml_manifest=True, |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.test("cq") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| } |
| ], |
| remote=remote, |
| ) |
| + api.buildbucket.try_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.test("script_failed", status="FAILURE") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| } |
| ], |
| remote=remote, |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| + api.step_data("run run-tests.sh", retcode=1) |
| ) |
| |
| yield ( |
| api.test("no_buildbucket_input") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| } |
| ], |
| remote=remote, |
| ) |
| ) |
| |
| yield ( |
| api.test("attempt_roll") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "update.sh", |
| } |
| ], |
| remote=remote, |
| attempt_roll=True, |
| roll_dir="foo/bar", |
| roll_options=api.auto_roller.Options(remote=remote), |
| ) |
| + api.auto_roller.success() |
| + api.step_data("read commit message file", api.file.read_text("New commit.")) |
| ) |
| |
| yield ( |
| api.test("ci_with_fuchsia_build") |
| + api.properties( |
| script_configs=[ |
| { |
| "script": "run-tests.sh", |
| } |
| ], |
| remote=remote, |
| jiri_manifest="manifest.xml", |
| jiri_project="manifest", |
| fint_params_path="fint_params", |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |