| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for running a specified script from a given repo.""" |
| |
| from urlparse import urlparse |
| |
| from PB.recipes.fuchsia.run_script import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "fuchsia/repo", |
| "fuchsia/sso", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/python", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps(api, props): |
| checkout_dir = api.path["start_dir"].join("checkout") |
| with api.step.nest("checkout"), api.context(infra_steps=True): |
| if props.checkout_with_repo: |
| # For repo checkouts, Git must be configured to rewrite SSO URLs to |
| # HTTPS. |
| if props.remote.startswith("sso://"): |
| api.sso.configure_insteadof(props.remote) |
| api.repo.checkout_from_build_input( |
| props.remote, |
| path=checkout_dir, |
| fallback_ref=props.fallback_ref, |
| ) |
| else: |
| api.git.checkout_from_build_input( |
| repo=props.remote, |
| path=checkout_dir, |
| fallback_ref=props.fallback_ref, |
| ) |
| |
| with api.context(cwd=checkout_dir): |
| script_args = list(props.script_args) |
| if props.upload_to_cipd: |
| # Script must emit a newline-separated txt file of CIPD yaml paths, |
| # e.g. "/path/to/foo.yaml\n/path/to/bar.yaml" relative to the |
| # working dir i.e. the checkout dir. |
| assert ( |
| api.buildbucket.build.input.gitiles_commit.project |
| ), "we should only be uploading in CI" |
| cipd_yaml_manifest = api.path.mkstemp("cipd_manifest") |
| script_args += ["--cipd-yaml-manifest", cipd_yaml_manifest] |
| |
| step_name = "run %s" % props.script |
| if props.script.endswith(".py"): |
| # Run python scripts with vpython, rather than system Python, to |
| # enable installation of packages via .vpython files. |
| api.python(step_name, props.script, script_args, venv=True) |
| else: |
| api.step(step_name, [props.script] + script_args) |
| |
| if props.attempt_roll: |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(props.remote), |
| gerrit_project=urlparse(props.remote).path.lstrip("/"), |
| repo_dir=checkout_dir, |
| commit_message='Run "%s"' % props.script, |
| commit_untracked=props.roll_props.commit_untracked_files, |
| dry_run=props.roll_props.dry_run, |
| ) |
| return api.auto_roller.raw_result(change) |
| |
| if props.upload_to_cipd: |
| upload_to_cipd( |
| api, |
| checkout_dir, |
| cipd_yaml_manifest, |
| set_repo_tags=props.set_repo_tags, |
| use_json_cipd_yaml_manifest=props.use_json_cipd_yaml_manifest, |
| ) |
| |
| |
| def upload_to_cipd( |
| api, |
| checkout_dir, |
| cipd_yaml_manifest, |
| set_repo_tags=False, |
| use_json_cipd_yaml_manifest=False, |
| ): |
| """Upload package(s) to CIPD from the script-generated CIPD .yaml manifest. |
| |
| Args: |
| cipd_yaml_manifest (Path): Path to CIPD .yaml manifest. |
| set_repo_tags (bool): If True, set CIPD tags based on a repo snapshot. |
| Otherwise, set CIPD tags based on the input gitiles commit. |
| use_json_cipd_yaml_manifest (bool): If True, read the script-generated |
| CIPD .yaml manifest as JSON. |
| """ |
| if use_json_cipd_yaml_manifest: |
| # The modern JSON format is an array of objects each describing a path |
| # to a CIPD yaml file and optionally, additional refs and/or tags to |
| # attach. |
| cipd_yaml_metadata = api.file.read_json( |
| "read CIPD yaml manifest JSON", |
| cipd_yaml_manifest, |
| test_data=[ |
| { |
| "path": "path/to/foo.yaml", |
| }, |
| { |
| "path": "path/to/bar.yaml", |
| "refs": ["stable"], |
| "tags": {"git_revision": "abc"}, |
| }, |
| ], |
| ) |
| else: |
| # The deprecated format is a newline-separated file of paths to CIPD |
| # yaml files. It does not support additional refs and/or tags. |
| cipd_yaml_paths = ( |
| api.file.read_text( |
| "read CIPD yaml manifest", |
| cipd_yaml_manifest, |
| test_data="path/to/foo.yaml\npath/to/bar.yaml\n", |
| ) |
| .rstrip() |
| .split("\n") |
| ) |
| # Convert to modern JSON format without refs or tags. |
| cipd_yaml_metadata = [{"path": path} for path in cipd_yaml_paths] |
| if set_repo_tags: |
| tags = { |
| # Replace any slash characters in a project's name, as they are not |
| # allowed in CIPD tag keys. |
| project.replace("/", "_"): revision |
| for project, revision in api.repo.snapshot( |
| "repo snapshot", path=checkout_dir |
| ).iteritems() |
| } |
| |
| else: |
| tags = {"git_revision": api.buildbucket.build.input.gitiles_commit.id} |
| for metadata in cipd_yaml_metadata: |
| # Attach any additional tags and/or refs specified for each package. |
| final_tags = dict(tags) |
| final_tags.update(metadata.get("tags", {})) |
| api.cipd.create_from_yaml( |
| checkout_dir.join(metadata["path"]), |
| refs=["latest"] + metadata.get("refs", []), |
| tags=final_tags, |
| ) |
| |
| |
| def GenTests(api): |
| remote = "https://fuchsia.googlesource.com/foo" |
| sso_remote = "sso://fuchsia/foo" |
| |
| yield ( |
| api.status_check.test("ci") |
| + api.properties(script="run-tests.sh", remote=remote) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.status_check.test("ci_with_repo") |
| + api.properties( |
| script="run-tests.sh", |
| remote=remote, |
| checkout_with_repo=True, |
| upload_to_cipd=True, |
| set_repo_tags=True, |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| + api.repo.snapshot("repo snapshot", ["project/a", "project/b"]) |
| ) |
| |
| yield ( |
| api.status_check.test("sso") |
| + api.properties( |
| script="run-tests.py", |
| remote=sso_remote, |
| checkout_with_repo=True, |
| script_args=["-flag", "flagval"], |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.status_check.test("ci_upload") |
| + api.properties(script="build-pkg.sh", remote=remote, upload_to_cipd=True) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.status_check.test("use_json_cipd_manifest") |
| + api.properties( |
| script="build-pkg.sh", |
| remote=remote, |
| upload_to_cipd=True, |
| use_json_cipd_yaml_manifest=True, |
| ) |
| + api.buildbucket.ci_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.status_check.test("cq") |
| + api.properties(script="run-tests.sh", remote=remote) |
| + api.buildbucket.try_build(git_repo=remote) |
| ) |
| |
| yield ( |
| api.status_check.test("script_failed", status="failure") |
| + api.properties(script="run-tests.sh", remote=remote) |
| + api.buildbucket.ci_build(git_repo=remote) |
| + api.step_data("run run-tests.sh", retcode=1) |
| ) |
| |
| yield ( |
| api.status_check.test("no_buildbucket_input") |
| + api.properties(script="run-tests.sh", remote=remote) |
| ) |
| |
| yield ( |
| api.status_check.test("attempt_roll") |
| + api.properties(script="update.sh", remote=remote, attempt_roll=True) |
| + api.auto_roller.success() |
| ) |