recipes/3p_prebuilts.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# found in the LICENSE file.

# A general-purpose recipe for uploading third-party prebuilts to CIPD.
#
# This recipe assumes that
# (1) everything needed to produce a prebuilt can be obtained from a given
#     jiri checkout.
# (2) in that checkout, a JSON file of "contents" is found that describes all
#     of the file i/o that the recipe should perform from the checkout to a
#     staging directory. The schema is that of CONTENTS_JSON_TEST_DATA below.
# (3) also in that checkout, there is a script that takes a single positional
#     argument of a directory that it is meant to populate; this directory
#     will be uploaded to CIPD after script invocation. The script may assume
#     that the above staging directory will be set as its CWD.
#

from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/archive",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/jiri",
    "fuchsia/status_check",
    "fuchsia/upload",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = {
    "project": Property(kind=str, help="Jiri remote manifest project"),
    "manifest": Property(kind=str, help="Jiri manifest to use"),
    "remote": Property(kind=str, help="Remote manifest repository"),
    "content_manifest": Property(
        kind=str,
        help="A JSON file describing the i/o that the recipe should undertake to prepare a staging directory.",
        default="contents.json",
    ),
    "packing_script": Property(
        kind=str,
        help="A script that will consume a staging directory and populate another directory to be uploaded to CIPD.",
    ),
    "cipd_pkg_name": Property(
        kind=str,
        help="pairings of the form '<src>=<CIPD path>'; see recipe documentation for more details",
    ),
}

CONTENTS_JSON_TEST_DATA = [
    {
        # `destination` is a directory relative to the packing directory.
        "destination": "baz",
        # `files` is a list of files to move to `destination`, each relative to
        # the checkout root,
        "files": ["data/bar.txt", "data/foo.json"],
    },
]


def RunSteps(
    api, project, manifest, remote, content_manifest, packing_script, cipd_pkg_name
):
    checkout_root = api.path["start_dir"].join("3p_prebuilts")
    api.checkout.with_options(
        path=checkout_root, manifest=manifest, remote=remote, project=project,
    )
    with api.context(infra_steps=True, cwd=checkout_root):
        revision = api.checkout.project(project)["revision"]

    staging_dir = api.path.mkdtemp("3p_prebuilts.staging")

    with api.step.nest("gather contents"):
        contents = api.file.read_json(
            "read %s" % api.path.basename(content_manifest),
            checkout_root.join(content_manifest),
            test_data=CONTENTS_JSON_TEST_DATA,
        )
        for entry in contents:
            rel_dest = entry["destination"]
            dest = staging_dir.join(rel_dest)
            with api.step.nest(rel_dest if rel_dest else "."):
                api.file.ensure_directory("ensure %s" % rel_dest, dest)
                for rel_src in entry["files"]:
                    src = checkout_root.join(rel_src)
                    api.file.copy("copy %s" % rel_src, src, dest)

    output_dir = api.path.mkdtemp("3p_prebuilts.output")
    with api.context(cwd=staging_dir):
        api.step("pack contents", [checkout_root.join(packing_script), output_dir])

    with api.step.nest(cipd_pkg_name):
        if api.buildbucket_util.is_tryjob:
            api.archive.upload(output_dir, step_name="isolate prebuilts")
        else:
            api.upload.cipd_package(
                pkg_name=cipd_pkg_name,
                pkg_root=output_dir,
                pkg_paths=[api.upload.DirectoryPath(output_dir)],
                search_tag={"git_revision": revision},
            )


def GenTests(api):
    properties = api.properties(
        project="third_party/prebuilts",
        manifest="third_party/prebuilts",
        remote="https://fuchsia.googlesource.com/manifest",
        packing_script="generate.sh",
        cipd_pkg_name="fuchsia/third_party/foo",
    )
    tryjob = api.buildbucket.try_build()
    ci = api.buildbucket.ci_build()

    yield api.status_check.test("try") + properties + tryjob
    yield api.status_check.test("ci_cipd_up_to_date") + properties + ci
    yield api.status_check.test("ci_cipd_new") + properties + ci + api.step_data(
        "fuchsia/third_party/foo.cipd.cipd search fuchsia/third_party/foo git_revision:%s"
        % api.jiri.example_revision,
        api.cipd.example_search("fuchsia/third_party/foo", []),
    )