recipes/3p_prebuilts.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# found in the LICENSE file.

# A general-purpose recipe for uploading third-party prebuilts to CIPD.
#
# This recipe assumes that
# (1) everything needed to produce a prebuilt can be obtained from a given
#     jiri checkout.
# (2) in that checkout, a JSON file of "contents" is found that describes all
#     of the file i/o that the recipe should perform from the checkout to a
#     staging directory. The schema is that of CONTENTS_JSON_TEST_DATA below.
# (3) also in that checkout, there is a script that takes a single positional
#     argument of a directory that it is meant to populate; this directory
#     will be uploaded to CIPD after script invocation. The script may assume
#     that the above staging directory will be set as its CWD.
#

from recipe_engine.recipe_api import Property

DEPS = [
    'fuchsia/buildbucket_util',
    'fuchsia/checkout',
    'fuchsia/git',
    'fuchsia/jiri',
    'fuchsia/upload',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/isolated',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/step',
]

PROPERTIES = {
    'project':
        Property(kind=str, help='Jiri remote manifest project'),
    'manifest':
        Property(kind=str, help='Jiri manifest to use'),
    'remote':
        Property(kind=str, help='Remote manifest repository'),
    'content_manifest':
        Property(
            kind=str,
            help='A JSON file describing the i/o that the recipe should undertake to prepare a staging directory.',
            default='contents.json',
        ),
    'packing_script':
        Property(
            kind=str,
            help='A script that will consume a staging directory and populate another directory to be uploaded to CIPD.',
        ),
    'cipd_pkg_name':
        Property(
            kind=str,
            help='pairings of the form \'<src>=<CIPD path>\'; see recipe documentation for more details',
        ),
}

CONTENTS_JSON_TEST_DATA = [
    {
        # `destination` is a directory relative to the packing directory.
        'destination': 'baz',
        # `files` is a list of files to move to `destination`, each relative to
        # the checkout root,
        'files': ['data/bar.txt', 'data/foo.json'],
    },
]


def RunSteps(api, project, manifest, remote, content_manifest, packing_script,
             cipd_pkg_name):
  checkout_root = api.path['start_dir'].join('3p_prebuilts')
  with api.step.nest('checkout'):
    with api.context(infra_steps=True):
      api.checkout.with_options(
          path=checkout_root,
          manifest=manifest,
          remote=remote,
          project=project,
          build_input=api.buildbucket.build_input,
      )
    with api.context(infra_steps=True, cwd=checkout_root):
      revision = api.jiri.project([project]).json.output[0]['revision']

  staging_dir = api.path.mkdtemp('3p_prebuilts.staging')

  with api.step.nest('gather contents'):
    contents = api.file.read_json(
        'read %s' % api.path.basename(content_manifest),
        checkout_root.join(content_manifest),
        test_data=CONTENTS_JSON_TEST_DATA,
    )
    for entry in contents:
      rel_dest = entry['destination']
      dest = staging_dir.join(rel_dest)
      with api.step.nest(rel_dest):
        api.file.ensure_directory('ensure %s' % rel_dest, dest)
        for rel_src in entry['files']:
          src = checkout_root.join(rel_src)
          api.file.copy('copy %s' % rel_src, src, dest)

  output_dir = api.path.mkdtemp('3p_prebuilts.output')
  with api.context(cwd=staging_dir):
    api.step('pack contents', [checkout_root.join(packing_script), output_dir])

  with api.step.nest(cipd_pkg_name):
    if api.buildbucket_util.is_tryjob:
      isolated = api.isolated.isolated(output_dir)
      isolated.add_dir(output_dir)
      isolated.archive('isolate prebuilts')
    else:
      api.upload.cipd_package(
          pkg_name=cipd_pkg_name,
          pkg_root=output_dir,
          pkg_paths=[api.upload.DirectoryPath(output_dir)],
          search_tag={'git_revision': revision},
      )


def GenTests(api):
  properties = api.properties(
      project='third_party/prebuilts',
      manifest='third_party/prebuilts',
      remote='https://fuchsia.googlesource.com/manifest',
      packing_script='generate.sh',
      cipd_pkg_name='fuchsia/third_party/foo',
  )
  tryjob = api.buildbucket.try_build()
  ci = api.buildbucket.ci_build()

  yield api.test('try') + properties + tryjob
  yield api.test('ci_cipd_up_to_date') + properties + ci
  yield api.test('ci_cipd_new') + properties + ci + api.step_data(
      'fuchsia/third_party/foo.cipd.cipd search fuchsia/third_party/foo git_revision:%s'
      % api.jiri.example_revision,
      api.cipd.example_search('fuchsia/third_party/foo', []))