recipes/tools.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building and publishing tools."""

from recipe_engine.recipe_api import Property
from recipe_engine.config import List
from recipe_engine import config

from collections import namedtuple

DEPS = [
    'infra/checkout',
    'infra/jiri',
    'infra/git',
    'infra/go',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/isolated',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/url',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/step',
]

PROPERTIES = {
    'project':
        Property(kind=str, help='Jiri remote manifest project', default=None),
    'manifest':
        Property(kind=str, help='Jiri manifest to use'),
    'remote':
        Property(kind=str, help='Remote manifest repository'),
}

# The current list of platforms that we build for.
GO_OS_ARCH = (
    ('linux', 'amd64'),
    ('linux', 'arm64'),
    ('darwin', 'amd64'),
)


def upload_isolated(api, staging_dir):
  isolated = api.isolated.isolated(staging_dir)
  isolated.add_dir(staging_dir)
  return isolated.archive('isolate')


def upload_package(api, name, platform, staging_dir, revision, remote):
  cipd_pkg_name = 'fuchsia/tools/%s/%s' % (name, platform)
  pkg_def = api.cipd.PackageDefinition(
      package_name=cipd_pkg_name,
      package_root=staging_dir,
      install_mode='copy',
  )
  pkg_def.add_file(staging_dir.join(name))
  pkg_def.add_version_file('.versions/%s.cipd_version' % name)

  cipd_pkg_file = api.path['cleanup'].join('%s.cipd' % name)
  api.cipd.build_from_pkg(
      pkg_def=pkg_def,
      output_package=cipd_pkg_file,
  )

  pins = api.cipd.search(cipd_pkg_name, 'git_revision:' + revision)
  if pins:
    api.step('Package is up-to-date', cmd=None)
    return

  step_result = api.cipd.register(
      package_name=cipd_pkg_name,
      package_path=cipd_pkg_file,
      refs=['latest'],
      tags={
          'git_repository': remote,
          'git_revision': revision,
      },
  )


def RunSteps(api, project, manifest, remote):
  api.jiri.ensure_jiri()
  api.go.ensure_go()

  build_input = api.buildbucket.build.input

  with api.context(infra_steps=True):
    api.checkout.deprecated_checkout(
        path=api.path['start_dir'],
        manifest=manifest,
        remote=remote,
        project=project,
        build_input=build_input,
    )
    revision = api.jiri.project([project]).json.output[0]['revision']

  path = api.jiri.project([project]).json.output[0]['path']
  staging_dir = api.path.mkdtemp('tools')

  with api.context(cwd=api.path.abs_to_path(path)):
    # Run all the tests.
    api.go('test', '-v', './...')

    step_result = api.go('list', './cmd/...', stdout=api.raw_io.output())
    for pkg in step_result.stdout.splitlines():
      with api.step.nest(pkg):
        for goos, goarch in GO_OS_ARCH:
          env = {'GOOS': goos, 'GOARCH': goarch}
          platform = '%s-%s' % (goos.replace('darwin', 'mac'), goarch)
          with api.context(env=env), api.step.nest(platform):
            output = pkg.split('/')[-1]

            args = [
                'build', '-o',
                staging_dir.join(output),
                '-gcflags=-trimpath=%s' % path
            ] + (['-ldflags=-s -w'] if
                 api.buildbucket.builder_id.bucket == 'prod' else []) + [pkg]

            # Build the package.
            api.go(*args)

            # Upload the package to CIPD.
            if api.buildbucket.builder_id.bucket == 'ci':
              upload_isolated(api, staging_dir)
            elif api.buildbucket.builder_id.bucket == 'prod':
              upload_package(api, output, platform, staging_dir, revision,
                             remote)


def GenTests(api):
  packages = [
      'fuchsia.googlesource.com/tools/gndoc',
      'fuchsia.googlesource.com/tools/symbolizer'
  ]
  list_step_data = api.step_data(
      'go list', stdout=api.raw_io.output('\n'.join(packages)))
  cipd_search_step_data = []
  for goos, goarch in GO_OS_ARCH:
    for pkg in packages:
      platform = '%s-%s' % (goos.replace('darwin', 'mac'), goarch)
      cipd_search_step_data.append(
          api.step_data(
              '{0}.{1}.cipd search fuchsia/tools/{2}/{1} git_revision:{3}'
              .format(pkg, platform,
                      pkg.split('/')[-1], api.jiri.example_revision),
              api.json.output({
                  'result': []
              })))

  yield (api.test('prod_new') + api.buildbucket.ci_build(
      bucket='prod',
      git_repo='https://fuchsia.googlesource.com/tools',
      revision=api.jiri.example_revision,
  ) + api.properties(
      project='tools',
      manifest='tools',
      remote='https://fuchsia.googlesource.com/tools',
      packages=packages) + list_step_data + reduce(lambda a, b: a + b,
                                                   cipd_search_step_data))

  yield (api.test('prod') + api.buildbucket.ci_build(
      bucket='prod',
      git_repo='https://fuchsia.googlesource.com/tools',
      revision=api.jiri.example_revision,
  ) + api.properties(
      project='tools',
      manifest='tools',
      remote='https://fuchsia.googlesource.com/tools',
      packages=packages) + list_step_data)

  yield (api.test('ci') + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/tools',
      revision=api.jiri.example_revision,
  ) + api.properties(
      project='tools',
      manifest='tools',
      remote='https://fuchsia.googlesource.com/tools',
      packages=packages) + list_step_data)

  yield (api.test('cq_try') + api.buildbucket.try_build(
      git_repo='https://fuchsia.googlesource.com/tools',) + api.properties(
          project='tools',
          manifest='tools',
          remote='https://fuchsia.googlesource.com/tools',
          packages=packages,
          tryjob=True,
      ) + list_step_data)