recipes/tools.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building and publishing tools."""

from recipe_engine.recipe_api import Property

import os

DEPS = [
    'fuchsia/checkout',
    'fuchsia/jiri',
    'fuchsia/git',
    'fuchsia/go',
    'fuchsia/upload',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/isolated',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/url',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/step',
]

PROPERTIES = {
    'project':
        Property(kind=str, help='Jiri remote manifest project', default=None),
    'manifest':
        Property(kind=str, help='Jiri manifest to use'),
    'remote':
        Property(kind=str, help='Remote manifest repository'),
    'publish':
        Property(kind=bool, help='Whether to publish the tools', default=False),
}

# The current list of platforms that we build for.
GO_OS_ARCH = (
    ('linux', 'amd64'),
    ('linux', 'arm64'),
    ('darwin', 'amd64'),
)


def list_main_packages(api, path):
  """Returns the 'main' go packages seq(str) on a given path (str)."""
  # Template to pass to `go list` under -f; the listed output will be in that
  # format. See https://golang.org/pkg/cmd/go/internal/list/ for more details.
  # While it might seem cleaner to use the -json list option over templating,
  # the output format is not parsable by python's json.loads()
  list_entry_template = '{{ .ImportPath  }},{{ .Name }}'
  list_entries = api.go(
      'list', '-f', list_entry_template, path,
      stdout=api.raw_io.output()).stdout.splitlines()
  main_packages = []
  for entry in list_entries:
    pkg, name = entry.split(',')
    if name == 'main':
      main_packages.append(pkg)
  return tuple(main_packages)


def tool_name(pkg_path):
  """Parses a package's source path to determine the associated tool's name."""
  tokens = pkg_path.split('/')
  assert 'cmd' in tokens, 'See https://github.com/golang-standards/project-layout#cmd'
  tokens.remove('cmd')
  # This assumes the main package is defined within a subdirectory structure of
  # $name/cmd/ or cmd/$name.
  return tokens[-1]


def RunSteps(api, project, manifest, remote, publish):
  build_input = api.buildbucket.build.input

  with api.context(infra_steps=True):
    api.checkout.with_options(
        path=api.path['start_dir'],
        manifest=manifest,
        remote=remote,
        project=project,
        build_input=build_input,
    )
    revision = api.jiri.project([project]).json.output[0]['revision']

  path = api.jiri.project([project]).json.output[0]['path']
  staging_dir = api.path.mkdtemp('staging')

  with api.context(cwd=api.path['start_dir'].join(path)):
    # Build everything before running tests. Otherwise `go test` will itself
    # do the building and a build error might confusingly be presented as a
    # test error.
    api.go('build', '-v', './...')
    api.go('test', '-v', './...')

    for pkg in list_main_packages(api, './...'):
      with api.step.nest(pkg):
        for goos, goarch in GO_OS_ARCH:
          env = {'GOOS': goos, 'GOARCH': goarch}
          platform = '%s-%s' % (goos.replace('darwin', 'mac'), goarch)
          with api.context(env=env), api.step.nest(platform):
            name = tool_name(pkg)
            args = [
                'build', '-o',
                staging_dir.join(name),
                '-gcflags=-trimpath=%s' % path
            ] + (['-ldflags=-s -w'] if publish else []) + [pkg]

            # Build the package.
            api.go(*args)

            # Upload the outputs.
            if publish:
              gitiles_commit = api.buildbucket.build.input.gitiles_commit
              assert gitiles_commit.host, 'we should only be uploading to CIPD in CI'

              cipd_dir = gitiles_commit.host.split('.')[0].replace('-', '_')
              api.upload.cipd_package(
                  '%s/%s/%s/%s' %
                  (cipd_dir, os.path.basename(project), name, platform),
                  staging_dir, [api.upload.FilePath(staging_dir.join(name))],
                  {'git_revision': revision},
                  repository=remote)
            else:
              api.upload.upload_isolated(staging_dir)


def GenTests(api):
  packages = [
      'go.fuchsia.dev/tools/cmd/gndoc', 'go.fuchsia.dev/tools/symbolizer/cmd'
  ]
  list_entries = ['%s,main' % pkg for pkg in packages]
  list_step_data = api.step_data(
      'go list', stdout=api.raw_io.output('\n'.join(list_entries)))
  cipd_search_step_data = []
  for goos, goarch in GO_OS_ARCH:
    for pkg in packages:
      platform = '%s-%s' % (goos.replace('darwin', 'mac'), goarch)
      cipd_search_step_data.append(
          api.step_data(
              '{0}.{1}.cipd.cipd search fuchsia/tools/{2}/{1} git_revision:{3}'
              .format(pkg, platform, tool_name(pkg), api.jiri.example_revision),
              api.json.output({'result': []})))

  yield (api.test('publish_new_pkg') + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/tools',
      revision=api.jiri.example_revision,
  ) + api.properties(
      project='tools',
      manifest='tools',
      remote='https://fuchsia.googlesource.com/tools',
      packages=packages,
      publish=True) + list_step_data +
         reduce(lambda a, b: a + b, cipd_search_step_data))

  yield (api.test('publish_existing_pkg') + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/tools',
      revision=api.jiri.example_revision,
  ) + api.properties(
      project='tools',
      manifest='tools',
      remote='https://fuchsia.googlesource.com/tools',
      packages=packages,
      publish=True) + list_step_data)

  yield (api.test('ci') + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/tools',
      revision=api.jiri.example_revision,
  ) + api.properties(
      project='tools',
      manifest='tools',
      remote='https://fuchsia.googlesource.com/tools',
      packages=packages) + list_step_data)

  yield (api.test('cq_try') + api.buildbucket.try_build(
      git_repo='https://fuchsia.googlesource.com/tools',) + api.properties(
          project='tools',
          manifest='tools',
          remote='https://fuchsia.googlesource.com/tools',
          packages=packages,
      ) + list_step_data)