| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building and publishing tools.""" |
| |
| from recipe_engine.recipe_api import Property |
| |
| import os |
| |
| DEPS = [ |
| 'fuchsia/checkout', |
| 'fuchsia/jiri', |
| 'fuchsia/git', |
| 'fuchsia/go', |
| 'fuchsia/upload', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/isolated', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/url', |
| 'recipe_engine/platform', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use'), |
| 'remote': |
| Property(kind=str, help='Remote manifest repository'), |
| 'output_name_map': |
| Property( |
| kind=dict, help='Mapping from tool to binary name', default={}), |
| 'publish': |
| Property(kind=bool, help='Whether to publish the tools', default=False), |
| } |
| |
| # The current list of platforms that we build for. |
| GO_OS_ARCH = ( |
| ('linux', 'amd64'), |
| ('linux', 'arm64'), |
| ('darwin', 'amd64'), |
| ) |
| |
| |
| def list_main_packages(api, path): |
| """Returns the 'main' go packages seq(str) on a given path (str).""" |
| # Template to pass to `go list` under -f; the listed output will be in that |
| # format. See https://golang.org/pkg/cmd/go/internal/list/ for more details. |
| # While it might seem cleaner to use the -json list option over templating, |
| # the output format is not parsable by python's json.loads() |
| list_entry_template = '{{ .ImportPath }},{{ .Name }}' |
| list_entries = api.go( |
| 'list', '-f', list_entry_template, path, |
| stdout=api.raw_io.output()).stdout.splitlines() |
| main_packages = [] |
| for entry in list_entries: |
| pkg, name = entry.split(',') |
| if name == 'main': |
| main_packages.append(pkg) |
| return tuple(main_packages) |
| |
| |
| def tool_name(pkg_path): |
| """Parses a package's source path to determine the associated tool's name.""" |
| tokens = pkg_path.split('/') |
| assert 'cmd' in tokens, 'See https://github.com/golang-standards/project-layout#cmd' |
| tokens.remove('cmd') |
| # This assumes the main package is defined within a subdirectory structure of |
| # $name/cmd/ or cmd/$name. |
| return tokens[-1] |
| |
| |
| def RunSteps(api, project, manifest, remote, output_name_map, publish): |
| build_input = api.buildbucket.build.input |
| |
| with api.context(infra_steps=True): |
| api.checkout.with_options( |
| path=api.path['start_dir'], |
| manifest=manifest, |
| remote=remote, |
| project=project, |
| build_input=build_input, |
| ) |
| revision = api.jiri.project([project]).json.output[0]['revision'] |
| |
| path = api.jiri.project([project]).json.output[0]['path'] |
| staging_dir = api.path.mkdtemp('staging') |
| |
| with api.context(cwd=api.path['start_dir'].join(path)): |
| # Build everything before running tests. Otherwise `go test` will itself |
| # do the building and a build error might confusingly be presented as a |
| # test error. |
| api.go('build', '-v', './...') |
| api.go('test', '-v', './...') |
| |
| for pkg in list_main_packages(api, './...'): |
| with api.step.nest(pkg): |
| for goos, goarch in GO_OS_ARCH: |
| env = {'GOOS': goos, 'GOARCH': goarch} |
| platform = '%s-%s' % (goos.replace('darwin', 'mac'), goarch) |
| with api.context(env=env), api.step.nest(platform): |
| name = tool_name(pkg) |
| # Rename the tool binary if a renaming is specified. |
| binary_name = output_name_map.get(name, name) |
| binary_path = staging_dir.join(binary_name) |
| args = ['build', '-o', binary_path, |
| '-gcflags=-trimpath=%s' % path |
| ] + (['-ldflags=-s -w'] if publish else []) + [pkg] |
| |
| # Build the package. |
| api.go(*args) |
| |
| # Upload the outputs. |
| if publish: |
| gitiles_commit = api.buildbucket.build.input.gitiles_commit |
| assert gitiles_commit.host, 'we should only be uploading to CIPD in CI' |
| |
| cipd_dir = gitiles_commit.host.split('.')[0].replace('-', '_') |
| api.upload.cipd_package( |
| '%s/%s/%s/%s' % |
| (cipd_dir, os.path.basename(project), name, platform), |
| staging_dir, [api.upload.FilePath(binary_path)], |
| {'git_revision': revision}, |
| repository=remote) |
| else: |
| api.upload.upload_isolated(staging_dir) |
| |
| |
| def GenTests(api): |
| packages = [ |
| 'go.fuchsia.dev/tools/cmd/gndoc', |
| 'go.fuchsia.dev/tools/symbolizer/cmd', |
| ] |
| output_name_map = { |
| 'gndoc': 'renamed_gndoc', |
| } |
| list_entries = ['%s,main' % pkg for pkg in packages] |
| list_step_data = api.step_data( |
| 'go list', stdout=api.raw_io.output('\n'.join(list_entries))) |
| cipd_search_step_data = [] |
| for goos, goarch in GO_OS_ARCH: |
| for pkg in packages: |
| platform = '%s-%s' % (goos.replace('darwin', 'mac'), goarch) |
| cipd_search_step_data.append( |
| api.step_data( |
| '{0}.{1}.cipd.cipd search fuchsia/tools/{2}/{1} git_revision:{3}' |
| .format(pkg, platform, tool_name(pkg), api.jiri.example_revision), |
| api.json.output({'result': []}))) |
| |
| yield (api.test('publish_new_pkg') + api.buildbucket.ci_build( |
| git_repo='https://fuchsia.googlesource.com/tools', |
| revision=api.jiri.example_revision, |
| ) + api.properties( |
| project='tools', |
| manifest='tools', |
| remote='https://fuchsia.googlesource.com/tools', |
| packages=packages, |
| output_name_map=output_name_map, |
| publish=True) + list_step_data + |
| reduce(lambda a, b: a + b, cipd_search_step_data)) |
| |
| yield (api.test('publish_existing_pkg') + api.buildbucket.ci_build( |
| git_repo='https://fuchsia.googlesource.com/tools', |
| revision=api.jiri.example_revision, |
| ) + api.properties( |
| project='tools', |
| manifest='tools', |
| remote='https://fuchsia.googlesource.com/tools', |
| packages=packages, |
| output_name_map=output_name_map, |
| publish=True) + list_step_data) |
| |
| yield (api.test('ci') + api.buildbucket.ci_build( |
| git_repo='https://fuchsia.googlesource.com/tools', |
| revision=api.jiri.example_revision, |
| ) + api.properties( |
| project='tools', |
| manifest='tools', |
| remote='https://fuchsia.googlesource.com/tools', |
| output_name_map=output_name_map, |
| packages=packages) + list_step_data) |
| |
| yield (api.test('cq_try') + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/tools',) + api.properties( |
| project='tools', |
| manifest='tools', |
| remote='https://fuchsia.googlesource.com/tools', |
| packages=packages, |
| output_name_map=output_name_map, |
| ) + list_step_data) |