| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building goma toolchain images.""" |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/docker', |
| 'fuchsia/gcloud', |
| 'fuchsia/git', |
| 'fuchsia/goma', |
| 'fuchsia/gsutil', |
| 'recipe_engine/archive', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/platform', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| 'recipe_engine/url', |
| ] |
| |
| PROPERTIES = { |
| 'package': |
| Property( |
| kind=str, |
| help='cipd path of the toolchain package', |
| default='fuchsia/third_party/clang'), |
| 'version': |
| Property( |
| kind=str, |
| help='cipd version of the toolchain package', |
| default='git_revision:1475fad1d0d45572471575cdda8fed6eeab5b0ec'), |
| 'toolchain': |
| Property(kind=str, help='type of the toolchain', default='clang'), |
| 'platforms': |
| Property( |
| kind=List(str), |
| help='toolchain host platform', |
| default=['linux-amd64', 'mac-amd64']), |
| 'domain': |
| Property( |
| kind=str, |
| help='domain name of container registry', |
| default='gcr.io'), |
| 'project': |
| Property( |
| kind=str, help='name of goma cloud project', |
| default='goma-fuchsia'), |
| 'root': |
| Property( |
| kind=str, |
| help='root directory name for container registry', |
| default='fuchsia_linux'), |
| 'update_goma_base_images': |
| Property( |
| kind=bool, |
| help='Update goma base images even if they already exist', |
| default=False), |
| } |
| |
| GOMATOOLS_VERSION = 'git_revision:22684ee9174606073a2604e1d58fbf03ba3ad564' |
| |
| |
| def ensure_gomatools(api, gomatools_dir): |
| with api.step.nest('ensure gomatools'): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package( |
| 'fuchsia_internal/third_party/goma/server/gomatools/linux-amd64', |
| GOMATOOLS_VERSION) |
| api.cipd.ensure(gomatools_dir, pkgs) |
| |
| |
| def generate_time_stamp(api): |
| return "{:%Y%m%d_%H%M%S}".format(api.time.utcnow()) |
| |
| |
| def build_goma_base_image(api, dockerfile, image, timestamp=None): |
| api.file.copy('copy dockerfile', api.resource(dockerfile), |
| api.path['cleanup'].join(dockerfile)) |
| tags = [image] |
| if timestamp: |
| tags.append(image + ':' + timestamp) |
| with api.context(cwd=api.path['cleanup']): |
| api.docker.build( |
| dockerfile=api.path['cleanup'].join(dockerfile), |
| tags=tags, |
| ) |
| api.docker('push', image) |
| |
| |
| def ensure_goma_base_images(api, |
| domain, |
| project_id, |
| root, |
| timestamp=None, |
| update_if_exist=False): |
| """Ensure the base goma images in container registry. |
| |
| E.g. ensure_goma_base_images(api, |
| 'gcr.io',\ |
| 'goma-fuchsia',\ |
| 'fuchsia_linux') |
| will create docker images at |
| gcr.io/goma-fuchsia/fuchsia_linux/base, |
| gcr.io/goma-fuchsia/fuchsia_linux/runtime, |
| gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform, |
| using 'debian:buster-slim' as bottom layer image. |
| |
| Args: |
| * domain (str) - The domain name of the container register. |
| * project (str) - The gcloud project ID. |
| * root (str) - The name of container registry root. |
| * update_if_exist (str) - Update the container if it exists. |
| """ |
| with api.step.nest('ensure goma base images'): |
| # Build goma base image. |
| goma_base_image = api.url.join(domain, project_id, root, 'base') |
| if not api.gcloud.container_image_exists( |
| goma_base_image) or update_if_exist: |
| build_goma_base_image(api, 'Dockerfile_goma_base', goma_base_image, |
| timestamp) |
| # Build goma runtime image. |
| runtime_image = api.url.join(domain, project_id, root, 'runtime') |
| if not api.gcloud.container_image_exists(runtime_image) or update_if_exist: |
| api.docker('pull', goma_base_image) |
| api.docker('tag', goma_base_image, 'base') |
| build_goma_base_image(api, 'Dockerfile_goma_runtime', runtime_image, |
| timestamp) |
| # pull-toolchain-runtime |
| api.docker('pull', runtime_image) |
| api.docker('tag', runtime_image, 'toolchain/runtime') |
| # Build goma remoteexec-platform |
| remote_image = api.url.join(domain, project_id, root, 'remoteexec-platform') |
| if not api.gcloud.container_image_exists(remote_image) or update_if_exist: |
| build_goma_base_image(api, 'Dockerfile_goma_remoteexec_platform', |
| remote_image, timestamp) |
| |
| |
| def push_toolchain_to_goma(api, |
| goma_temp_dir, |
| project_id, |
| root, |
| toolchain_tag, |
| container_tag, |
| container_version='latest', |
| stamp=None): |
| """Push and register the toolchain docker image with Goma. |
| |
| E.g. push_toolchain_to_goma(goma_temp_dir,\ |
| 'goma-fuchsia',\ |
| 'fuchsia_linux',\ |
| 'clang-some_hash-linux-amd64',\ |
| 'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64',\ |
| 'latest', |
| '20190909_122500') |
| will copy the toolchain binaries from docker image |
| 'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64:latest' |
| to appropriate gs bucket of the Goma gcloud project and |
| register it with Goma server. |
| |
| Args: |
| * goma_temp_dir (str) - The temp directory for Goma resources. |
| * project_id (str) - The gcloud project ID. |
| * root (str) - The root path of all toolchains. E.g. 'fuchsia_linux'. |
| * toolchain_tag (str) - Toolchain version and platform tag. |
| * container_tag (str) - Toolchain container tag. |
| * container_version (str) - The container image version. 'latest' by default. |
| * stamp (str) - Time stamp string. |
| """ |
| with api.step.nest('push toolchain to goma'): |
| # URI = ${domain}/${project_id}/${root}/${toolchain}-${toolchain_version}:${container_version} |
| # e.g. URI = gcr.io/goma-fuchsia/fuchsia_linux/clang-0a5975a5bcd59cb07efdc86b47498fbb7c095ab:latest |
| if stamp is None: |
| stamp = generate_time_stamp(api) # pragma: no cover |
| # extract-goma-files |
| with api.docker.create(container_tag + ':' + |
| container_version) as temp_copy: |
| api.docker.copy(temp_copy, ['/var/cache/goma', '/etc/goma/descriptors'], |
| goma_temp_dir) |
| api.file.move('rename cmdstorage', goma_temp_dir.join('goma', 'cmdstorage'), |
| goma_temp_dir.join('goma', 'sha256')) |
| api.file.remove( |
| 'remove dockerfile if it exists', |
| goma_temp_dir.join('Dockerfile_fuchsia_toolchain_push_test')) |
| api.file.copy('copy dockerfile', |
| api.resource('Dockerfile_fuchsia_toolchain_push_test'), |
| goma_temp_dir) |
| # build-test-image |
| local_toolchain_tag = api.url.join(root, toolchain_tag) |
| # Set env |
| with api.context(cwd=goma_temp_dir): |
| api.docker.build( |
| dockerfile=goma_temp_dir.join( |
| "Dockerfile_fuchsia_toolchain_push_test"), |
| tags=['%s_test' % container_tag, |
| '%s_test' % local_toolchain_tag], |
| ) |
| # run test |
| api.docker('run', '%s_test' % local_toolchain_tag) |
| |
| # store-files |
| api.gsutil.upload( |
| bucket='%s-files' % project_id, |
| src=goma_temp_dir.join('goma', 'sha256'), |
| dst='', |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| api.gsutil.upload( |
| bucket='%s-toolchain-config' % project_id, |
| src=goma_temp_dir.join('descriptors'), |
| dst='%s/%s/' % (root, toolchain_tag), |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| api.file.remove('remove seq file', goma_temp_dir.join('seq')) |
| api.file.write_text('write seq file', goma_temp_dir.join('seq'), stamp) |
| |
| api.gsutil.upload( |
| bucket='%s-toolchain-config' % project_id, |
| src=goma_temp_dir.join('seq'), |
| dst='%s/seq' % root, |
| recursive=False, |
| multithreaded=True, |
| ) |
| |
| |
| def build_and_push_goma_toolchain(api, toolchain, version, domain, project, |
| root, platform, goma_temp_dir, timestamp): |
| toolchain_tag = '%s-%s-%s' % (toolchain, version, platform) |
| container_tag = api.url.join(domain, project, root, toolchain_tag) |
| with api.step.nest('copy goma docker configurations'): |
| dockerfile_platform = platform.replace('-', '_') |
| dockerfile = 'Dockerfile_fuchsia_toolchain_{}'.format(dockerfile_platform) |
| api.file.copy('copy dockerfile', api.resource(dockerfile), |
| goma_temp_dir.join(dockerfile)) |
| setupfile = 'setup_fuchsia_{}_{}'.format(toolchain, dockerfile_platform) |
| # On CI bots, file.copy complains error 13 if the setup file already |
| # exists. Remove it first to avoid this issue. |
| api.file.remove('remove existing setup file', goma_temp_dir.join('setup')) |
| api.file.copy('copy toolchain setup file', api.resource(setupfile), |
| goma_temp_dir.join('setup')) |
| with api.context(cwd=goma_temp_dir): |
| api.docker.build( |
| dockerfile=dockerfile, |
| tags=[container_tag, container_tag + ':' + timestamp], |
| build_args=['TOOLCHAIN=%s' % toolchain]) |
| api.docker('push', container_tag) |
| api.docker('push', container_tag + ':' + timestamp) |
| push_toolchain_to_goma( |
| api, |
| goma_temp_dir, |
| project, |
| root, |
| toolchain_tag, |
| container_tag, |
| stamp=timestamp) |
| |
| |
| def RunSteps(api, package, version, toolchain, platforms, domain, project, root, |
| update_goma_base_images): |
| preprocess_platforms = platforms[:] |
| if 'mac-amd64' in preprocess_platforms and 'linux-amd64' not in preprocess_platforms: |
| # To cross compile mac target on goma linux workers, both mac and linux |
| # toolchains are required. |
| preprocess_platforms.append('linux-amd64') |
| pkgs = api.cipd.EnsureFile() |
| for platform in preprocess_platforms: |
| pkgs.add_package(api.url.join(package, platform), version, platform) |
| toolchain_dir = api.path['cleanup'].join('toolchain') |
| with api.step.nest('ensure toolchain'): |
| api.cipd.ensure(toolchain_dir, pkgs) |
| |
| timestamp = generate_time_stamp(api) |
| if version.startswith('git_revision:'): |
| version = version[len('git_revision:'):] |
| |
| goma_temp_dir = api.path['cleanup'].join('goma') |
| ensure_gomatools(api, goma_temp_dir) |
| with api.step.nest('preprocess toolchain'): |
| if toolchain == 'clang': |
| dest_dir = api.path['cleanup'].join('toolchain_copy') |
| api.python( |
| 'preprocess clang', |
| script=api.resource('preprocess_clang.py'), |
| args=[ |
| toolchain_dir, |
| dest_dir, |
| ] + ['--platform=%s' % platform for platform in preprocess_platforms], |
| ) |
| toolchain_dir = dest_dir |
| for platform in preprocess_platforms: |
| api.archive.package(toolchain_dir.join(platform)).archive( |
| 'pack toolchain package for %s' % platform, |
| goma_temp_dir.join('{}-{}.tgz'.format(toolchain, platform))) |
| |
| ensure_goma_base_images(api, domain, project, root, timestamp, |
| update_goma_base_images) |
| api.docker( |
| 'pull', |
| api.url.join(domain, project, root, 'base'), |
| step_name='pull base') |
| api.docker( |
| 'tag', |
| api.url.join(domain, project, root, 'base'), |
| 'base', |
| step_name='tag base') |
| try: |
| for platform in platforms: |
| with api.step.nest('build toolchain image for %s' % platform): |
| build_and_push_goma_toolchain(api, toolchain, version, domain, project, |
| root, platform, goma_temp_dir, timestamp) |
| finally: |
| api.docker.cleanup() |
| |
| |
| def GenTests(api): |
| default_properties = api.properties( |
| package='fuchsia/third_party/clang', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| properties_missing_platform = api.properties( |
| package='fuchsia/third_party/clang/linux-amd64', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| wrong_platform_properties = api.properties( |
| package='fuchsia/third_party/clang', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| platforms=['linux_amd64'], |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| cross_compile_properties = api.properties( |
| package='fuchsia/third_party/clang', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| platforms=['mac-amd64'], |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| yield api.test('default') + default_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration') |
| |
| yield api.test( |
| 'base image missing' |
| ) + default_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration' |
| ) + api.step_data( |
| 'ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/runtime', |
| retcode=1 |
| ) + api.step_data( |
| 'ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/base', |
| retcode=1 |
| ) + api.step_data( |
| 'ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform', |
| retcode=1) |
| |
| yield api.test( |
| 'wrong_platform') + wrong_platform_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration') |
| |
| yield api.test( |
| 'cross_compile') + cross_compile_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration') |
