| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building goma toolchain images.""" |
| |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/docker', |
| 'fuchsia/gcloud', |
| 'fuchsia/git', |
| 'fuchsia/goma', |
| 'fuchsia/gsutil', |
| 'recipe_engine/archive', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/platform', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| 'recipe_engine/url', |
| ] |
| |
| PROPERTIES = { |
| 'package': |
| Property(kind=str, help='cipd path of the toolchain package'), |
| 'version': |
| Property( |
| kind=str, |
| help='cipd version of the toolchain package', |
| default='git_revision:1475fad1d0d45572471575cdda8fed6eeab5b0ec'), |
| 'toolchain': |
| Property(kind=str, help='type of the toolchain', default='clang'), |
| 'platform': |
| Property(kind=str, help='toolchain host platform', default=None), |
| 'domain': |
| Property( |
| kind=str, |
| help='domain name of container registry', |
| default='gcr.io'), |
| 'project': |
| Property( |
| kind=str, help='name of goma cloud project', |
| default='goma-fuchsia'), |
| 'root': |
| Property( |
| kind=str, |
| help='root directory name for container registry', |
| default='fuchsia_linux'), |
| 'update_goma_base_images': |
| Property( |
| kind=bool, |
| help='Update goma base images even if they already exist', |
| default=False), |
| } |
| |
| GOMATOOLS_VERSION = 'git_revision:22684ee9174606073a2604e1d58fbf03ba3ad564' |
| |
| |
| def ensure_gomatools(api, gomatools_dir): |
| with api.step.nest('ensure gomatools'): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package( |
| 'fuchsia_internal/third_party/goma/server/gomatools/linux-amd64', |
| GOMATOOLS_VERSION) |
| api.cipd.ensure(gomatools_dir, pkgs) |
| |
| |
| def generate_time_stamp(api): |
| return "{:%Y%m%d_%H%M%S}".format(api.time.utcnow()) |
| |
| |
| def build_goma_base_image(api, dockerfile, image, timestamp=None): |
| api.file.copy('copy dockerfile', api.resource(dockerfile), |
| api.path['cleanup'].join(dockerfile)) |
| tags = [image] |
| if timestamp: |
| tags.append(image + ':' + timestamp) |
| with api.context(cwd=api.path['cleanup']): |
| api.docker.build( |
| dockerfile=api.path['cleanup'].join(dockerfile), |
| tags=tags, |
| ) |
| api.docker('push', image) |
| |
| |
| def ensure_goma_base_images(api, |
| domain, |
| project_id, |
| root, |
| timestamp=None, |
| update_if_exist=False): |
| """Ensure the base goma images in container registry. |
| |
| E.g. ensure_goma_base_images(api, |
| 'gcr.io',\ |
| 'goma-fuchsia',\ |
| 'fuchsia_linux') |
| will create docker images at |
| gcr.io/goma-fuchsia/fuchsia_linux/base, |
| gcr.io/goma-fuchsia/fuchsia_linux/runtime, |
| gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform, |
| using 'debian:buster-slim' as bottom layer image. |
| |
| Args: |
| * domain (str) - The domain name of the container register. |
| * project (str) - The gcloud project ID. |
| * root (str) - The name of container registry root. |
| * update_if_exist (str) - Update the container if it exists. |
| """ |
| with api.step.nest('ensure goma base images'): |
| # Build goma base image. |
| goma_base_image = api.url.join(domain, project_id, root, 'base') |
| if not api.gcloud.container_image_exists( |
| goma_base_image) or update_if_exist: |
| build_goma_base_image(api, 'Dockerfile_goma_base', goma_base_image, |
| timestamp) |
| # Build goma runtime image. |
| runtime_image = api.url.join(domain, project_id, root, 'runtime') |
| if not api.gcloud.container_image_exists(runtime_image) or update_if_exist: |
| api.docker('pull', goma_base_image) |
| api.docker('tag', goma_base_image, 'base') |
| build_goma_base_image(api, 'Dockerfile_goma_runtime', runtime_image, |
| timestamp) |
| # Build goma remoteexec-platform |
| remote_image = api.url.join(domain, project_id, root, 'remoteexec-platform') |
| if not api.gcloud.container_image_exists(remote_image) or update_if_exist: |
| build_goma_base_image(api, 'Dockerfile_goma_remoteexec_platform', |
| remote_image, timestamp) |
| |
| |
| def push_toolchain_to_goma(api, |
| goma_temp_dir, |
| domain, |
| project_id, |
| root, |
| toolchain, |
| toolchain_version, |
| container_version='latest', |
| stamp=None): |
| """Push and register the toolchain docker image with Goma. |
| |
| E.g. push_toolchain_to_goma('gcr.io',\ |
| goma_temp_dir,\ |
| 'goma-fuchsia',\ |
| 'fuchsia_linux',\ |
| 'clang',\ |
| 'some_hash',\ |
| 'latest', |
| '20190909_122500') |
| will copy the toolchain binaries from docker image |
| 'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash:latest' |
| to appropriate gs bucket of the Goma gcloud project and |
| register it with Goma server. |
| |
| Args: |
| * domain (str) - The domain name of the clontainer register. |
| * project_id (str) - The gcloud project ID. |
| * root (str) - The root path of all toolchains. E.g. 'fuchsia_linux'. |
| * toolchain (str) - The name of the toolchain. |
| * toolchain_version (str) - The version string of the toolchain. |
| Usually it is the commit hash. |
| * container_version (str) - The container image version. 'latest' by default. |
| * stamp (str) - Time stamp string. |
| """ |
| with api.step.nest('push toolchain to goma'): |
| # URI = ${domain}/${project_id}/${root}/${toolchain}-${toolchain_version}:${container_version} |
| # e.g. URI = gcr.io/goma-fuchsia/fuchsia_linux/clang-0a5975a5bcd59cb07efdc86b47498fbb7c095ab:latest |
| runtime_image = api.url.join(domain, project_id, root, 'runtime') |
| if stamp is None: |
| stamp = generate_time_stamp(api) # pragma: no cover |
| # extract-goma-files |
| toolchain_tag = '%s-%s' % (toolchain, toolchain_version) |
| toolchain_image = api.url.join(domain, project_id, root, toolchain_tag) |
| with api.docker.create(toolchain_image + ':' + |
| container_version) as temp_copy: |
| api.docker.copy(temp_copy, ['/var/cache/goma', '/etc/goma/descriptors'], |
| goma_temp_dir) |
| api.file.move('rename cmdstorage', goma_temp_dir.join('goma', 'cmdstorage'), |
| goma_temp_dir.join('goma', 'sha256')) |
| |
| # pull-toolchain-runtime |
| api.docker('pull', runtime_image) |
| api.docker('tag', runtime_image, 'toolchain/runtime') |
| |
| api.file.copy('copy dockerfile', |
| api.resource('Dockerfile_fuchsia_toolchain_push_test'), |
| goma_temp_dir) |
| # build-test-image |
| local_toolchain_tag = api.url.join(root, toolchain_tag) |
| # Set env |
| with api.context(cwd=goma_temp_dir): |
| api.docker.build( |
| dockerfile=goma_temp_dir.join( |
| "Dockerfile_fuchsia_toolchain_push_test"), |
| tags=['%s_test' % toolchain_image, |
| '%s_test' % local_toolchain_tag], |
| ) |
| # run test |
| api.docker('run', '%s_test' % local_toolchain_tag) |
| |
| # store-files |
| api.gsutil.upload( |
| bucket='%s-files' % project_id, |
| src=goma_temp_dir.join('goma', 'sha256'), |
| dst='', |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| api.gsutil.upload( |
| bucket='%s-toolchain-config' % project_id, |
| src=goma_temp_dir.join('descriptors'), |
| dst='%s/%s/' % (root, toolchain_tag), |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| api.file.remove('remove seq file', goma_temp_dir.join('seq')) |
| api.file.write_text('write seq file', goma_temp_dir.join('seq'), stamp) |
| |
| api.gsutil.upload( |
| bucket='%s-toolchain-config' % project_id, |
| src=goma_temp_dir.join('seq'), |
| dst='%s/seq' % root, |
| recursive=False, |
| multithreaded=True, |
| ) |
| |
| |
| def RunSteps(api, package, version, toolchain, platform, domain, project, root, |
| update_goma_base_images): |
| assert package.endswith('${platform}') |
| # TODO: factor this out into a host_build recipe module. |
| host_platform = '%s-%s' % (api.platform.name.replace('win', 'windows'), { |
| 'intel': { |
| 32: '386', |
| 64: 'amd64', |
| }, |
| 'arm': { |
| 32: 'armv6', |
| 64: 'arm64', |
| }, |
| }[api.platform.arch][api.platform.bits]) |
| target_platform = platform or host_platform |
| pkgs = api.cipd.EnsureFile() |
| platforms = [target_platform] |
| if target_platform == 'mac-amd64': |
| # Cross compilation is required, packing |
| # linux toolchain as well. |
| platforms.append('linux-amd64') |
| |
| for item in platforms: |
| pkgs.add_package(package.replace('${platform}', item), version, item) |
| toolchain_dir = api.path['cleanup'].join('toolchain') |
| with api.step.nest('ensure toolchain'): |
| api.cipd.ensure(toolchain_dir, pkgs) |
| |
| timestamp = generate_time_stamp(api) |
| if version.startswith('git_revision:'): |
| version = version[len('git_revision:'):] |
| |
| goma_temp_dir = api.path['cleanup'].join('goma') |
| ensure_gomatools(api, goma_temp_dir) |
| with api.step.nest('preprocess toolchain'): |
| if toolchain == 'clang': |
| dest_dir = api.path['cleanup'].join('toolchain_copy') |
| py_args = [ |
| str(toolchain_dir), |
| str(dest_dir), |
| ] |
| for item in platforms: |
| py_args.append('--platform=%s' % item) |
| api.python( |
| 'minimize clang', |
| script=api.resource('preprocess_clang.py'), |
| args=py_args, |
| ) |
| toolchain_dir = dest_dir |
| for item in platforms: |
| api.archive.package(toolchain_dir.join(item)).archive( |
| 'pack toolchain package', |
| goma_temp_dir.join('{}-{}.tgz'.format(toolchain, item))) |
| |
| toolchain_tag = '%s-%s-%s' % (toolchain, version, target_platform) |
| container_tag = api.url.join(domain, project, root, toolchain_tag) |
| |
| with api.step.nest('build toolchain image'): |
| try: |
| ensure_goma_base_images(api, domain, project, root, timestamp, |
| update_goma_base_images) |
| api.docker( |
| 'pull', |
| api.url.join(domain, project, root, 'base'), |
| step_name='pull base') |
| |
| api.docker( |
| 'tag', |
| api.url.join(domain, project, root, 'base'), |
| 'base', |
| step_name='tag base') |
| |
| with api.step.nest('copy goma docker configurations'): |
| dockerfile_platform = target_platform.replace('-', '_') |
| dockerfile = 'Dockerfile_fuchsia_toolchain_{}'.format( |
| dockerfile_platform) |
| api.file.copy('copy dockerfile', api.resource(dockerfile), |
| goma_temp_dir.join(dockerfile)) |
| setupfile = 'setup_fuchsia_{}_{}'.format(toolchain, dockerfile_platform) |
| api.file.copy('copy toolchain setup file', api.resource(setupfile), |
| goma_temp_dir.join('setup')) |
| with api.context(cwd=goma_temp_dir): |
| api.docker.build( |
| dockerfile=dockerfile, |
| tags=[container_tag, container_tag + ':' + timestamp], |
| build_args=['TOOLCHAIN=%s' % toolchain]) |
| api.docker('push', container_tag) |
| api.docker('push', container_tag + ':' + timestamp) |
| push_toolchain_to_goma( |
| api, |
| goma_temp_dir, |
| domain, |
| project, |
| root, |
| toolchain, |
| version + '-' + target_platform, |
| stamp=timestamp) |
| finally: |
| api.docker.cleanup() |
| |
| |
| def GenTests(api): |
| default_properties = api.properties( |
| package='fuchsia/third_party/clang/${platform}', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| properties_missing_platform = api.properties( |
| package='fuchsia/third_party/clang/linux-amd64', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| wrong_platform_properties = api.properties( |
| package='fuchsia/third_party/clang/${platform}', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| platform='linux_amd64', |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| cross_compile_properties = api.properties( |
| package='fuchsia/third_party/clang/${platform}', |
| version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e', |
| toolchain='clang', |
| platform='mac-amd64', |
| domain='gcr.io', |
| project='goma-fuchsia', |
| root='fuchsia_linux') |
| |
| yield api.test('default') + default_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration') |
| |
| yield api.test( |
| 'base image missing' |
| ) + default_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration' |
| ) + api.step_data( |
| 'build toolchain image.ensure goma base images.check container existence', |
| retcode=1 |
| ) + api.step_data( |
| 'build toolchain image.ensure goma base images.check container existence (2)', |
| retcode=1 |
| ) + api.step_data( |
| 'build toolchain image.ensure goma base images.check container existence (3)', |
| retcode=1) |
| |
| yield api.test('missing_platform' |
| ) + properties_missing_platform + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration' |
| ) + api.expect_exception('AssertionError') |
| |
| yield api.test( |
| 'wrong_platform') + wrong_platform_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration') |
| |
| yield api.test( |
| 'cross_compile') + cross_compile_properties + api.buildbucket.try_build( |
| git_repo='https://fuchsia.googlesource.com/integration') |
