| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building goma toolchain images.""" |
| |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/docker", |
| "fuchsia/gcloud", |
| "fuchsia/gsutil", |
| "recipe_engine/archive", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| "recipe_engine/time", |
| "recipe_engine/url", |
| ] |
| |
| PROPERTIES = { |
| "package": Property( |
| kind=str, |
| help="cipd path of the toolchain package", |
| default="fuchsia/third_party/clang", |
| ), |
| "version": Property( |
| kind=str, help="CIPD version of the toolchain package", default=None |
| ), |
| "toolchain": Property(kind=str, help="type of the toolchain", default="clang"), |
| "platforms": Property( |
| kind=List(str), |
| help="toolchain host platform", |
| default=["linux-amd64", "mac-amd64"], |
| ), |
| "domain": Property( |
| kind=str, help="domain name of container registry", default="gcr.io" |
| ), |
| "project": Property( |
| kind=str, help="name of goma cloud project", default="goma-fuchsia" |
| ), |
| "root": Property( |
| kind=str, |
| help="root directory name for container registry", |
| default="fuchsia_linux", |
| ), |
| "update_goma_base_images": Property( |
| kind=bool, |
| help="Update goma base images even if they already exist", |
| default=False, |
| ), |
| } |
| |
| GOMATOOLS_VERSION = "git_revision:c75592e9e1ee38f90275ca7bb101c5a7c9a5bfbc" |
| |
| # "A tag name must be valid ASCII and may contain lowercase and uppercase |
| # letters, digits, underscores, periods and dashes." So sayeth |
| # https://docs.docker.com/engine/reference/commandline/tag/ |
| DOCKER_TAG_DISALLOWED_RE = re.compile(r"[^-._a-zA-Z0-9]") |
| |
| |
| def RunSteps( |
| api, |
| package, |
| version, |
| toolchain, |
| platforms, |
| domain, |
| project, |
| root, |
| update_goma_base_images, |
| ): |
| if not version: |
| gitiles_commit = api.buildbucket.build_input.gitiles_commit |
| if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id: |
| version = str("git_revision:" + gitiles_commit.id) |
| preprocess_platforms = platforms[:] |
| if ( |
| "mac-amd64" in preprocess_platforms or "windows-amd64" in preprocess_platforms |
| ) and "linux-amd64" not in preprocess_platforms: |
| # To cross compile mac target on goma linux workers, both mac and linux |
| # toolchains are required. |
| preprocess_platforms.append("linux-amd64") |
| preprocess_platforms.sort() |
| pkgs = api.cipd.EnsureFile() |
| for platform in preprocess_platforms: |
| pkgs.add_package(api.url.join(package, platform), version, platform) |
| toolchain_dir = api.path["cleanup"].join("toolchain") |
| with api.step.nest("ensure toolchain"): |
| api.cipd.ensure(toolchain_dir, pkgs) |
| |
| timestamp = generate_time_stamp(api) |
| if version.startswith("git_revision:"): |
| version = version[len("git_revision:") :] |
| |
| goma_temp_dir = api.path["cleanup"].join("goma") |
| ensure_gomatools(api, goma_temp_dir) |
| with api.step.nest("preprocess toolchain"): |
| if toolchain in ("clang", "gcc"): |
| dest_dir = api.path["cleanup"].join("toolchain_copy") |
| api.step( |
| "preprocess %s" % toolchain, |
| cmd=[ |
| "python", |
| api.resource("preprocess_%s.py" % toolchain), |
| toolchain_dir, |
| dest_dir, |
| ] |
| + ["--platform=%s" % platform for platform in preprocess_platforms], |
| ) |
| toolchain_dir = dest_dir |
| for platform in preprocess_platforms: |
| api.archive.package(toolchain_dir.join(platform)).archive( |
| "pack toolchain package for %s" % platform, |
| goma_temp_dir.join("{}-{}.tgz".format(toolchain, platform)), |
| ) |
| |
| ensure_goma_base_images( |
| api, domain, project, root, timestamp, update_goma_base_images |
| ) |
| api.docker( |
| "pull", api.url.join(domain, project, root, "base"), step_name="pull base" |
| ) |
| api.docker( |
| "tag", api.url.join(domain, project, root, "base"), "base", step_name="tag base" |
| ) |
| try: |
| for platform in platforms: |
| with api.step.nest("build toolchain image for %s" % platform): |
| build_and_push_goma_toolchain( |
| api, |
| toolchain, |
| version, |
| domain, |
| project, |
| root, |
| platform, |
| goma_temp_dir, |
| timestamp, |
| ) |
| finally: |
| api.docker.cleanup() |
| |
| |
| def ensure_gomatools(api, gomatools_dir): |
| with api.step.nest("ensure gomatools"): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package( |
| "fuchsia_internal/third_party/goma/server/gomatools/linux-amd64", |
| GOMATOOLS_VERSION, |
| ) |
| api.cipd.ensure(gomatools_dir, pkgs) |
| |
| |
| def generate_time_stamp(api): |
| return "{:%Y%m%d_%H%M%S}".format(api.time.utcnow()) |
| |
| |
| def build_goma_base_image(api, dockerfile, image, timestamp=None): |
| api.file.copy( |
| "copy dockerfile", |
| api.resource(dockerfile), |
| api.path["cleanup"].join(dockerfile), |
| ) |
| tags = [image] |
| if timestamp: |
| tags.append(image + ":" + timestamp) |
| with api.context(cwd=api.path["cleanup"]): |
| api.docker.build( |
| dockerfile=api.path["cleanup"].join(dockerfile), |
| tags=tags, |
| ) |
| api.docker("push", image) |
| |
| |
| def ensure_goma_base_images( |
| api, domain, project_id, root, timestamp=None, update_if_exist=False |
| ): |
| """Ensure the base goma images in container registry. |
| |
| E.g. ensure_goma_base_images(api, |
| 'gcr.io',\ |
| 'goma-fuchsia',\ |
| 'fuchsia_linux') |
| will create docker images at |
| gcr.io/goma-fuchsia/fuchsia_linux/base, |
| gcr.io/goma-fuchsia/fuchsia_linux/runtime, |
| gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform, |
| using 'debian:buster-slim' as bottom layer image. |
| |
| Args: |
| * domain (str) - The domain name of the container register. |
| * project (str) - The gcloud project ID. |
| * root (str) - The name of container registry root. |
| * update_if_exist (str) - Update the container if it exists. |
| """ |
| with api.step.nest("ensure goma base images"): |
| # Build goma base image. |
| goma_base_image = api.url.join(domain, project_id, root, "base") |
| if not api.gcloud.container_image_exists(goma_base_image) or update_if_exist: |
| build_goma_base_image( |
| api, "Dockerfile_goma_base", goma_base_image, timestamp |
| ) |
| # Build goma runtime image. |
| runtime_image = api.url.join(domain, project_id, root, "runtime") |
| if not api.gcloud.container_image_exists(runtime_image) or update_if_exist: |
| api.docker("pull", goma_base_image) |
| api.docker("tag", goma_base_image, "base") |
| build_goma_base_image( |
| api, "Dockerfile_goma_runtime", runtime_image, timestamp |
| ) |
| # pull-toolchain-runtime |
| api.docker("pull", runtime_image) |
| api.docker("tag", runtime_image, "toolchain/runtime") |
| # Build goma remoteexec-platform |
| remote_image = api.url.join(domain, project_id, root, "remoteexec-platform") |
| if not api.gcloud.container_image_exists(remote_image) or update_if_exist: |
| build_goma_base_image( |
| api, "Dockerfile_goma_remoteexec_platform", remote_image, timestamp |
| ) |
| |
| |
| def push_toolchain_to_goma( |
| api, |
| goma_temp_dir, |
| project_id, |
| root, |
| toolchain_tag, |
| container_tag, |
| container_version="latest", |
| stamp=None, |
| ): |
| """Push and register the toolchain docker image with Goma. |
| |
| E.g. push_toolchain_to_goma(goma_temp_dir,\ |
| 'goma-fuchsia',\ |
| 'fuchsia_linux',\ |
| 'clang-some_hash-linux-amd64',\ |
| 'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64',\ |
| 'latest', |
| '20190909_122500') |
| will copy the toolchain binaries from docker image |
| 'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64:latest' |
| to appropriate gs bucket of the Goma gcloud project and |
| register it with Goma server. |
| |
| Args: |
| * goma_temp_dir (str) - The temp directory for Goma resources. |
| * project_id (str) - The gcloud project ID. |
| * root (str) - The root path of all toolchains. E.g. 'fuchsia_linux'. |
| * toolchain_tag (str) - Toolchain version and platform tag. |
| * container_tag (str) - Toolchain container tag. |
| * container_version (str) - The container image version. 'latest' by default. |
| * stamp (str) - Time stamp string. |
| """ |
| with api.step.nest("push toolchain to goma"): |
| # URI = ${domain}/${project_id}/${root}/${toolchain}-${toolchain_version}:${container_version} |
| # e.g. URI = gcr.io/goma-fuchsia/fuchsia_linux/clang-0a5975a5bcd59cb07efdc86b47498fbb7c095ab:latest |
| if stamp is None: |
| stamp = generate_time_stamp(api) # pragma: no cover |
| api.file.rmtree( |
| "remove descriptors directory if it already exists", |
| goma_temp_dir.join("goma", "cmdstorage"), |
| ) |
| api.file.rmtree( |
| "remove sha256 directory if it already exists", |
| goma_temp_dir.join("goma", "sha256"), |
| ) |
| # extract-goma-files |
| with api.docker.create(container_tag + ":" + container_version) as temp_copy: |
| api.docker.copy( |
| temp_copy, ["/var/cache/goma", "/etc/goma/descriptors"], goma_temp_dir |
| ) |
| api.file.move( |
| "rename cmdstorage", |
| goma_temp_dir.join("goma", "cmdstorage"), |
| goma_temp_dir.join("goma", "sha256"), |
| ) |
| api.file.remove( |
| "remove dockerfile if it exists", |
| goma_temp_dir.join("Dockerfile_fuchsia_toolchain_push_test"), |
| ) |
| api.file.copy( |
| "copy dockerfile", |
| api.resource("Dockerfile_fuchsia_toolchain_push_test"), |
| goma_temp_dir, |
| ) |
| # build-test-image |
| local_toolchain_tag = api.url.join(root, toolchain_tag) |
| # Set env |
| with api.context(cwd=goma_temp_dir): |
| api.docker.build( |
| dockerfile=goma_temp_dir.join("Dockerfile_fuchsia_toolchain_push_test"), |
| tags=["%s_test" % container_tag, "%s_test" % local_toolchain_tag], |
| ) |
| # run test |
| if not local_toolchain_tag[local_toolchain_tag.find("/") + 1 :].startswith( |
| "gcc" |
| ): |
| # TODO(haowei): Fix the goma backend so we can run the |
| # toolchain test for gcc. |
| api.docker("run", "%s_test" % local_toolchain_tag) |
| |
| # store-files |
| api.gsutil.upload( |
| bucket="%s-files" % project_id, |
| src=goma_temp_dir.join("goma", "sha256"), |
| dst="", |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| api.gsutil.upload( |
| bucket="%s-toolchain-config" % project_id, |
| src=goma_temp_dir.join("descriptors"), |
| dst="%s/%s/" % (root, toolchain_tag), |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| api.file.remove("remove seq file", goma_temp_dir.join("seq")) |
| api.file.write_text("write seq file", goma_temp_dir.join("seq"), stamp) |
| |
| api.gsutil.upload( |
| bucket="%s-toolchain-config" % project_id, |
| src=goma_temp_dir.join("seq"), |
| dst="%s/seq" % root, |
| recursive=False, |
| multithreaded=True, |
| ) |
| |
| |
| def build_and_push_goma_toolchain( |
| api, toolchain, version, domain, project, root, platform, goma_temp_dir, timestamp |
| ): |
| toolchain_tag = "%s-%s-%s" % (toolchain, version, platform) |
| toolchain_tag = DOCKER_TAG_DISALLOWED_RE.sub("_", toolchain_tag) |
| container_tag = api.url.join(domain, project, root, toolchain_tag) |
| with api.step.nest("copy goma docker configurations"): |
| dockerfile_platform = platform.replace("-", "_") |
| dockerfile = "Dockerfile_fuchsia_toolchain_{}".format(dockerfile_platform) |
| api.file.copy( |
| "copy dockerfile", api.resource(dockerfile), goma_temp_dir.join(dockerfile) |
| ) |
| setupfile = "setup_fuchsia_{}_{}".format(toolchain, dockerfile_platform) |
| # On CI bots, file.copy complains error 13 if the setup file already |
| # exists. Remove it first to avoid this issue. |
| api.file.remove("remove existing setup file", goma_temp_dir.join("setup")) |
| api.file.copy( |
| "copy toolchain setup file", |
| api.resource(setupfile), |
| goma_temp_dir.join("setup"), |
| ) |
| with api.context(cwd=goma_temp_dir): |
| api.docker.build( |
| dockerfile=dockerfile, |
| tags=[container_tag, container_tag + ":" + timestamp], |
| build_args=["TOOLCHAIN=%s" % toolchain], |
| ) |
| api.docker("push", container_tag) |
| api.docker("push", container_tag + ":" + timestamp) |
| push_toolchain_to_goma( |
| api, |
| goma_temp_dir, |
| project, |
| root, |
| toolchain_tag, |
| container_tag, |
| stamp=timestamp, |
| ) |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "package": "fuchsia/third_party/clang", |
| "version": "git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e", |
| "toolchain": "clang", |
| "domain": "gcr.io", |
| "project": "goma-fuchsia", |
| "root": "fuchsia_linux", |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| yield api.buildbucket_util.test("default") |
| |
| yield ( |
| api.buildbucket_util.test("base_image_missing") |
| + properties() |
| + api.step_data( |
| "ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/runtime", |
| retcode=1, |
| ) |
| + api.step_data( |
| "ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/base", |
| retcode=1, |
| ) |
| + api.step_data( |
| "ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform", |
| retcode=1, |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("wrong_platform") |
| + properties(platforms=["linux_amd64"]) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("cross_compile") + properties(platforms=["mac-amd64"]) |
| ) |
| |
| yield api.buildbucket_util.test("with_buildset", revision="a" * 40) + properties() |