recipes/contrib/clang.py - infra/recipes.git - Git at Google

 # Copyright 2024 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe for building Clang toolchain without any runtimes."""

 from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
 from PB.recipes.fuchsia.contrib.clang import InputProperties

 DEPS = [
     "fuchsia/cas_util",
     "fuchsia/cipd_util",
     "fuchsia/cmake",
     "fuchsia/git_checkout",
     "fuchsia/go",
     "fuchsia/goma",
     "fuchsia/ninja",
     "fuchsia/platform_util",
     "fuchsia/tensorflow",
     "fuchsia/toolchain",
     "fuchsia/toolchain_3p_libs",
     "recipe_engine/buildbucket",
     "recipe_engine/cipd",
     "recipe_engine/context",
     "recipe_engine/file",
     "recipe_engine/path",
     "recipe_engine/platform",
     "recipe_engine/properties",
     "recipe_engine/step",
 ]

 PROPERTIES = InputProperties

 CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"


 def RunSteps(api, props):
     # default values
     if not props.repository:
         props.repository = "https://llvm.googlesource.com/llvm-project"
     if not props.revision:
         props.revision = "refs/heads/main"
     use_goma = (
         not api.platform.arch == "arm" and api.platform.bits == 64
     ) and not api.platform.is_win
     if use_goma:
         ninja_jobs = api.goma.jobs
         remote_build_context = api.goma()
     host_platform = api.cipd_util.platform_name
     target_platform = props.platform or host_platform
     use_breakpad = host_platform == "linux-amd64"
     enable_ml_inliner = target_platform not in ("linux-arm64", "mac-arm64")
     manifest = Manifest()
     staging_dir = api.path.start_dir
     pkg_dir = staging_dir.joinpath("llvm_install")
     api.file.ensure_directory("create pkg dir", pkg_dir)

     cipd_dir = ensure_llvm_deps_prebuilts(
         api,
         target_platform,
         use_breakpad=use_breakpad,
         enable_ml_inliner=enable_ml_inliner,
         manifest=manifest,
     )
     llvm_dir, revision = api.git_checkout(props.repository, fallback_ref=props.revision)
     git_checkout = manifest.directories[str(llvm_dir)].git_checkout
     git_checkout.repo_url = props.repository
     git_checkout.revision = revision
     env_prefixes = {"PATH": [cipd_dir.joinpath("bin")]}
     target_triple = api.platform_util.platform(target_platform).triple
     host_triple = api.platform_util.platform(host_platform).triple
     linux_sysroot = cipd_dir.joinpath("linux")

     with remote_build_context, api.context(env_prefixes=env_prefixes):
         arguments = {
             "target_triple": target_triple,
             "host_triple": host_triple,
             "target_sysroot": linux_sysroot,
             "host_sysroot": linux_sysroot,
             "linux_sysroot": linux_sysroot,
             "llvm_repo": props.repository,
             "lto_option": "False",
             "ninja": api.ninja.path,
             "install_prefix": "",
         }
         arguments.update(api.toolchain.toolchain_base_args(api.platform.name, cipd_dir))
         options = api.toolchain.toolchain_base_flags(api.platform.name)
         options.extend(api.toolchain.toolchain_repo_flags())
         options.extend(api.toolchain.toolchain_lto_flags())

         arguments.update({"cflags": "", "cxxflags": "", "ldflags": ""})
         api.file.write_proto(
             "source manifest",
             pkg_dir.joinpath("source_manifest.json"),
             manifest,
             "JSONPB",
         )
         # build clang+llvm
         build_dir = staging_dir.joinpath("llvm_build")
         api.file.ensure_directory("create llvm build dir", build_dir)
         options.extend(
             [
                 # Unprefixed flags are used by the first stage compiler.
                 f"-DCMAKE_{mode}_LINKER_FLAGS=-static-libstdc++"
                 for mode in ["SHARED", "MODULE", "EXE"]
             ]
         )
         env = {
             "CLANG_MODULE_CACHE_PATH": "",
         }

         compiler_wrapper = None
         if use_goma:
             compiler_wrapper = api.goma.goma_dir.joinpath("gomacc")
             arguments["compiler_wrapper"] = compiler_wrapper
             options.extend(api.toolchain.compiler_wrapper_flags())

         with api.step.nest("build llvm dependencies"):
             options.append("-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON")
             api.toolchain_3p_libs.set_toolchain_dir(cipd_dir)
             api.toolchain_3p_libs.set_sysroot_dir(linux_sysroot)
             api.toolchain_3p_libs.set_source_manifest(manifest)
             api.toolchain_3p_libs.set_compiler_wrapper(compiler_wrapper)

             zlib_install_dir = api.toolchain_3p_libs.zlib.build(host_triple)
             options.extend(
                 api.toolchain.zlib_deps_flags(zlib_install_dir, api.platform.is_win)
             )

             zstd_install_dir = api.toolchain_3p_libs.zstd.build(host_triple)
             options.extend(api.toolchain.zstd_deps_flags(zstd_install_dir))

             libxml2_install_dir = api.toolchain_3p_libs.libxml2.build(host_triple)
             options.extend(api.toolchain.libxml2_deps_flags(libxml2_install_dir))

             boringssl_install_dir = api.toolchain_3p_libs.boringssl.build(
                 host_triple,
                 cmake_extra_args=[
                     "-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
                     f"-DGO_EXECUTABLE={api.go.go_root.joinpath('bin', 'go')}",
                 ],
             )
             options.extend(api.toolchain.openssl_deps_flags(boringssl_install_dir))

             curl_install_dir = api.toolchain_3p_libs.curl.build(
                 host_triple,
                 ssl_install_dir=boringssl_install_dir,
                 zlib_install_dir=zlib_install_dir,
             )
             options.extend(api.toolchain.curl_deps_flags(curl_install_dir))

             cpp_httplib_install_dir = api.toolchain_3p_libs.cpp_httplib.build(
                 host_triple, zlib_install_dir=zlib_install_dir
             )
             options.extend(api.toolchain.httplib_deps_flags(cpp_httplib_install_dir))

         if enable_ml_inliner:
             options.extend(
                 [
                     f"-DTENSORFLOW_AOT_PATH={api.tensorflow.path}",
                     f"-DLLVM_OVERRIDE_MODEL_HEADER_INLINERSIZEMODEL={cipd_dir.joinpath('model', 'InlinerSizeModel.h')}",
                     f"-DLLVM_OVERRIDE_MODEL_OBJECT_INLINERSIZEMODEL={cipd_dir.joinpath('model', 'InlinerSizeModel.o')}",
                     "-DLLVM_RAEVICT_MODEL_PATH=none",
                 ]
             )

         with api.step.nest("clang"), api.context(env=env):
             api.cmake(
                 step_name="configure",
                 src_dir=llvm_dir.joinpath("llvm"),
                 build_dir=build_dir,
                 extra_args=[option.format(**arguments) for option in options]
                 + ["-C", api.resource("Fuchsia-toolchain.cmake")],
             )
             api.file.read_text(
                 "read CMakeConfigureLog.yaml",
                 build_dir.joinpath("CMakeFiles", "CMakeConfigureLog.yaml"),
             )
             # Build the toolchain distribution.
             api.ninja(
                 "build",
                 ["toolchain-distribution", "libc-hdrgen"],
                 ninja_jobs=ninja_jobs,
                 build_dir=build_dir,
             )
             # Pack the toolchain distribution
             with api.context(env={"DESTDIR": pkg_dir}):
                 api.ninja(
                     "install",
                     ["install-toolchain-distribution"],
                     ninja_jobs=ninja_jobs,
                     build_dir=build_dir,
                 )
                 hdrgen = "libc-hdrgen.exe" if api.platform.is_win else "libc-hdrgen"
                 api.file.copy(
                     "copy libc-hdrgen",
                     build_dir.joinpath("bin", hdrgen),
                     pkg_dir.joinpath("bin", hdrgen),
                 )
             api.cas_util.upload(pkg_dir, output_property="isolated")
             api.ninja(
                 "tests",
                 [
                     "check-clang",
                     "check-llvm",
                     "check-lld",
                 ],
                 ninja_jobs=ninja_jobs,
                 build_dir=build_dir,
             )


 def ensure_llvm_deps_prebuilts(
     api, target_platform, use_breakpad=False, enable_ml_inliner=False, manifest=None
 ):
     with api.step.nest("ensure packages"):
         cipd_dir = api.path.start_dir.joinpath("cipd")
         pkgs = api.cipd.EnsureFile()
         # Use Linux x64 SDK on all platforms. We only need the sysroot, we
         # don't use any tools so the host architecture is irrelevant.
         pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
         pkgs.add_package(
             "fuchsia/third_party/sysroot/linux",
             "integration",
             "linux",
         )
         pkgs.add_package(
             "fuchsia/third_party/sysroot/focal",
             "git_revision:fa7a5a9710540f30ff98ae48b62f2cdf72ed2acd",
             "focal",
         )
         if enable_ml_inliner:
             pkgs.add_package(
                 f"fuchsia/model/inlining/{target_platform}",
                 "git_revision:0ee1ce61a844b8fd285ff1a2c47c4b8690fd2b7d",
                 "model",
             )
         if use_breakpad:
             pkgs.add_package(
                 "fuchsia/tools/breakpad/${platform}", "integration", "breakpad"
             )

         ensured = api.cipd.ensure(cipd_dir, pkgs)
         for subdir, pins in ensured.items():
             directory = manifest.directories[str(cipd_dir.joinpath(subdir))]
             directory.cipd_server_host = CIPD_SERVER_HOST
             for pin in pins:
                 directory.cipd_package[pin.package].instance_id = pin.instance_id
         return cipd_dir


 def GenTests(api):
     yield (
         api.test("ci_linux_x64")
         + api.buildbucket.ci_build(
             project="fuchsia",
             bucket="ci",
             git_repo="https://fuchsia.googlesource.com/third_party/llvm-project",
             revision="a" * 40,
         )
         + api.platform.name("linux")
         + api.properties()
     )
	# Copyright 2024 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe for building Clang toolchain without any runtimes."""

	from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
	from PB.recipes.fuchsia.contrib.clang import InputProperties

	DEPS = [
	"fuchsia/cas_util",
	"fuchsia/cipd_util",
	"fuchsia/cmake",
	"fuchsia/git_checkout",
	"fuchsia/go",
	"fuchsia/goma",
	"fuchsia/ninja",
	"fuchsia/platform_util",
	"fuchsia/tensorflow",
	"fuchsia/toolchain",
	"fuchsia/toolchain_3p_libs",
	"recipe_engine/buildbucket",
	"recipe_engine/cipd",
	"recipe_engine/context",
	"recipe_engine/file",
	"recipe_engine/path",
	"recipe_engine/platform",
	"recipe_engine/properties",
	"recipe_engine/step",
	]

	PROPERTIES = InputProperties

	CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"


	def RunSteps(api, props):
	# default values
	if not props.repository:
	props.repository = "https://llvm.googlesource.com/llvm-project"
	if not props.revision:
	props.revision = "refs/heads/main"
	use_goma = (
	not api.platform.arch == "arm" and api.platform.bits == 64
	) and not api.platform.is_win
	if use_goma:
	ninja_jobs = api.goma.jobs
	remote_build_context = api.goma()
	host_platform = api.cipd_util.platform_name
	target_platform = props.platform or host_platform
	use_breakpad = host_platform == "linux-amd64"
	enable_ml_inliner = target_platform not in ("linux-arm64", "mac-arm64")
	manifest = Manifest()
	staging_dir = api.path.start_dir
	pkg_dir = staging_dir.joinpath("llvm_install")
	api.file.ensure_directory("create pkg dir", pkg_dir)

	cipd_dir = ensure_llvm_deps_prebuilts(
	api,
	target_platform,
	use_breakpad=use_breakpad,
	enable_ml_inliner=enable_ml_inliner,
	manifest=manifest,
	)
	llvm_dir, revision = api.git_checkout(props.repository, fallback_ref=props.revision)
	git_checkout = manifest.directories[str(llvm_dir)].git_checkout
	git_checkout.repo_url = props.repository
	git_checkout.revision = revision
	env_prefixes = {"PATH": [cipd_dir.joinpath("bin")]}
	target_triple = api.platform_util.platform(target_platform).triple
	host_triple = api.platform_util.platform(host_platform).triple
	linux_sysroot = cipd_dir.joinpath("linux")

	with remote_build_context, api.context(env_prefixes=env_prefixes):
	arguments = {
	"target_triple": target_triple,
	"host_triple": host_triple,
	"target_sysroot": linux_sysroot,
	"host_sysroot": linux_sysroot,
	"linux_sysroot": linux_sysroot,
	"llvm_repo": props.repository,
	"lto_option": "False",
	"ninja": api.ninja.path,
	"install_prefix": "",
	}
	arguments.update(api.toolchain.toolchain_base_args(api.platform.name, cipd_dir))
	options = api.toolchain.toolchain_base_flags(api.platform.name)
	options.extend(api.toolchain.toolchain_repo_flags())
	options.extend(api.toolchain.toolchain_lto_flags())

	arguments.update({"cflags": "", "cxxflags": "", "ldflags": ""})
	api.file.write_proto(
	"source manifest",
	pkg_dir.joinpath("source_manifest.json"),
	manifest,
	"JSONPB",
	)
	# build clang+llvm
	build_dir = staging_dir.joinpath("llvm_build")
	api.file.ensure_directory("create llvm build dir", build_dir)
	options.extend(
	[
	# Unprefixed flags are used by the first stage compiler.
	f"-DCMAKE_{mode}_LINKER_FLAGS=-static-libstdc++"
	for mode in ["SHARED", "MODULE", "EXE"]
	]
	)
	env = {
	"CLANG_MODULE_CACHE_PATH": "",
	}

	compiler_wrapper = None
	if use_goma:
	compiler_wrapper = api.goma.goma_dir.joinpath("gomacc")
	arguments["compiler_wrapper"] = compiler_wrapper
	options.extend(api.toolchain.compiler_wrapper_flags())

	with api.step.nest("build llvm dependencies"):
	options.append("-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON")
	api.toolchain_3p_libs.set_toolchain_dir(cipd_dir)
	api.toolchain_3p_libs.set_sysroot_dir(linux_sysroot)
	api.toolchain_3p_libs.set_source_manifest(manifest)
	api.toolchain_3p_libs.set_compiler_wrapper(compiler_wrapper)

	zlib_install_dir = api.toolchain_3p_libs.zlib.build(host_triple)
	options.extend(
	api.toolchain.zlib_deps_flags(zlib_install_dir, api.platform.is_win)
	)

	zstd_install_dir = api.toolchain_3p_libs.zstd.build(host_triple)
	options.extend(api.toolchain.zstd_deps_flags(zstd_install_dir))

	libxml2_install_dir = api.toolchain_3p_libs.libxml2.build(host_triple)
	options.extend(api.toolchain.libxml2_deps_flags(libxml2_install_dir))

	boringssl_install_dir = api.toolchain_3p_libs.boringssl.build(
	host_triple,
	cmake_extra_args=[
	"-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
	f"-DGO_EXECUTABLE={api.go.go_root.joinpath('bin', 'go')}",
	],
	)
	options.extend(api.toolchain.openssl_deps_flags(boringssl_install_dir))

	curl_install_dir = api.toolchain_3p_libs.curl.build(
	host_triple,
	ssl_install_dir=boringssl_install_dir,
	zlib_install_dir=zlib_install_dir,
	)
	options.extend(api.toolchain.curl_deps_flags(curl_install_dir))

	cpp_httplib_install_dir = api.toolchain_3p_libs.cpp_httplib.build(
	host_triple, zlib_install_dir=zlib_install_dir
	)
	options.extend(api.toolchain.httplib_deps_flags(cpp_httplib_install_dir))

	if enable_ml_inliner:
	options.extend(
	[
	f"-DTENSORFLOW_AOT_PATH={api.tensorflow.path}",
	f"-DLLVM_OVERRIDE_MODEL_HEADER_INLINERSIZEMODEL={cipd_dir.joinpath('model', 'InlinerSizeModel.h')}",
	f"-DLLVM_OVERRIDE_MODEL_OBJECT_INLINERSIZEMODEL={cipd_dir.joinpath('model', 'InlinerSizeModel.o')}",
	"-DLLVM_RAEVICT_MODEL_PATH=none",
	]
	)

	with api.step.nest("clang"), api.context(env=env):
	api.cmake(
	step_name="configure",
	src_dir=llvm_dir.joinpath("llvm"),
	build_dir=build_dir,
	extra_args=[option.format(**arguments) for option in options]
	+ ["-C", api.resource("Fuchsia-toolchain.cmake")],
	)
	api.file.read_text(
	"read CMakeConfigureLog.yaml",
	build_dir.joinpath("CMakeFiles", "CMakeConfigureLog.yaml"),
	)
	# Build the toolchain distribution.
	api.ninja(
	"build",
	["toolchain-distribution", "libc-hdrgen"],
	ninja_jobs=ninja_jobs,
	build_dir=build_dir,
	)
	# Pack the toolchain distribution
	with api.context(env={"DESTDIR": pkg_dir}):
	api.ninja(
	"install",
	["install-toolchain-distribution"],
	ninja_jobs=ninja_jobs,
	build_dir=build_dir,
	)
	hdrgen = "libc-hdrgen.exe" if api.platform.is_win else "libc-hdrgen"
	api.file.copy(
	"copy libc-hdrgen",
	build_dir.joinpath("bin", hdrgen),
	pkg_dir.joinpath("bin", hdrgen),
	)
	api.cas_util.upload(pkg_dir, output_property="isolated")
	api.ninja(
	"tests",
	[
	"check-clang",
	"check-llvm",
	"check-lld",
	],
	ninja_jobs=ninja_jobs,
	build_dir=build_dir,
	)


	def ensure_llvm_deps_prebuilts(
	api, target_platform, use_breakpad=False, enable_ml_inliner=False, manifest=None
	):
	with api.step.nest("ensure packages"):
	cipd_dir = api.path.start_dir.joinpath("cipd")
	pkgs = api.cipd.EnsureFile()
	# Use Linux x64 SDK on all platforms. We only need the sysroot, we
	# don't use any tools so the host architecture is irrelevant.
	pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
	pkgs.add_package(
	"fuchsia/third_party/sysroot/linux",
	"integration",
	"linux",
	)
	pkgs.add_package(
	"fuchsia/third_party/sysroot/focal",
	"git_revision:fa7a5a9710540f30ff98ae48b62f2cdf72ed2acd",
	"focal",
	)
	if enable_ml_inliner:
	pkgs.add_package(
	f"fuchsia/model/inlining/{target_platform}",
	"git_revision:0ee1ce61a844b8fd285ff1a2c47c4b8690fd2b7d",
	"model",
	)
	if use_breakpad:
	pkgs.add_package(
	"fuchsia/tools/breakpad/${platform}", "integration", "breakpad"
	)

	ensured = api.cipd.ensure(cipd_dir, pkgs)
	for subdir, pins in ensured.items():
	directory = manifest.directories[str(cipd_dir.joinpath(subdir))]
	directory.cipd_server_host = CIPD_SERVER_HOST
	for pin in pins:
	directory.cipd_package[pin.package].instance_id = pin.instance_id
	return cipd_dir


	def GenTests(api):
	yield (
	api.test("ci_linux_x64")
	+ api.buildbucket.ci_build(
	project="fuchsia",
	bucket="ci",
	git_repo="https://fuchsia.googlesource.com/third_party/llvm-project",
	revision="a" * 40,
	)
	+ api.platform.name("linux")
	+ api.properties()
	)