recipes/contrib/cmake.py - infra/recipes - Git at Google

 # Copyright 2019 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe for building CMake."""

 from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
 from PB.recipes.fuchsia.contrib.cmake import InputProperties

 import re

 DEPS = [
     "fuchsia/buildbucket_util",
     "fuchsia/cas_util",
     "fuchsia/cipd_util",
     "fuchsia/git_checkout",
     "fuchsia/go",
     "fuchsia/macos_sdk",
     "fuchsia/platform_util",
     "fuchsia/windows_sdk",
     "recipe_engine/buildbucket",
     "recipe_engine/cipd",
     "recipe_engine/context",
     "recipe_engine/file",
     "recipe_engine/path",
     "recipe_engine/platform",
     "recipe_engine/properties",
     "recipe_engine/raw_io",
     "recipe_engine/step",
 ]

 PROPERTIES = InputProperties

 BORINGSSL_GIT_URL = "https://boringssl.googlesource.com/boringssl"

 GIT_URL = "https://fuchsia.googlesource.com/third_party/github.com/Kitware/CMake"

 CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"


 def relpath(api, path):
     return str(api.path.relpath(path, api.path["start_dir"]))


 def slashes(api, path):
     # CMake only accept '/' as path delimiter. This helper function replaces '\'
     # with '/' on Windows platform.
     return path.replace("\\", "/") if api.platform.is_win else path


 def RunSteps(api, props):
     host_platform = api.platform_util.host
     target_platform = (
         api.platform_util.platform(props.platform) if props.platform else host_platform
     )

     manifest = Manifest()

     with api.step.nest("ensure packages"), api.context(infra_steps=True):
         cipd_dir = api.path["start_dir"].join("cipd")
         pkgs = api.cipd.EnsureFile()
         pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
         if target_platform.is_linux:
             pkgs.add_package(
                 "fuchsia/sysroot/${platform}",
                 "git_revision:a28dfa20af063e5ca00634024c85732e20220419",
                 "sysroot",
             )
         if target_platform.is_win:
             pkgs.add_package(
                 "fuchsia/third_party/yasm/${platform}",
                 "git_revision:51af4082cc898b122b88f11fd34033fc00fad81e",
             )
         pkgs.add_package(
             "fuchsia/third_party/cmake/${platform}",
             "integration",
         )
         pkgs.add_package(
             "fuchsia/third_party/ninja/${platform}",
             "integration",
         )
         ensured = api.cipd.ensure(cipd_dir, pkgs)
         for subdir, pins in ensured.items():
             directory = manifest.directories[relpath(api, cipd_dir.join(subdir))]
             directory.cipd_server_host = CIPD_SERVER_HOST
             for pin in pins:
                 directory.cipd_package[pin.package].instance_id = pin.instance_id

     with api.macos_sdk(), api.windows_sdk():
         options = []
         if target_platform.is_win:
             options.extend(
                 [
                     "-DCMAKE_ASM_COMPILER=%s" % cipd_dir.join("bin", "clang-cl.exe"),
                     "-DCMAKE_C_COMPILER=%s" % cipd_dir.join("bin", "clang-cl.exe"),
                     "-DCMAKE_CXX_COMPILER=%s" % cipd_dir.join("bin", "clang-cl.exe"),
                     "-DCMAKE_MAKE_PROGRAM=%s" % cipd_dir.join("ninja.exe"),
                     "-DCMAKE_SYSROOT=%s" % api.windows_sdk.sdk_dir,
                     "-DCMAKE_MT=mt.exe",
                     "-DCMAKE_RC=rc.exe",
                 ]
             )
         else:
             options.extend(
                 [
                     "-DCMAKE_ASM_COMPILER=%s" % cipd_dir.join("bin", "clang"),
                     "-DCMAKE_C_COMPILER=%s" % cipd_dir.join("bin", "clang"),
                     "-DCMAKE_CXX_COMPILER=%s" % cipd_dir.join("bin", "clang++"),
                     "-DCMAKE_MAKE_PROGRAM=%s" % cipd_dir.join("ninja"),
                 ]
             )
         if target_platform.is_linux:
             options.extend(["-DCMAKE_SYSROOT=%s" % cipd_dir.join("sysroot")])
         if target_platform.is_mac:
             options.extend(
                 [
                     "-DCMAKE_SYSROOT=%s"
                     % api.step(
                         "xcrun",
                         ["xcrun", "--sdk", "macosx", "--show-sdk-path"],
                         stdout=api.raw_io.output_text(
                             name="sdk-path", add_output_log=True
                         ),
                         step_test_data=lambda: api.raw_io.test_api.stream_output_text(
                             "/some/xcode/path"
                         ),
                     ).stdout.strip(),
                 ]
                 + [
                     "-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s"
                     % (mode, cipd_dir.join("lib", "libc++.a"))
                     for mode in ["SHARED", "MODULE", "EXE"]
                 ]
             )
         if target_platform != host_platform:
             options.extend(
                 [
                     "-DCMAKE_C_COMPILER_TARGET=%s" % target_platform.triple,
                     "-DCMAKE_CXX_COMPILER_TARGET=%s" % target_platform.triple,
                     "-DCMAKE_ASM_COMPILER_TARGET=%s" % target_platform.triple,
                     "-DCMAKE_SYSTEM_NAME=%s"
                     % target_platform.os.replace("mac", "darwin").title(),
                     "-DCMAKE_SYSTEM_PROCESSOR=%s"
                     % target_platform.arch.replace("amd64", "x86_64").replace(
                         "arm64", "aarch64"
                     ),
                 ]
             )

         with api.step.nest("boringssl"):
             boringssl_src_dir = api.path["start_dir"].join("src", "boringssl")
             _, revision = api.git_checkout(
                 BORINGSSL_GIT_URL,
                 path=boringssl_src_dir,
                 fallback_ref="refs/heads/master",
             )
             git_checkout = manifest.directories[
                 relpath(api, boringssl_src_dir)
             ].git_checkout
             git_checkout.repo_url = BORINGSSL_GIT_URL
             git_checkout.revision = revision

             boringssl_build_dir = api.path["start_dir"].join("src", "boringssl-build")
             api.step(
                 "configure",
                 [
                     cipd_dir.join("bin", "cmake"),
                     "-S",
                     boringssl_src_dir,
                     "-B",
                     boringssl_build_dir,
                     "-G",
                     "Ninja",
                     "-DCMAKE_INSTALL_PREFIX=",
                     "-DCMAKE_BUILD_TYPE=Release",
                     slashes(
                         api, "-DGO_EXECUTABLE=%s" % api.go.go_root.join("bin", "go")
                     ),
                 ]
                 + (
                     [
                         slashes(
                             api,
                             "-DCMAKE_ASM_NASM_COMPILER=%s"
                             % cipd_dir.join("bin", "yasm.exe"),
                         )
                     ]
                     if target_platform.is_win
                     else []
                 )
                 + [slashes(api, option) for option in options],
             )
             boringssl_install_dir = api.path["start_dir"].join("boringssl")
             with api.context(env={"DESTDIR": boringssl_install_dir}):
                 api.step(
                     "install",
                     [cipd_dir.join("ninja"), "-C", boringssl_build_dir, "install"],
                 )

         src_dir = api.path["start_dir"].join("src", "cmake")
         _, revision = api.git_checkout(
             GIT_URL, path=src_dir, fallback_ref="refs/heads/master"
         )
         git_checkout = manifest.directories[relpath(api, src_dir)].git_checkout
         git_checkout.repo_url = GIT_URL
         git_checkout.revision = revision

         build_dir = api.path["start_dir"].join("src", "cmake-build")
         api.step(
             "configure",
             [
                 cipd_dir.join("bin", "cmake"),
                 "-S",
                 src_dir,
                 "-B",
                 build_dir,
                 "-G",
                 "Ninja",
                 "-DCMAKE_INSTALL_PREFIX=",
                 "-DCMAKE_BUILD_TYPE=Release",
                 "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON",
                 "-DCMAKE_USE_OPENSSL=ON",
                 slashes(
                     api,
                     "-DOpenSSL_DIR=%s"
                     % boringssl_install_dir.join("lib", "cmake", "OpenSSL"),
                 ),
                 "-DCMake_BUILD_LTO=ON",
                 # TODO(phosek): build ncurses?
                 "-DBUILD_CursesDialog=OFF",
             ]
             + [slashes(api, option) for option in options],
         )
         version_data = api.file.read_text(
             "read cmake revision",
             build_dir.join("Source", "cmVersionConfig.h"),
             test_data='\n#define CMake_VERSION "3.22.20211015-gc5ae200"\n',
         )
         regex = re.compile(r"CMake\_VERSION (\"[a-z0-9.-]+\")")
         match = regex.search(version_data)
         if not match:
             raise api.step.StepFailure("revision string not found")  # pragma no cover
         cmake_version = match.group(1).strip(' "')
         api.step.empty("cmake version", step_text=cmake_version)
         api.step("build", [cipd_dir.join("ninja"), "-C", build_dir])
         # TODO: CMake tests are failing, disable them until we figure out why.
         # api.step('test', [cipd_dir.join('ninja'), 'test'])
         install_dir = api.path["start_dir"].join("cmake")
         with api.context(env={"DESTDIR": install_dir}):
             # CMake does not have "install/strip" target on Windows.
             if api.platform.is_win:
                 api.step(
                     "install", [cipd_dir.join("ninja"), "-C", build_dir, "install"]
                 )
             else:
                 api.step(
                     "install",
                     [cipd_dir.join("ninja"), "-C", build_dir, "install/strip"],
                 )

     api.file.write_proto(
         "source manifest", install_dir.join("source_manifest.json"), manifest, "JSONPB"
     )

     # Upload the installation to CAS.
     api.cas_util.upload(install_dir, output_property="isolated")

     if api.buildbucket.builder_id.bucket == "prod":
         # Upload the installation to CIPD for production builds.
         api.cipd_util.upload_package(
             "fuchsia/third_party/cmake/%s" % target_platform,
             install_dir,
             search_tag={"git_revision": revision},
             repository=GIT_URL,
             metadata=[("version", cmake_version)],
         )


 def GenTests(api):
     for platform in ("linux", "mac", "win"):
         yield (
             api.buildbucket_util.test(
                 platform, bucket="prod", git_repo=GIT_URL, revision="a" * 40
             )
             + api.platform.name(platform)
         )

     yield (
         api.buildbucket_util.test(
             "cross", bucket="prod", git_repo=GIT_URL, revision="a" * 40
         )
         + api.platform.name("mac")
         + api.properties(platform="mac-arm64")
     )
	# Copyright 2019 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe for building CMake."""

	from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
	from PB.recipes.fuchsia.contrib.cmake import InputProperties

	import re

	DEPS = [
	"fuchsia/buildbucket_util",
	"fuchsia/cas_util",
	"fuchsia/cipd_util",
	"fuchsia/git_checkout",
	"fuchsia/go",
	"fuchsia/macos_sdk",
	"fuchsia/platform_util",
	"fuchsia/windows_sdk",
	"recipe_engine/buildbucket",
	"recipe_engine/cipd",
	"recipe_engine/context",
	"recipe_engine/file",
	"recipe_engine/path",
	"recipe_engine/platform",
	"recipe_engine/properties",
	"recipe_engine/raw_io",
	"recipe_engine/step",
	]

	PROPERTIES = InputProperties

	BORINGSSL_GIT_URL = "https://boringssl.googlesource.com/boringssl"

	GIT_URL = "https://fuchsia.googlesource.com/third_party/github.com/Kitware/CMake"

	CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"


	def relpath(api, path):
	return str(api.path.relpath(path, api.path["start_dir"]))


	def slashes(api, path):
	# CMake only accept '/' as path delimiter. This helper function replaces '\'
	# with '/' on Windows platform.
	return path.replace("\\", "/") if api.platform.is_win else path


	def RunSteps(api, props):
	host_platform = api.platform_util.host
	target_platform = (
	api.platform_util.platform(props.platform) if props.platform else host_platform
	)

	manifest = Manifest()

	with api.step.nest("ensure packages"), api.context(infra_steps=True):
	cipd_dir = api.path["start_dir"].join("cipd")
	pkgs = api.cipd.EnsureFile()
	pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
	if target_platform.is_linux:
	pkgs.add_package(
	"fuchsia/sysroot/${platform}",
	"git_revision:a28dfa20af063e5ca00634024c85732e20220419",
	"sysroot",
	)
	if target_platform.is_win:
	pkgs.add_package(
	"fuchsia/third_party/yasm/${platform}",
	"git_revision:51af4082cc898b122b88f11fd34033fc00fad81e",
	)
	pkgs.add_package(
	"fuchsia/third_party/cmake/${platform}",
	"integration",
	)
	pkgs.add_package(
	"fuchsia/third_party/ninja/${platform}",
	"integration",
	)
	ensured = api.cipd.ensure(cipd_dir, pkgs)
	for subdir, pins in ensured.items():
	directory = manifest.directories[relpath(api, cipd_dir.join(subdir))]
	directory.cipd_server_host = CIPD_SERVER_HOST
	for pin in pins:
	directory.cipd_package[pin.package].instance_id = pin.instance_id

	with api.macos_sdk(), api.windows_sdk():
	options = []
	if target_platform.is_win:
	options.extend(
	[
	"-DCMAKE_ASM_COMPILER=%s" % cipd_dir.join("bin", "clang-cl.exe"),
	"-DCMAKE_C_COMPILER=%s" % cipd_dir.join("bin", "clang-cl.exe"),
	"-DCMAKE_CXX_COMPILER=%s" % cipd_dir.join("bin", "clang-cl.exe"),
	"-DCMAKE_MAKE_PROGRAM=%s" % cipd_dir.join("ninja.exe"),
	"-DCMAKE_SYSROOT=%s" % api.windows_sdk.sdk_dir,
	"-DCMAKE_MT=mt.exe",
	"-DCMAKE_RC=rc.exe",
	]
	)
	else:
	options.extend(
	[
	"-DCMAKE_ASM_COMPILER=%s" % cipd_dir.join("bin", "clang"),
	"-DCMAKE_C_COMPILER=%s" % cipd_dir.join("bin", "clang"),
	"-DCMAKE_CXX_COMPILER=%s" % cipd_dir.join("bin", "clang++"),
	"-DCMAKE_MAKE_PROGRAM=%s" % cipd_dir.join("ninja"),
	]
	)
	if target_platform.is_linux:
	options.extend(["-DCMAKE_SYSROOT=%s" % cipd_dir.join("sysroot")])
	if target_platform.is_mac:
	options.extend(
	[
	"-DCMAKE_SYSROOT=%s"
	% api.step(
	"xcrun",
	["xcrun", "--sdk", "macosx", "--show-sdk-path"],
	stdout=api.raw_io.output_text(
	name="sdk-path", add_output_log=True
	),
	step_test_data=lambda: api.raw_io.test_api.stream_output_text(
	"/some/xcode/path"
	),
	).stdout.strip(),
	]
	+ [
	"-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s"
	% (mode, cipd_dir.join("lib", "libc++.a"))
	for mode in ["SHARED", "MODULE", "EXE"]
	]
	)
	if target_platform != host_platform:
	options.extend(
	[
	"-DCMAKE_C_COMPILER_TARGET=%s" % target_platform.triple,
	"-DCMAKE_CXX_COMPILER_TARGET=%s" % target_platform.triple,
	"-DCMAKE_ASM_COMPILER_TARGET=%s" % target_platform.triple,
	"-DCMAKE_SYSTEM_NAME=%s"
	% target_platform.os.replace("mac", "darwin").title(),
	"-DCMAKE_SYSTEM_PROCESSOR=%s"
	% target_platform.arch.replace("amd64", "x86_64").replace(
	"arm64", "aarch64"
	),
	]
	)

	with api.step.nest("boringssl"):
	boringssl_src_dir = api.path["start_dir"].join("src", "boringssl")
	_, revision = api.git_checkout(
	BORINGSSL_GIT_URL,
	path=boringssl_src_dir,
	fallback_ref="refs/heads/master",
	)
	git_checkout = manifest.directories[
	relpath(api, boringssl_src_dir)
	].git_checkout
	git_checkout.repo_url = BORINGSSL_GIT_URL
	git_checkout.revision = revision

	boringssl_build_dir = api.path["start_dir"].join("src", "boringssl-build")
	api.step(
	"configure",
	[
	cipd_dir.join("bin", "cmake"),
	"-S",
	boringssl_src_dir,
	"-B",
	boringssl_build_dir,
	"-G",
	"Ninja",
	"-DCMAKE_INSTALL_PREFIX=",
	"-DCMAKE_BUILD_TYPE=Release",
	slashes(
	api, "-DGO_EXECUTABLE=%s" % api.go.go_root.join("bin", "go")
	),
	]
	+ (
	[
	slashes(
	api,
	"-DCMAKE_ASM_NASM_COMPILER=%s"
	% cipd_dir.join("bin", "yasm.exe"),
	)
	]
	if target_platform.is_win
	else []
	)
	+ [slashes(api, option) for option in options],
	)
	boringssl_install_dir = api.path["start_dir"].join("boringssl")
	with api.context(env={"DESTDIR": boringssl_install_dir}):
	api.step(
	"install",
	[cipd_dir.join("ninja"), "-C", boringssl_build_dir, "install"],
	)

	src_dir = api.path["start_dir"].join("src", "cmake")
	_, revision = api.git_checkout(
	GIT_URL, path=src_dir, fallback_ref="refs/heads/master"
	)
	git_checkout = manifest.directories[relpath(api, src_dir)].git_checkout
	git_checkout.repo_url = GIT_URL
	git_checkout.revision = revision

	build_dir = api.path["start_dir"].join("src", "cmake-build")
	api.step(
	"configure",
	[
	cipd_dir.join("bin", "cmake"),
	"-S",
	src_dir,
	"-B",
	build_dir,
	"-G",
	"Ninja",
	"-DCMAKE_INSTALL_PREFIX=",
	"-DCMAKE_BUILD_TYPE=Release",
	"-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON",
	"-DCMAKE_USE_OPENSSL=ON",
	slashes(
	api,
	"-DOpenSSL_DIR=%s"
	% boringssl_install_dir.join("lib", "cmake", "OpenSSL"),
	),
	"-DCMake_BUILD_LTO=ON",
	# TODO(phosek): build ncurses?
	"-DBUILD_CursesDialog=OFF",
	]
	+ [slashes(api, option) for option in options],
	)
	version_data = api.file.read_text(
	"read cmake revision",
	build_dir.join("Source", "cmVersionConfig.h"),
	test_data='\n#define CMake_VERSION "3.22.20211015-gc5ae200"\n',
	)
	regex = re.compile(r"CMake\_VERSION (\"[a-z0-9.-]+\")")
	match = regex.search(version_data)
	if not match:
	raise api.step.StepFailure("revision string not found") # pragma no cover
	cmake_version = match.group(1).strip(' "')
	api.step.empty("cmake version", step_text=cmake_version)
	api.step("build", [cipd_dir.join("ninja"), "-C", build_dir])
	# TODO: CMake tests are failing, disable them until we figure out why.
	# api.step('test', [cipd_dir.join('ninja'), 'test'])
	install_dir = api.path["start_dir"].join("cmake")
	with api.context(env={"DESTDIR": install_dir}):
	# CMake does not have "install/strip" target on Windows.
	if api.platform.is_win:
	api.step(
	"install", [cipd_dir.join("ninja"), "-C", build_dir, "install"]
	)
	else:
	api.step(
	"install",
	[cipd_dir.join("ninja"), "-C", build_dir, "install/strip"],
	)

	api.file.write_proto(
	"source manifest", install_dir.join("source_manifest.json"), manifest, "JSONPB"
	)

	# Upload the installation to CAS.
	api.cas_util.upload(install_dir, output_property="isolated")

	if api.buildbucket.builder_id.bucket == "prod":
	# Upload the installation to CIPD for production builds.
	api.cipd_util.upload_package(
	"fuchsia/third_party/cmake/%s" % target_platform,
	install_dir,
	search_tag={"git_revision": revision},
	repository=GIT_URL,
	metadata=[("version", cmake_version)],
	)


	def GenTests(api):
	for platform in ("linux", "mac", "win"):
	yield (
	api.buildbucket_util.test(
	platform, bucket="prod", git_repo=GIT_URL, revision="a" * 40
	)
	+ api.platform.name(platform)
	)

	yield (
	api.buildbucket_util.test(
	"cross", bucket="prod", git_repo=GIT_URL, revision="a" * 40
	)
	+ api.platform.name("mac")
	+ api.properties(platform="mac-arm64")
	)