[build] Add copy_tree.gni and migrate all users
Fixed: 74196
Change-Id: I2dcb8825e5fd402507d6ef8bf8a196202d559dcb
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/514622
Commit-Queue: Jay Zhuang <jayzhuang@google.com>
Reviewed-by: Adam Perry <adamperry@google.com>
Reviewed-by: Shai Barack <shayba@google.com>
diff --git a/build/copy_tree.gni b/build/copy_tree.gni
new file mode 100644
index 0000000..e220e8e
--- /dev/null
+++ b/build/copy_tree.gni
@@ -0,0 +1,113 @@
+# Copyright 2021 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Copies a directory preserving its structure.
+#
+# NOTE: Don't use this tempalte if GN's built-in `copy` can be used instead, see
+# example below, also `gn help copy`. Usually the reason to use this template is
+# the content of the directory cannot be determined in BUILD.gn. Because of
+# this, it is impossible for this template to guarantee incremental correctness.
+# For example, changes to files in a directory may not cause its mtime to
+# change, so the build system won't rerun this action when building
+# incrementally. The `inputs` parameter is required to mitigate this problem,
+# but it provides no guarantees. See more details in https://fxbug.dev/73250.
+#
+# Example:
+#
+# ```
+# copy_tree("my_copy") {
+# src_dir = "path/to/src/dir"
+# dest_dir = "path/to/dest/dir"
+# inputs = [
+# "version/of/src/dir",
+# ]
+# ignore_patterns = [
+# "*not_useful*",
+# "*.to_ignore",
+# ]
+# }
+# ```
+#
+# Use `copy` if content of dir can be determined in BUILD.gn:
+#
+# ```
+# copy("my_copy") {
+# sources = [
+# "path/to/src/dir/file1",
+# "path/to/src/dir/file2",
+# "path/to/src/dir/file3",
+# ]
+# outputs = [ "path/to/dest/dir/{{source_file_part}}" ]
+# }
+# ```
+#
+# Parameters
+#
+# src_dir (required)
+# Path to the directory to copy.
+# Type: path
+#
+# dest_dir (required)
+# Path to copy the directory to.
+# Type: path
+#
+# inputs (optional)
+# A list of files that changes when the content of the directory changes, so
+# in incremental builds a rerun of this action can be correctly triggered.
+# Type: list(path)
+# Default: empty
+#
+# ignore_patterns (optional)
+# Glob-style patterns to ignore when copying.
+# Type: list(string)
+# Default: empty
+#
+# deps
+# testonly
+# visibility
+template("copy_tree") {
+ action(target_name) {
+ # Not all inputs and outputs are listed by this action, so it is not
+ # hermetic. This is usually the very reason a user would want this template
+ # instead of, GN's built-in copy tool.
+ hermetic_deps = false
+
+ forward_variables_from(invoker,
+ [
+ "deps",
+ "dest_dir",
+ "ignore_patterns",
+ "inputs",
+ "src_dir",
+ "testonly",
+ "visibility",
+ ])
+
+ assert(defined(src_dir), "src_dir must be defined for ${target_name}")
+ assert(defined(dest_dir), "dest_dir must be defined for ${target_name}")
+
+ if (defined(inputs)) {
+ inputs += [ src_dir ]
+ } else {
+ inputs = [ src_dir ]
+ }
+
+ script = "//build/copy_tree.py"
+
+ stamp_file = "${dest_dir}.stamp"
+ args = [
+ rebase_path(src_dir, root_build_dir),
+ rebase_path(dest_dir, root_build_dir),
+ rebase_path(stamp_file, root_build_dir),
+ ]
+ if (defined(ignore_patterns)) {
+ args += [ "--ignore_patterns" ] + ignore_patterns
+ }
+
+ outputs = [
+ dest_dir,
+ stamp_file,
+ ]
+ }
+}
diff --git a/build/copy_tree.py b/build/copy_tree.py
index a2c92ec..549d0f0 100755
--- a/build/copy_tree.py
+++ b/build/copy_tree.py
@@ -17,7 +17,7 @@
params.add_argument("source", type=Path)
params.add_argument("target", type=Path)
params.add_argument("stamp", type=Path)
- params.add_argument("--ignore_pattern", action="append")
+ params.add_argument("--ignore_patterns", nargs="+")
args = params.parse_args()
if args.target.is_file():
@@ -26,8 +26,8 @@
shutil.rmtree(args.target, ignore_errors=True)
ignore = None
- if args.ignore_pattern:
- ignore = shutil.ignore_patterns(*args.ignore_pattern)
+ if args.ignore_patterns:
+ ignore = shutil.ignore_patterns(*args.ignore_patterns)
shutil.copytree(args.source, args.target, symlinks=True, ignore=ignore)
diff --git a/build/python/BUILD.gn b/build/python/BUILD.gn
index e3409a86..a9004a9 100644
--- a/build/python/BUILD.gn
+++ b/build/python/BUILD.gn
@@ -2,40 +2,27 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import("//build/copy_tree.gni")
import("//build/python/python.gni")
import("//build/testing/host_test_data.gni")
# Make the host python prebuilt available in the out dir so it
# can be used in python_host_test without making multiple copies.
-# A regular copy() doesn't work properly with directories.
-action("copy_lib") {
- # TODO(https://fxbug.dev/73140): fix hermeticity of copy_tree.py
- hermetic_deps = false
- script = "//build/copy_tree.py"
- args = [
- rebase_path("//prebuilt/third_party/python3/${host_platform}/lib",
- root_build_dir),
- rebase_path("${python_out_dir}/lib", root_build_dir),
- rebase_path("${python_out_dir}/lib.stamp", root_build_dir),
-
+copy_tree("copy_lib") {
+ src_dir = "//prebuilt/third_party/python3/${host_platform}/lib"
+ dest_dir = "${python_out_dir}/lib"
+ ignore_patterns = [
# The .pyc files may be produced while this action is running,
# so we don't want to try to copy them while the're being written.
- "--ignore_pattern",
"__pycache__",
- "--ignore_pattern",
"*.pyc.*",
- "--ignore_pattern",
"*.pyc",
]
inputs = [
# This file should change when the package version changes.
"//prebuilt/third_party/python3/${host_platform}/include/python${python_version}/pyconfig.h",
]
- outputs = [
- "${python_out_dir}/lib",
- "${python_out_dir}/lib.stamp",
- ]
visibility = [ ":*" ]
}
diff --git a/tools/auto_owners/BUILD.gn b/tools/auto_owners/BUILD.gn
index b4bc131..77b98ce 100644
--- a/tools/auto_owners/BUILD.gn
+++ b/tools/auto_owners/BUILD.gn
@@ -2,9 +2,11 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import("//build/copy_tree.gni")
import("//build/host.gni")
import("//build/rust/rustc_binary.gni")
import("//build/rust/rustc_test.gni")
+import("//build/testing/host_test_data.gni")
if (is_host) {
crate_deps = [
@@ -58,38 +60,19 @@
outputs = [ "$test_output_dir/{{source_target_relative}}" ]
}
- # we need to copy the GN binary from the builder into the test output directory so we can
- # run it on a different bot than does the builds
- # TODO(fxbug.dev/74196) extract this, cargo-gnaw's, potentially others to a file in //build
- template("_copy_tool") {
- action(target_name) {
- hermetic_deps = false
- source = "//prebuilt/third_party/${invoker.tool}/${host_platform}"
- target = "${invoker.out_dir}/${invoker.tool}"
- inputs = [ "$source/.versions/${invoker.tool}.cipd_version" ]
- script = "//build/copy_tree.py"
- stamp = "${target}.stamp"
- args = [
- rebase_path(source, root_build_dir),
- rebase_path(target, root_build_dir),
- rebase_path(stamp, root_build_dir),
- ]
-
- # In theory we could use the version file as our output, but that causes
- # a problem: ninja will attempt to mkdir $target, and if $target exists
- # but is not a directory, the build will fail. So we use a stamp file in
- # the parent directory instead.
- outputs = [ stamp ]
-
- metadata = {
- test_runtime_deps = [ target ]
- }
- }
+ # we need to copy the GN binary from the builder into the test output
+ # directory so we can run it on a different bot than does the builds.
+ auto_owners_gn_out_dir = "${test_output_dir}/runfiles/gn"
+ copy_tree("auto_owners_gn_copy") {
+ src_dir = "//prebuilt/third_party/gn/${host_platform}"
+ dest_dir = auto_owners_gn_out_dir
+ inputs = [ "${src_dir}/.versions/gn.cipd_version" ]
}
- _copy_tool("auto_owners_gn") {
- tool = "gn"
- out_dir = "$test_output_dir/runfiles"
+ # Make the copied directory available at test runtime.
+ host_test_data("auto_owners_gn") {
+ sources = [ auto_owners_gn_out_dir ]
+ deps = [ ":auto_owners_gn_copy" ]
}
}
diff --git a/tools/cargo-gnaw/tests/BUILD.gn b/tools/cargo-gnaw/tests/BUILD.gn
index 7665ab6..6a1d761 100644
--- a/tools/cargo-gnaw/tests/BUILD.gn
+++ b/tools/cargo-gnaw/tests/BUILD.gn
@@ -2,6 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import("//build/copy_tree.gni")
import("//build/rust/rustc_binary.gni")
import("//build/rust/rustc_library.gni")
import("//build/rust/rustc_test.gni")
@@ -127,39 +128,31 @@
outputs = [ "$root_out_dir/test_data/tools/cargo-gnaw/tests/{{source_target_relative}}" ]
}
- template("_copy_tool") {
- action(target_name) {
- hermetic_deps = false
- source = "//prebuilt/third_party/${invoker.tool}/${host_platform}"
- target = "${invoker.out_dir}/${invoker.tool}"
- inputs = [ "$source/.versions/${invoker.tool}.cipd_version" ]
- script = "//build/copy_tree.py"
- stamp = "${target}.stamp"
- args = [
- rebase_path(source, root_build_dir),
- rebase_path(target, root_build_dir),
- rebase_path(stamp, root_build_dir),
- ]
-
- # In theory we could use the version file as our output, but that causes
- # a problem: ninja will attempt to mkdir $target, and if $target exists
- # but is not a directory, the build will fail. So we use a stamp file in
- # the parent directory instead.
- outputs = [ stamp ]
-
- metadata = {
- test_runtime_deps = [ target ]
- }
- }
+ cargo_gnaw_rust_out_dir =
+ "${root_out_dir}/test_data/tools/cargo-gnaw/runfiles/rust"
+ copy_tree("cargo_gnaw_rust_copy") {
+ src_dir = "//prebuilt/third_party/rust/${host_platform}"
+ dest_dir = cargo_gnaw_rust_out_dir
+ inputs = [ "${src_dir}/.versions/rust.cipd_version" ]
}
- _copy_tool("cargo_gnaw_rust") {
- tool = "rust"
- out_dir = "$root_out_dir/test_data/tools/cargo-gnaw/runfiles"
+ # Make the copied directory available at test runtime.
+ host_test_data("cargo_gnaw_rust") {
+ sources = [ cargo_gnaw_rust_out_dir ]
+ deps = [ ":cargo_gnaw_rust_copy" ]
}
- _copy_tool("cargo_gnaw_gn") {
- tool = "gn"
- out_dir = "$root_out_dir/test_data/tools/cargo-gnaw/runfiles"
+ cargo_gnaw_gn_out_dir =
+ "${root_out_dir}/test_data/tools/cargo-gnaw/runfiles/gn"
+ copy_tree("cargo_gnaw_gn_copy") {
+ src_dir = "//prebuilt/third_party/gn/${host_platform}"
+ dest_dir = cargo_gnaw_gn_out_dir
+ inputs = [ "${src_dir}/.versions/gn.cipd_version" ]
+ }
+
+ # Make the copied directory available at test runtime.
+ host_test_data("cargo_gnaw_gn") {
+ sources = [ cargo_gnaw_gn_out_dir ]
+ deps = [ ":cargo_gnaw_gn_copy" ]
}
}
diff --git a/tools/symbolizer/BUILD.gn b/tools/symbolizer/BUILD.gn
index 2c4e695..0458ade3 100644
--- a/tools/symbolizer/BUILD.gn
+++ b/tools/symbolizer/BUILD.gn
@@ -2,6 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+import("//build/copy_tree.gni")
import("//build/host.gni")
import("//build/sdk/sdk_host_tool.gni")
import("//build/test.gni")
@@ -78,29 +79,27 @@
[ "$root_out_dir/test_data/symbolizer/test_cases/{{source_file_part}}" ]
}
- # Manually copy the symbol directory since host_test_data does not support copying directory
- # any more (fxbug.dev/73250). It's not a big issue for us since the symbol directory seldom
- # changes, and when it changes, we usually only add new files to it which will change the
- # directory's mtime.
- action("copy_e2e_test_symbols") {
- hermetic_deps = false
- script = "//build/copy_tree.py"
- inputs = [ "//prebuilt/test_data/symbolizer/symbols" ]
- outputs = [
- "$root_out_dir/test_data/symbolizer/symbols",
- "$root_out_dir/test_data/symbolizer/symbols.stamp",
- ]
- args = rebase_path(inputs + outputs)
- metadata = {
- test_runtime_deps = outputs
- }
+ # Manually copy the symbol directory since host_test_data does not support
+ # copying directory any more (fxbug.dev/73250). It's not a big issue for us
+ # since the symbol directory seldom changes, and when it changes, we usually
+ # only add new files to it which will change the directory's mtime.
+ e2e_test_symbols_out_dir = "$root_out_dir/test_data/symbolizer/symbols"
+ copy_tree("e2e_test_symbols_copy") {
+ src_dir = "//prebuilt/test_data/symbolizer/symbols"
+ dest_dir = e2e_test_symbols_out_dir
+ }
+
+ # Make the copied directory available at test runtime.
+ host_test_data("e2e_test_symbols") {
+ sources = [ e2e_test_symbols_out_dir ]
+ deps = [ ":e2e_test_symbols_copy" ]
}
test("symbolizer_e2e_tests") {
sources = [ "e2e_test.cc" ]
deps = [
- ":copy_e2e_test_symbols",
":e2e_test_cases",
+ ":e2e_test_symbols",
":src",
"//third_party/googletest:gtest",
]
