makefuchsia.go: use prebuilt compiler
Just rebuild cmd/go and cmd/cgo with new defaults.
Pulls 60s out of the build process for reduced networking builds.
Change-Id: Idd8e42c35e8f549e64b0fd1f07941e1759fb5cfe
diff --git a/BUILD.gn b/BUILD.gn
index 91ae57b..fefb1f5 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -5,27 +5,42 @@
# The go_runtime action wraps go_toolchain, rebuilding just the standard
# if it has changed.
action("go_runtime") {
- outputs = [ "$root_gen_dir/goroot/stdlib.done" ]
+ outputs = [ "$root_gen_dir/gostdlib.done" ]
depfile = "$root_gen_dir/gostdlib.d"
script = rebase_path("//buildtools/go")
args = [
"run",
rebase_path("makestdlib.go", root_build_dir),
+ "-fuchsia-root",
+ rebase_path("//."),
+ "-depfile",
+ rebase_path(depfile),
"-root-out-dir",
rebase_path("$root_out_dir"),
"-root-gen-dir",
rebase_path("$root_gen_dir"),
- "-depfile",
- rebase_path(depfile),
+ "-target-cpu",
+ target_cpu,
]
deps = [ ":go_toolchain" ]
+
+ if (host_os == "linux") {
+ args += [
+ "--bootstrap-path",
+ rebase_path("//buildtools/linux64/go"),
+ ]
+ } else if (host_os == "mac") {
+ args += [
+ "--bootstrap-path",
+ rebase_path("//buildtools/mac/go"),
+ ]
+ } else {
+ assert(false, "Go cross compiler currently only supported on Linux and Mac")
+ }
}
action("go_toolchain") {
- outputs = [ "$root_gen_dir/goroot/bin/go" ]
- depfile_path = "$root_gen_dir/goroot.d"
-
- depfile = depfile_path
+ outputs = [ "$root_gen_dir/goroot.done" ]
script = rebase_path("//buildtools/go")
args = [
@@ -33,18 +48,8 @@
rebase_path("makefuchsia.go", root_build_dir),
"-fuchsia-root",
rebase_path("//."),
- "-root-out-dir",
- rebase_path("$root_out_dir"),
"-root-gen-dir",
rebase_path("$root_gen_dir"),
- "-depfile",
- rebase_path(depfile_path),
- "-host-os",
- host_os,
- "-host-cpu",
- host_cpu,
- "-target-cpu",
- target_cpu,
]
if (host_os == "linux") {
@@ -61,3 +66,7 @@
assert(false, "Go cross compiler currently only supported on Linux and Mac")
}
}
+
+pool("gobuild_pool") {
+ depth = 1
+}
diff --git a/makefuchsia.go b/makefuchsia.go
index eb0510b..581bc6a 100644
--- a/makefuchsia.go
+++ b/makefuchsia.go
@@ -2,19 +2,15 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-// The makefuchsia.go script builds a GOROOT for cross compiling Fuchsia
-// binaries from the host. It is used as part of the GN+ninja build.
+// The makefuchsia.go script symlinks the src contents from third_party/go
+// into the active GOROOT in ./buildtools/{ARCH}/go.
package main
import (
- "bytes"
"flag"
- "fmt"
- "io"
"io/ioutil"
"log"
"os"
- "os/exec"
"path/filepath"
"strings"
"sync"
@@ -22,149 +18,28 @@
var (
flagFuchsiaRoot = flag.String("fuchsia-root", "", "Path to root of the Fuchsia project")
- flagDepfile = flag.String("depfile", "", "Toolchain depfile")
- flagRootOutDir = flag.String("root-out-dir", "", "GN variable")
flagRootGenDir = flag.String("root-gen-dir", "", "GN variable")
- flagHostOS = flag.String("host-os", "", "GN variable")
- flagHostCPU = flag.String("host-cpu", "", "GN variable")
- flagTargetCPU = flag.String("target-cpu", "", "GN variable")
flagBootstrapPath = flag.String("bootstrap-path", "", "Path to bootstrap Go installation")
)
-func genDepfileToolchain() {
- // Change any of the src/cmd/* files, and we rebuild the entire toolchain.
- // This is not perfect. The toolchain does depend on other stdlib packages.
- // But in practice it works well, changes to the rest of the stdlib don't
- // have big semantic effects on the compiler/linker/assembler, because
- // these programs are designed to be built with old versions of the stdlib
- // for bootstrapping.
- path := *flagDepfile
- if path == "" {
- return
- }
- os.Remove(path)
- buf := new(bytes.Buffer)
- buf.WriteString(relGenDir() + "/goroot/bin/go:")
- if err := collectDeps(buf, "goroot/src/cmd"); err != nil {
- log.Fatal("gen depfile toolchain: %v", err)
- }
- if err := ioutil.WriteFile(path, buf.Bytes(), 0664); err != nil {
- log.Fatal("gen depfile toolchain: %v", err)
- }
-}
-
-func relGenDir() string {
- return (*flagRootGenDir)[len(*flagRootOutDir)+1:]
-}
-
-func collectDeps(w io.Writer, path string) error {
- fn := func(path string, info os.FileInfo, err error) error {
- if info.IsDir() || strings.HasSuffix(path, "_test.go") {
- return err
- }
- fmt.Fprintf(w, " %s", path)
- return err
- }
- return filepath.Walk(filepath.Join(*flagRootGenDir, path), fn)
-}
-
-func genGoroot() {
- if *flagFuchsiaRoot == "" {
- log.Fatal("must specify -fuchsia-root when building toolchain")
- }
-
- // Setup a minimal GOROOT in a target-specific generated files
- // directory. This ensures that the ninja build for a target
- // does not touch the source directories, an important invariant
- // for building against multiple architectures.
- dstGoroot := filepath.Join(*flagRootGenDir, "goroot")
- if err := os.RemoveAll(dstGoroot); err != nil {
+func mkdir(path ...string) {
+ if err := os.MkdirAll(join(path...), 0755); err != nil {
log.Fatal(err)
}
- dst := filepath.Join(dstGoroot, "src")
- if err := os.MkdirAll(dst, 0755); err != nil {
- log.Fatal(err)
- }
- if err := os.MkdirAll(filepath.Join(dstGoroot, "misc"), 0755); err != nil {
- log.Fatal(err)
- }
- if err := os.Symlink(filepath.Join(*flagFuchsiaRoot, "third_party/go/misc/fuchsia"), filepath.Join(dstGoroot, "misc/fuchsia")); err != nil {
- log.Fatalf("goroot misc symlink failed: %v", err)
- }
- src := filepath.Join(*flagFuchsiaRoot, "third_party/go/src")
- cmd := exec.Command("git", "log", "-n", "1", "--format=format:fuchsia:%h %cd", "HEAD")
- cmd.Dir = src
- verData, err := cmd.Output()
- if err != nil {
- log.Fatalf("could not get third_party/go git version: %v", err)
- }
- if err := ioutil.WriteFile(filepath.Join(dstGoroot, "VERSION"), verData, 0664); err != nil {
- log.Fatalf("could not write VERSION: %v", err)
- }
-
- // The make.bash script adds files to the packages:
- //
- // src/cmd
- // src/go/build
- // src/runtime
- //
- // So we deep symlink these subtrees in the minimal GOROOT.
- // For the rest of the packages, symlinking the top-level
- // directories is enough.
- deepSymlink := map[string]bool{"cmd": true, "go": true, "runtime": true}
- srcFiles, err := filepath.Glob(filepath.Join(src, "*"))
- if err != nil {
- log.Fatalf("could not glob source files: %v", err)
- }
-
- var wg sync.WaitGroup
- for _, file := range srcFiles {
- base := filepath.Base(file)
- if !deepSymlink[base] {
- //log.Printf("shallow link %s (base=%s)", file, base)
- if err := os.Symlink(file, filepath.Join(dst, base)); err != nil {
- log.Fatalf("goroot symlink failed: %v", err)
- }
- continue
- }
- file := file
- deepSymlink := func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if strings.HasSuffix(path, "_test.go") {
- return nil
- }
- if filepath.Base(path) == "testdata" {
- return filepath.SkipDir
- }
- if info.IsDir() {
- return nil
- }
- dstPath := filepath.Join(dst, path[len(src)+1:])
- os.MkdirAll(filepath.Dir(dstPath), 0775)
- //log.Printf("deepSymlink path=%s -> dst=%s", path, dstPath)
- return os.Symlink(path, dstPath)
- }
- wg.Add(1)
- go func() {
- if err := filepath.Walk(file, deepSymlink); err != nil {
- log.Fatalf("goroot deep symlink failed: %v", err)
- }
- wg.Done()
- }()
- }
- wg.Wait()
}
-func convCPU(cpu string) (arch string) {
- switch cpu {
- case "x64":
- return "amd64"
- case "arm64":
- return "arm64"
+func join(path ...string) string { return filepath.Join(path...) }
+
+func rm(path string) {
+ if err := os.RemoveAll(path); err != nil {
+ log.Fatal(err)
}
- return ""
+}
+
+func symlink(src, dst string) {
+ if err := os.Symlink(src, dst); err != nil {
+ log.Fatal(err)
+ }
}
func main() {
@@ -176,46 +51,98 @@
log.Fatal("must specify -root-gen-dir")
}
- genGoroot()
- genDepfileToolchain()
-
- var gohostos string
- switch *flagHostOS {
- case "linux":
- gohostos = "linux"
- case "mac":
- gohostos = "darwin"
- default:
- log.Fatalf("unsupported host OS: %s", *flagHostOS)
+ if *flagFuchsiaRoot == "" {
+ log.Fatal("must specify -fuchsia-root when building toolchain")
}
- gohostarch := convCPU(*flagHostCPU)
- if gohostarch == "" {
- log.Fatalf("unsupported host CPU: %s", *flagHostCPU)
+ srcGoroot := join(*flagFuchsiaRoot, "third_party/go")
+ dstGoroot := *flagBootstrapPath
+ src := join(srcGoroot, "src")
+ dst := join(dstGoroot, "src")
+
+ // The make.bash script adds these files to the tree.
+ // We make a copy of them now, blow away the src tree,
+ // shallow symlink what we can, and deep-symlink any directory
+ // containing one of these generated files.
+ genFiles := []string{
+ "cmd/cgo/zdefaultcc.go",
+ "cmd/go/zdefaultcc.go",
+ "cmd/go/zosarch.go",
+ "cmd/internal/obj/zbootstrap.go",
+ "go/build/zcgo.go",
+ "runtime/internal/sys/zversion.go",
+ }
+ deepSymlink := map[string]bool{}
+ for _, file := range genFiles {
+ deepSymlink[file[:strings.Index(file, "/")]] = true
+ }
+ genFilesContents := map[string][]byte{}
+ for _, file := range genFiles {
+ b, err := ioutil.ReadFile(join(dstGoroot, "src", file))
+ if err != nil {
+ log.Fatal(err)
+ }
+ genFilesContents[file] = b
}
- goarch := convCPU(*flagTargetCPU)
- if gohostarch == "" {
- log.Fatalf("unsupported target CPU: %s", *flagTargetCPU)
- }
+ rm(dst)
+ mkdir(dst)
+ rm(join(dstGoroot, "misc/fuchsia"))
+ symlink(join(srcGoroot, "misc/fuchsia"), join(dstGoroot, "misc/fuchsia"))
- cmd := exec.Command(filepath.Join(*flagRootGenDir, "goroot/src/make.bash"))
- cmd.Dir = filepath.Join(*flagRootGenDir, "goroot/src")
- cmd.Env = []string{
- "PATH=" + os.Getenv("PATH"),
- "GOROOT_BOOTSTRAP=" + *flagBootstrapPath,
- "CGO_ENABLED=1",
- "FUCHSIA=" + *flagFuchsiaRoot,
- "MAGENTA=" + filepath.Join(*flagFuchsiaRoot, "magenta"),
- "CC_FOR_TARGET=" + filepath.Join(*flagRootGenDir, "goroot/misc/fuchsia/gccwrap.sh"),
- "GOHOSTOS=" + gohostos,
- "GOHOSTARCH=" + gohostarch,
- "GOOS=fuchsia",
- "GOARCH=" + goarch,
- }
- out, err := cmd.CombinedOutput()
+ srcFiles, err := filepath.Glob(join(src, "*"))
if err != nil {
- log.Printf("make.bash failed: %s", string(out))
- os.Exit(1)
+ log.Fatalf("could not glob source files: %v", err)
+ }
+
+ var wg sync.WaitGroup
+ for _, file := range srcFiles {
+ base := filepath.Base(file)
+ if !deepSymlink[base] {
+ symlink(file, join(dst, base))
+ continue
+ }
+ file := file
+ deepSymlink := func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if strings.HasSuffix(path, "_test.go") {
+ return nil
+ }
+ if strings.HasSuffix(path, "go/zdefaultcc.go") {
+ return nil // created below
+ }
+ if filepath.Base(path) == "testdata" {
+ return filepath.SkipDir
+ }
+ if info.IsDir() {
+ return nil
+ }
+ dstPath := join(dst, path[len(src)+1:])
+ mkdir(filepath.Dir(dstPath))
+ symlink(path, dstPath)
+ return nil
+ }
+ wg.Add(1)
+ go func() {
+ if err := filepath.Walk(file, deepSymlink); err != nil {
+ log.Fatalf("goroot deep symlink failed: %v", err)
+ }
+ wg.Done()
+ }()
+ }
+ wg.Wait()
+
+ for file, contents := range genFilesContents {
+ if err := ioutil.WriteFile(join(dstGoroot, "src", file), contents, 0664); err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ donePath := join(*flagRootGenDir, "goroot.done")
+ rm(donePath)
+ if err := ioutil.WriteFile(donePath, []byte("done"), 0664); err != nil {
+ log.Fatalf("writing goroot.done failed: %v", err)
}
}
diff --git a/makestdlib.go b/makestdlib.go
index 767a8e1..2d4f9a7 100644
--- a/makestdlib.go
+++ b/makestdlib.go
@@ -17,12 +17,16 @@
"os/exec"
"path/filepath"
"strings"
+ "sync"
)
var (
- flagDepfile = flag.String("depfile", "", "Stdlib depfile")
- flagRootOutDir = flag.String("root-out-dir", "", "GN variable")
- flagRootGenDir = flag.String("root-gen-dir", "", "GN variable")
+ flagFuchsiaRoot = flag.String("fuchsia-root", "", "Path to root of the Fuchsia project")
+ flagDepfile = flag.String("depfile", "", "Stdlib depfile")
+ flagRootOutDir = flag.String("root-out-dir", "", "GN variable")
+ flagRootGenDir = flag.String("root-gen-dir", "", "GN variable")
+ flagTargetCPU = flag.String("target-cpu", "", "GN variable")
+ flagBootstrapPath = flag.String("bootstrap-path", "", "Path to bootstrap Go installation")
)
func genDepfileStdlib() {
@@ -35,8 +39,8 @@
}
os.Remove(path)
buf := new(bytes.Buffer)
- buf.WriteString(relGenDir() + "/goroot/stdlib.done:")
- if err := collectDeps(buf, "goroot/src"); err != nil {
+ buf.WriteString(relGenDir() + "/gostdlib.done:")
+ if err := collectDeps(buf); err != nil {
log.Fatal("gen depfile stdlib: %v", err)
}
if err := ioutil.WriteFile(path, buf.Bytes(), 0664); err != nil {
@@ -48,7 +52,7 @@
return (*flagRootGenDir)[len(*flagRootOutDir)+1:]
}
-func collectDeps(w io.Writer, path string) error {
+func collectDeps(w io.Writer) error {
fn := func(path string, info os.FileInfo, err error) error {
if info.IsDir() || strings.HasSuffix(path, "_test.go") {
return err
@@ -56,7 +60,17 @@
fmt.Fprintf(w, " %s", path)
return err
}
- return filepath.Walk(filepath.Join(*flagRootGenDir, path), fn)
+ return filepath.Walk(filepath.Join(*flagBootstrapPath, "src"), fn)
+}
+
+func convCPU(cpu string) (arch string) {
+ switch cpu {
+ case "x64":
+ return "amd64"
+ case "arm64":
+ return "arm64"
+ }
+ return ""
}
func main() {
@@ -68,15 +82,56 @@
log.Fatal("must specify -root-gen-dir")
}
+ goarchTarget := convCPU(*flagTargetCPU)
+ if goarchTarget == "" {
+ log.Fatalf("unsupported target CPU: %s", *flagTargetCPU)
+ }
+
genDepfileStdlib()
- cmd := exec.Command(filepath.Join(*flagRootGenDir, "goroot/bin/go"), "install", "std")
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- if err := cmd.Run(); err != nil {
- log.Fatal("go install std failed")
- }
- donePath := filepath.Join(*flagRootGenDir, "goroot/stdlib.done")
+ var wg sync.WaitGroup
+
+ wg.Add(1)
+ go func() {
+ // Generate the stdlib for the host.
+ // It is changed by third_party/go srcs being symlinked into buildtools.
+ cmd := exec.Command(filepath.Join(*flagBootstrapPath, "bin/go"), "install", "std")
+ cmd.Env = []string{
+ "PATH=" + os.Getenv("PATH"),
+ "GOROOT=" + *flagBootstrapPath,
+ }
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ if err := cmd.Run(); err != nil {
+ log.Fatal("go install std for host failed")
+ }
+ wg.Done()
+ }()
+
+ wg.Add(1)
+ go func() {
+ // Now generate the stdlib for the Fuchsia target.
+ cmd := exec.Command(filepath.Join(*flagBootstrapPath, "bin/go"), "install", "std")
+ cmd.Env = []string{
+ "PATH=" + os.Getenv("PATH"),
+ "GOOS=fuchsia",
+ "GOARCH=" + goarchTarget,
+ "GOROOT=" + *flagBootstrapPath,
+ "CGO_ENABLED=1",
+ "CC=" + filepath.Join(*flagBootstrapPath, "misc/fuchsia/gccwrap.sh"),
+ "MAGENTA=" + filepath.Join(*flagFuchsiaRoot, "magenta"), // used by gccwrap.sh
+ }
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ if err := cmd.Run(); err != nil {
+ log.Fatal("go install std for target failed")
+ }
+ wg.Done()
+ }()
+
+ wg.Wait()
+
+ donePath := filepath.Join(*flagRootGenDir, "gostdlib.done")
os.Remove(donePath)
if err := ioutil.WriteFile(donePath, []byte("done"), 0664); err != nil {
log.Fatalf("writing done file failed: %v", err)
