makeroot/makeroot.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// The makeroot.go script copies a GOROOT. It is a precursor to makefuchsia.go,
// used to stage a clean GOROOT into the build directory.
// Note: this script exists because the `go` tool globs for inputs, as such, a
// copy tool is required that will actively remove files that are not found in
// the source.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/fs"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

var (
	sourceGoroot = flag.String("source-goroot", "", "GOROOT to copy from")
	targetGoroot = flag.String("target-goroot", "", "GOROOT to copy into")
	depfilePath  = flag.String("depfile", "", "depfile to write into")
	stampPath    = flag.String("stamp-file", "", "Path of a file to create upon completion")
)

// Returns true if `file` exists as a file.
func fileExists(file string) bool {
	info, err := os.Stat(file)
	if os.IsNotExist(err) {
		return false
	}
	return !info.IsDir()
}

// Compares two files line-by-line and returns false as soon as a mismatch is found, otherwise returns true.
func filesMatchByLine(file1 string, file2 string) bool {
	f1, err := os.Open(file1)
	if err != nil {
		log.Fatal(err)
	}
	defer f1.Close()

	f2, err := os.Open(file2)
	if err != nil {
		log.Fatal(err)
	}
	defer f2.Close()

	scan1 := bufio.NewScanner(f1)
	scan2 := bufio.NewScanner(f2)
	// Scan line by line, return false on first difference.
	for scan1.Scan() {
		if !scan2.Scan() {
			// scan1 has content, but scan2 already ended.
			return false
		}
		if !bytes.Equal(scan1.Bytes(), scan2.Bytes()) {
			return false
		}
	}
	if scan2.Scan() {
		// scan2 has more content, but scan1 already ended.
		return false
	}

	return true
}

// Copy a file with its permissions.
func copyFileWithMode(dst string, src string) error {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	info, err := srcFile.Stat()
	if err != nil {
		log.Fatal(err)
	}

	// Preserve the mode of the src in the dst file.
	if fileExists(dst) {
		if err := os.Remove(dst); err != nil {
			return err
		}
	}
	dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode())
	if err != nil {
		log.Fatal(err)
	}
	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)

	return err
}

// Returns a map of path -> IsDir.  Paths are relative to `root`.
func listFilesDirsRecursively(root string) map[string]bool {
	entries := map[string]bool{}
	err := filepath.WalkDir(root, func(path string, info fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if path != root {
			relpath, err := filepath.Rel(root, path)
			if err != nil {
				return err
			}
			entries[relpath] = info.IsDir()
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	return entries
}

// From a root directory, remove all files that are not in the set of `keepers`.
// Returns true if any file was actually removed.
func removeIfAbsent(root string, keepers map[string]bool) (bool, error) {
	somethingRemoved := false
	if _, err := os.Stat(root); os.IsNotExist(err) {
		return somethingRemoved, nil
	}
	err := filepath.WalkDir(root, func(path string, info fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if path != root {
			relpath, err := filepath.Rel(root, path)
			if err != nil {
				return err
			}
			// Only remove files, leave dirs alone.
			if _, ok := keepers[relpath]; !ok {
				if !info.IsDir() {
					// The following directories contain compilation outputs and should be
					// ignored.
					if strings.HasPrefix(relpath, "bin/") || strings.HasPrefix(relpath, "pkg/") {
						// skip the file if it matches the prefix.
						return nil
					}

					// The following files are all compiled sources and should be ignored
					// as expected files to find in the outdir.
					generatedSources := []string{"zdefaultcc.go", "zbootstrap.go", "zcgo.go", "zversion.go", "zzipdata.go"}
					for _, generatedSource := range generatedSources {
						if filepath.Base(relpath) == generatedSource {
							// skipping generated source file.
							return nil
						}
					}

					fmt.Printf("removing:  %s\n", relpath)
					somethingRemoved = true
					if err := os.Remove(path); err != nil {
						return err
					}
				}
			}
		}
		return nil
	})
	return somethingRemoved, err
}

// Essentially "rsync" from sourceGoroot to targetGoroot.
func main() {
	log.SetFlags(log.Lshortfile)
	flag.Parse()

	// Remove all files from the targetGoroot that are not in the sourceGoroot.
	// Keep the rest for now so they can be compared before copied-over.
	somethingRemoved, err := removeIfAbsent(*targetGoroot, listFilesDirsRecursively(*sourceGoroot))
	if err != nil {
		log.Fatal(err)
	}

	var depfileBuf bytes.Buffer // depfile contents
	depfileBuf.WriteString(*stampPath)
	depfileBuf.WriteByte(':')

	git, err := exec.LookPath("git")
	if err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command(git, "--no-optional-locks", "-C", *sourceGoroot, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
	out, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	targetUpdated := somethingRemoved
	for _, path := range bytes.Split(out, []byte{0}) {
		if len(path) == 0 {
			continue
		}
		path := string(path)
		src := filepath.Join(*sourceGoroot, path)
		dst := filepath.Join(*targetGoroot, path)

		// mkdir with parents
		dstDir := filepath.Dir(dst)
		if _, err := os.Stat(dstDir); os.IsNotExist(err) {
			info, err := os.Stat(filepath.Dir(src))
			if err != nil {
				log.Fatal(err)
			}
			if err := os.MkdirAll(dstDir, info.Mode()); err != nil {
				log.Fatal(err)
			}
		} else if err != nil {
			log.Fatal(err)
		}

		// Avoid touching timestamp if contents did not change to allow
		// ninja to prune downstream actions (restat).
		if !fileExists(dst) || !filesMatchByLine(dst, src) {
			if err := copyFileWithMode(dst, src); err != nil {
				// Renaming or deleting files will cause git ls-files to return the old (or deleted)
				// paths unless the change is staged, so ignore these errors.
				if os.IsNotExist(err) {
					log.Print("could not open file: ", src)
					continue
				}
				log.Fatal(err)
			}
			targetUpdated = true
		}
		depfileBuf.WriteByte(' ')
		depfileBuf.WriteString(src)
	}
	depfileBuf.WriteByte('\n')

	// Write the depfile.
	if err := os.WriteFile(*depfilePath, depfileBuf.Bytes(), os.ModePerm); err != nil {
		log.Fatal(err)
	}

	// Touch the stampfile if any file was actually updated.
	if !fileExists(*stampPath) || targetUpdated {
		f, err := os.OpenFile(*stampPath, os.O_CREATE|os.O_TRUNC, os.ModePerm)
		if err != nil {
			log.Fatal(err)
		}
		if err := f.Close(); err != nil {
			log.Fatal(err)
		}
		now := time.Now()
		if err := os.Chtimes(*stampPath, now, now); err != nil {
			log.Fatal(err)
		}
	}
}