go/tools/releaser/file.go

// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

var repoRootState = struct {
	once sync.Once
	dir  string
	err  error
}{}

// repoRoot returns the workspace root directory. If this program was invoked
// with 'bazel run', repoRoot returns the BUILD_WORKSPACE_DIRECTORY environment
// variable. Otherwise, repoRoot walks up the directory tree and finds a
// WORKSPACE file.
func repoRoot() (string, error) {
	repoRootState.once.Do(func() {
		if wsDir := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); wsDir != "" {
			repoRootState.dir = wsDir
			return
		}
		dir, err := os.Getwd()
		if err != nil {
			repoRootState.err = err
			return
		}
		for {
			_, err := os.Stat(filepath.Join(dir, "WORKSPACE"))
			if err == nil {
				repoRootState.dir = dir
				return
			}
			if err != os.ErrNotExist {
				repoRootState.err = err
				return
			}
			parent := filepath.Dir(dir)
			if parent == dir {
				repoRootState.err = errors.New("could not find workspace directory")
				return
			}
			dir = parent
		}
	})
	return repoRootState.dir, repoRootState.err
}

// extractArchive extracts a zip or tar.gz archive opened in f, into the
// directory dir, stripping stripPrefix from each entry before extraction.
// name is the name of the archive, used for error reporting.
func extractArchive(f *os.File, name, dir, stripPrefix string) (err error) {
	if strings.HasSuffix(name, ".zip") {
		return extractZip(f, name, dir, stripPrefix)
	}
	if strings.HasSuffix(name, ".tar.gz") {
		zr, err := gzip.NewReader(f)
		if err != nil {
			return fmt.Errorf("extracting %s: %w", name, err)
		}
		defer func() {
			if cerr := zr.Close(); err == nil && cerr != nil {
				err = cerr
			}
		}()
		return extractTar(zr, name, dir, stripPrefix)
	}
	return fmt.Errorf("could not determine archive format from extension: %s", name)
}

func extractZip(zf *os.File, name, dir, stripPrefix string) (err error) {
	stripPrefix += "/"
	fi, err := zf.Stat()
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			err = fmt.Errorf("extracting zip %s: %w", name, err)
		}
	}()

	zr, err := zip.NewReader(zf, fi.Size())
	if err != nil {
		return err
	}

	extractFile := func(f *zip.File) (err error) {
		defer func() {
			if err != nil {
				err = fmt.Errorf("extracting %s: %w", f.Name, err)
			}
		}()
		outPath, err := extractedPath(dir, stripPrefix, f.Name)
		if err != nil {
			return err
		}
		if strings.HasSuffix(f.Name, "/") {
			return os.MkdirAll(outPath, 0777)
		}
		r, err := f.Open()
		if err != nil {
			return err
		}
		defer r.Close()
		parent := filepath.Dir(outPath)
		if err := os.MkdirAll(parent, 0777); err != nil {
			return err
		}
		w, err := os.Create(outPath)
		if err != nil {
			return err
		}
		defer func() {
			if cerr := w.Close(); err == nil && cerr != nil {
				err = cerr
			}
		}()
		_, err = io.Copy(w, r)
		return err
	}

	for _, f := range zr.File {
		if err := extractFile(f); err != nil {
			return err
		}
	}

	return nil
}

func extractTar(r io.Reader, name, dir, stripPrefix string) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("extracting tar %s: %w", name, err)
		}
	}()

	tr := tar.NewReader(r)
	extractFile := func(hdr *tar.Header) (err error) {
		outPath, err := extractedPath(dir, stripPrefix, hdr.Name)
		if err != nil {
			return err
		}
		switch hdr.Typeflag {
		case tar.TypeDir:
			return os.MkdirAll(outPath, 0777)
		case tar.TypeReg:
			w, err := os.Create(outPath)
			if err != nil {
				return err
			}
			defer func() {
				if cerr := w.Close(); err == nil && cerr != nil {
					err = cerr
				}
			}()
			_, err = io.Copy(w, tr)
			return err
		default:
			return fmt.Errorf("unsupported file type %x: %q", hdr.Typeflag, hdr.Name)
		}
	}

	stripPrefix += "/"
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		if err := extractFile(hdr); err != nil {
			return err
		}
	}
	return nil
}

// extractedPath returns the file path that a file in an archive should be
// extracted to. It verifies that entryName starts with stripPrefix and does not
// point outside dir.
func extractedPath(dir, stripPrefix, entryName string) (string, error) {
	if !strings.HasPrefix(entryName, stripPrefix) {
		return "", fmt.Errorf("entry does not start with prefix %s: %q", stripPrefix, entryName)
	}
	entryName = entryName[len(stripPrefix):]
	if entryName == "" {
		return dir, nil
	}
	if path.IsAbs(entryName) {
		return "", fmt.Errorf("entry has an absolute path: %q", entryName)
	}
	if strings.HasPrefix(entryName, "../") {
		return "", fmt.Errorf("entry refers to something outside the archive: %q", entryName)
	}
	entryName = strings.TrimSuffix(entryName, "/")
	if path.Clean(entryName) != entryName {
		return "", fmt.Errorf("entry does not have a clean path: %q", entryName)
	}
	return filepath.Join(dir, entryName), nil
}

// copyDir recursively copies a directory tree.
func copyDir(toDir, fromDir string) error {
	if err := os.MkdirAll(toDir, 0777); err != nil {
		return err
	}
	return filepath.Walk(fromDir, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		rel, _ := filepath.Rel(fromDir, path)
		if rel == "." {
			return nil
		}
		outPath := filepath.Join(toDir, rel)
		if fi.IsDir() {
			return os.Mkdir(outPath, 0777)
		} else {
			return copyFile(outPath, path)
		}
	})
}

func copyFile(toFile, fromFile string) (err error) {
	r, err := os.Open(fromFile)
	if err != nil {
		return err
	}
	defer r.Close()
	w, err := os.Create(toFile)
	if err != nil {
		return err
	}
	defer func() {
		if cerr := w.Close(); err == nil && cerr != nil {
			err = cerr
		}
	}()
	_, err = io.Copy(w, r)
	return err
}

func sha256SumFile(name string) (string, error) {
	r, err := os.Open(name)
	if err != nil {
		return "", err
	}
	defer r.Close()
	h := sha256.New()
	if _, err := io.Copy(h, r); err != nil {
		return "", err
	}
	sum := h.Sum(nil)
	return hex.EncodeToString(sum), nil
}

// copyFileToMirror uploads a file to the GCS bucket backing mirror.bazel.build.
// gsutil must be installed, and the user must be authenticated with
// 'gcloud auth login' and be allowed to write files to the bucket.
func copyFileToMirror(ctx context.Context, path, fileName string) (err error) {
	dest := "gs://bazel-mirror/" + path
	defer func() {
		if err != nil {
			err = fmt.Errorf("copying file %s to %s: %w", fileName, dest, err)
		}
	}()

	// This function shells out to gsutil instead of using
	// cloud.google.com/go/storage because that package has a million
	// dependencies.
	return runForError(ctx, ".", "gsutil", "cp", "-n", fileName, dest)
}