cipd/cipd.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipd

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"sync"
	"time"

	"fuchsia.googlesource.com/jiri"
	"fuchsia.googlesource.com/jiri/osutil"
	"fuchsia.googlesource.com/jiri/version"
)

const (
	cipdBackend       = "https://chrome-infra-packages.appspot.com"
	cipdVersion       = "git_revision:00e2d8b49a4e7505d1c71f19d15c9e7c5b9245a5"
	cipdVersionDigest = `
# This file was generated by
#
#  cipd selfupdate-roll -version-file .cipd_version \
#      -version git_revision:00e2d8b49a4e7505d1c71f19d15c9e7c5b9245a5
#
# Do not modify manually. All changes will be overwritten.
# Use 'cipd selfupdate-roll ...' to modify.
linux-386       sha256  15c8c9c96ae1dfc1fff8ce5469eecb60dded782f6b8b7ba37d9edbeca443017e
linux-amd64     sha256  df37ffc2588e345a31ca790d773b6136fedbd2efbf9a34cb735dd34b6891c16c
linux-arm64     sha256  650f2a045f8587062a16299a650aa24ba5c5c0652585a2d9bd56594369d5f99e
linux-armv6l    sha256  61b657c860ddc39d3286ced073c843852b1dafc0222af0bdc22ad988b289d733
linux-mips64    sha256  ab58e3886e8e0805c6c19e58b2f5a96e9fa3921c9dd1fc3ee956bf07b35993df
linux-mips64le  sha256  acac10fdb9fd459ef318e6b10b971471e2b6814e4a7fe14f119a15bd89b4b163
linux-mipsle    sha256  a2d31d02c838cbd849b068fa371ec278a836d2f184a90fb9d1a63271cf152d82
linux-ppc64     sha256  a6f5a84de095bbe986dda8a16d17525280fc448d63347fd240c71d1cead78036
linux-ppc64le   sha256  77afc938916f84eec342acaabb8e32a845eac72e23b9db018cdf09fc31df585b
linux-s390x     sha256  86dd1092d1dc228d59d3862bb2d48ebe2e4f16a3c054fb04391f07d9d2b15d33
mac-amd64       sha256  4d015791ed6f03f305cf6a5a673a447e5c47ff5fdb701f43f99fba9ca73e61f8
windows-386     sha256  b8102c9a1b93915c128e7577c89fd77991ab83d52c356913e56ea505ab338735
windows-amd64   sha256  a117e3984c111c68698faf91815c4b7d374404fa82dff318aadb9f2f0582ca8d
`
)

var (
	cipdPlatform   string
	cipdOS         string
	cipdArch       string
	cipdBinary     string
	selfUpdateOnce sync.Once
)

func init() {
	cipdOS = runtime.GOOS
	cipdArch = runtime.GOARCH
	if cipdOS == "darwin" {
		cipdOS = "mac"
	}
	if cipdArch == "arm" {
		cipdArch = "armv6l"
	}
	cipdPlatform = cipdOS + "-" + cipdArch

	jiriPath, err := osutil.Executable()
	if err != nil {
		// Could not locate jiri, leave cipdBinary empty
		// It will be checked by bootstrap and reported
		return
	}
	// Assume cipd binary is located in the same directory of jiri
	jiriBinaryRoot := path.Dir(jiriPath)
	cipdBinary = path.Join(jiriBinaryRoot, "cipd")
}

func fetchBinary(binaryPath, platform, version, digest string) error {
	cipdURL := fmt.Sprintf("%s/client?platform=%s&version=%s", cipdBackend, platform, version)
	data, err := fetchFile(cipdURL)
	if err != nil {
		return err
	}
	if verified, err := verifyDigest(data, digest); err != nil || !verified {
		if err != nil {
			return err
		}
		return errors.New("cipd failed integrity test")
	}
	// cipd binary verified. Save to disk
	return writeFile(binaryPath, data)
}

// Bootstrap returns the path of a valid cipd binary. It will fetch cipd from
// remote if a valid cipd binary is not found. It will update cipd if there
// is a new version.
func Bootstrap() (string, error) {
	bootstrap := func() error {
		// Fetch cipd digest
		cipdDigest, _, err := fetchDigest(cipdPlatform)
		if err != nil {
			return err
		}
		if cipdBinary == "" {
			return errors.New("cipd binary path was not set")
		}
		if err != nil {
			return err
		}
		return fetchBinary(cipdBinary, cipdPlatform, cipdVersion, cipdDigest)
	}

	getCipd := func() (string, error) {
		if cipdBinary == "" {
			return "", errors.New("cipd binary path was not set")
		}
		fileInfo, err := os.Stat(cipdBinary)
		if err != nil {
			if os.IsNotExist(err) {
				return "", fmt.Errorf("cipd binary was not found at %q", cipdBinary)
			}
			return "", err
		}
		// Check if cipd binary has execution permission
		if fileInfo.Mode()&0111 == 0 {
			return "", fmt.Errorf("cipd binary at %q is not executable", cipdBinary)
		}
		return cipdBinary, nil
	}

	cipdPath, err := getCipd()
	if err != nil {
		// Could not find cipd binary or cipd is invalid
		// Bootstrap it from scratch
		if err := bootstrap(); err != nil {
			return "", err
		}
		return cipdPath, nil
	}
	// cipd is found, do self update
	var e error
	selfUpdateOnce.Do(func() {
		e = selfUpdate(cipdPath, cipdVersion)
	})
	if e != nil {
		// Self update is unsuccessful, redo bootstrap
		if err := bootstrap(); err != nil {
			return "", err
		}
	}
	return cipdPath, nil
}

func fetchDigest(platform string) (digest, method string, err error) {
	var digestBuf bytes.Buffer
	digestBuf.Write([]byte(cipdVersionDigest))
	digestScanner := bufio.NewScanner(&digestBuf)
	for digestScanner.Scan() {
		curLine := digestScanner.Text()
		if len(curLine) == 0 || curLine[0] == '#' {
			// Skip comment or empty line
			continue
		}
		fields := strings.Fields(curLine)
		if len(fields) != 3 {
			return "", "", errors.New("unsupported cipd digest file format")
		}
		if fields[0] == platform {
			digest = fields[2]
			method = fields[1]
			err = nil
			return
		}
	}
	return "", "", errors.New("no matching platform found in cipd digest file")
}

func selfUpdate(cipdPath, cipdVersion string) error {
	args := []string{"selfupdate", "-version", cipdVersion, "-service-url", cipdBackend}
	command := exec.Command(cipdPath, args...)
	return command.Run()
}

func writeFile(filePath string, data []byte) error {
	tempFile, err := ioutil.TempFile(path.Dir(filePath), "cipd.*")
	if err != nil {
		return err
	}
	defer tempFile.Close()
	defer os.Remove(tempFile.Name())
	if _, err := tempFile.Write(data); err != nil {
		// Write errors
		return errors.New("I/O error while downloading cipd binary")
	}
	// Set mode to rwxr-xr-x
	if err := tempFile.Chmod(0755); err != nil {
		// Chmod errors
		return errors.New("I/O error while adding executable permission to cipd binary")
	}
	tempFile.Close()
	if err := os.Rename(tempFile.Name(), filePath); err != nil {
		return err
	}
	return nil
}

func verifyDigest(data []byte, cipdDigest string) (bool, error) {
	hash := sha256.Sum256(data)
	hashString := fmt.Sprintf("%x", hash)
	if hashString == strings.ToLower(cipdDigest) {
		return true, nil
	}
	return false, nil
}

func getUserAgent() string {
	ua := "jiri/" + version.GitCommit
	if version.GitCommit == "" {
		ua += "debug"
	}
	return ua
}

func fetchFile(url string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", getUserAgent())
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

// Ensure runs cipd binary's ensure funcationality over file. Fetched packages will be
// saved to projectRoot directory. Parameter timeout is in minutes.
func Ensure(jirix *jiri.X, file, projectRoot string, timeout uint) error {
	cipdPath, err := Bootstrap()
	if err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute)
	defer cancel()
	args := []string{"ensure", "-ensure-file", file, "-root", projectRoot, "-log-level", "warning"}
	// Walkaround so tests do not have to create a new fake jirix, which would
	// result in an import cycle
	if jirix != nil {
		jirix.Logger.Debugf("Invoke cipd with %v", args)
	}
	// Construct arguments and invoke cipd for ensure file
	command := exec.CommandContext(ctx, cipdPath, args...)
	// Add User-Agent info for cipd
	command.Env = append(os.Environ(), "CIPD_HTTP_USER_AGENT_PREFIX="+getUserAgent())
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr

	return command.Run()
}