[cipd] Add cipd bootstrap to jiri
This patch add cipd binary bootstrap feature to jiri with unit
tests. Jiri will only perform cipd bootstrap if there is a
<package> tag found in manifest files. Cipd will be saved to the
same directory that contains Jiri executable file.
BLD-201
Change-Id: Ic82aa2784b47c14a51b939a3851a1c7a232d34e9
diff --git a/cipd/cipd.go b/cipd/cipd.go
index 3fa6743..f34fa95 100644
--- a/cipd/cipd.go
+++ b/cipd/cipd.go
@@ -5,52 +5,264 @@
package cipd
import (
+ "bufio"
+ "bytes"
"context"
+ "crypto/sha256"
+ "errors"
"fmt"
+ "io/ioutil"
+ "net/http"
"os"
"os/exec"
"path"
+ "runtime"
+ "strings"
+ "sync"
"time"
"fuchsia.googlesource.com/jiri"
"fuchsia.googlesource.com/jiri/osutil"
+ "fuchsia.googlesource.com/jiri/version"
)
-func getCipdPath() (string, error) {
+const (
+ cipdBackend = "https://chrome-infra-packages.appspot.com"
+ cipdVersion = "git_revision:00e2d8b49a4e7505d1c71f19d15c9e7c5b9245a5"
+ cipdVersionDigest = `
+# This file was generated by
+#
+# cipd selfupdate-roll -version-file .cipd_version \
+# -version git_revision:00e2d8b49a4e7505d1c71f19d15c9e7c5b9245a5
+#
+# Do not modify manually. All changes will be overwritten.
+# Use 'cipd selfupdate-roll ...' to modify.
+linux-386 sha256 15c8c9c96ae1dfc1fff8ce5469eecb60dded782f6b8b7ba37d9edbeca443017e
+linux-amd64 sha256 df37ffc2588e345a31ca790d773b6136fedbd2efbf9a34cb735dd34b6891c16c
+linux-arm64 sha256 650f2a045f8587062a16299a650aa24ba5c5c0652585a2d9bd56594369d5f99e
+linux-armv6l sha256 61b657c860ddc39d3286ced073c843852b1dafc0222af0bdc22ad988b289d733
+linux-mips64 sha256 ab58e3886e8e0805c6c19e58b2f5a96e9fa3921c9dd1fc3ee956bf07b35993df
+linux-mips64le sha256 acac10fdb9fd459ef318e6b10b971471e2b6814e4a7fe14f119a15bd89b4b163
+linux-mipsle sha256 a2d31d02c838cbd849b068fa371ec278a836d2f184a90fb9d1a63271cf152d82
+linux-ppc64 sha256 a6f5a84de095bbe986dda8a16d17525280fc448d63347fd240c71d1cead78036
+linux-ppc64le sha256 77afc938916f84eec342acaabb8e32a845eac72e23b9db018cdf09fc31df585b
+linux-s390x sha256 86dd1092d1dc228d59d3862bb2d48ebe2e4f16a3c054fb04391f07d9d2b15d33
+mac-amd64 sha256 4d015791ed6f03f305cf6a5a673a447e5c47ff5fdb701f43f99fba9ca73e61f8
+windows-386 sha256 b8102c9a1b93915c128e7577c89fd77991ab83d52c356913e56ea505ab338735
+windows-amd64 sha256 a117e3984c111c68698faf91815c4b7d374404fa82dff318aadb9f2f0582ca8d
+`
+)
+
+var (
+ cipdPlatform string
+ cipdOS string
+ cipdArch string
+ cipdBinary string
+ selfUpdateOnce sync.Once
+)
+
+func init() {
+ cipdOS = runtime.GOOS
+ cipdArch = runtime.GOARCH
+ if cipdOS == "darwin" {
+ cipdOS = "mac"
+ }
+ if cipdArch == "arm" {
+ cipdArch = "armv6l"
+ }
+ cipdPlatform = cipdOS + "-" + cipdArch
+
jiriPath, err := osutil.Executable()
if err != nil {
- return "", err
+ // Could not locate jiri, leave cipdBinary empty
+ // It will be checked by bootstrap and reported
+ return
}
// Assume cipd binary is located in the same directory of jiri
jiriBinaryRoot := path.Dir(jiriPath)
- cipdBinary := path.Join(jiriBinaryRoot, "cipd")
- fileInfo, err := os.Stat(cipdBinary)
- if err != nil {
- if os.IsNotExist(err) {
- return "", fmt.Errorf("cipd binary was not found at %q", cipdBinary)
- }
- return "", err
- }
- // Check if cipd binary has execution permission
- if fileInfo.Mode()&0111 == 0 {
- return "", fmt.Errorf("cipd binary at %q is not executable", cipdBinary)
- }
- return cipdBinary, nil
+ cipdBinary = path.Join(jiriBinaryRoot, "cipd")
+}
+func fetchBinary(binaryPath, platform, version, digest string) error {
+ cipdURL := fmt.Sprintf("%s/client?platform=%s&version=%s", cipdBackend, platform, version)
+ data, err := fetchFile(cipdURL)
+ if err != nil {
+ return err
+ }
+ if verified, err := verifyDigest(data, digest); err != nil || !verified {
+ if err != nil {
+ return err
+ }
+ return errors.New("cipd failed integrity test")
+ }
+ // cipd binary verified. Save to disk
+ return writeFile(binaryPath, data)
+}
+
+// Bootstrap returns the path of a valid cipd binary. It will fetch cipd from
+// remote if a valid cipd binary is not found. It will update cipd if there
+// is a new version.
+func Bootstrap() (string, error) {
+ bootstrap := func() error {
+ // Fetch cipd digest
+ cipdDigest, _, err := fetchDigest(cipdPlatform)
+ if err != nil {
+ return err
+ }
+ if cipdBinary == "" {
+ return errors.New("cipd binary path was not set")
+ }
+ if err != nil {
+ return err
+ }
+ return fetchBinary(cipdBinary, cipdPlatform, cipdVersion, cipdDigest)
+ }
+
+ getCipd := func() (string, error) {
+ if cipdBinary == "" {
+ return "", errors.New("cipd binary path was not set")
+ }
+ fileInfo, err := os.Stat(cipdBinary)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return "", fmt.Errorf("cipd binary was not found at %q", cipdBinary)
+ }
+ return "", err
+ }
+ // Check if cipd binary has execution permission
+ if fileInfo.Mode()&0111 == 0 {
+ return "", fmt.Errorf("cipd binary at %q is not executable", cipdBinary)
+ }
+ return cipdBinary, nil
+ }
+
+ cipdPath, err := getCipd()
+ if err != nil {
+ // Could not find cipd binary or cipd is invalid
+ // Bootstrap it from scratch
+ if err := bootstrap(); err != nil {
+ return "", err
+ }
+ return cipdPath, nil
+ }
+ // cipd is found, do self update
+ var e error
+ selfUpdateOnce.Do(func() {
+ e = selfUpdate(cipdPath, cipdVersion)
+ })
+ if e != nil {
+ // Self update is unsuccessful, redo bootstrap
+ if err := bootstrap(); err != nil {
+ return "", err
+ }
+ }
+ return cipdPath, nil
+}
+
+func fetchDigest(platform string) (digest, method string, err error) {
+ var digestBuf bytes.Buffer
+ digestBuf.Write([]byte(cipdVersionDigest))
+ digestScanner := bufio.NewScanner(&digestBuf)
+ for digestScanner.Scan() {
+ curLine := digestScanner.Text()
+ if len(curLine) == 0 || curLine[0] == '#' {
+ // Skip comment or empty line
+ continue
+ }
+ fields := strings.Fields(curLine)
+ if len(fields) != 3 {
+ return "", "", errors.New("unsupported cipd digest file format")
+ }
+ if fields[0] == platform {
+ digest = fields[2]
+ method = fields[1]
+ err = nil
+ return
+ }
+ }
+ return "", "", errors.New("no matching platform found in cipd digest file")
+}
+
+func selfUpdate(cipdPath, cipdVersion string) error {
+ args := []string{"selfupdate", "-version", cipdVersion, "-service-url", cipdBackend}
+ command := exec.Command(cipdPath, args...)
+ return command.Run()
+}
+
+func writeFile(filePath string, data []byte) error {
+ tempFile, err := ioutil.TempFile(path.Dir(filePath), "cipd.*")
+ if err != nil {
+ return err
+ }
+ defer tempFile.Close()
+ defer os.Remove(tempFile.Name())
+ if _, err := tempFile.Write(data); err != nil {
+ // Write errors
+ return errors.New("I/O error while downloading cipd binary")
+ }
+ // Set mode to rwxr-xr-x
+ if err := tempFile.Chmod(0755); err != nil {
+ // Chmod errors
+ return errors.New("I/O error while adding executable permission to cipd binary")
+ }
+ tempFile.Close()
+ if err := os.Rename(tempFile.Name(), filePath); err != nil {
+ return err
+ }
+ return nil
+}
+
+func verifyDigest(data []byte, cipdDigest string) (bool, error) {
+ hash := sha256.Sum256(data)
+ hashString := fmt.Sprintf("%x", hash)
+ if hashString == strings.ToLower(cipdDigest) {
+ return true, nil
+ }
+ return false, nil
+}
+
+func getUserAgent() string {
+ ua := "jiri/" + version.GitCommit
+ if version.GitCommit == "" {
+ ua += "debug"
+ }
+ return ua
+}
+
+func fetchFile(url string) ([]byte, error) {
+ client := &http.Client{}
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", getUserAgent())
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ return ioutil.ReadAll(resp.Body)
}
// Ensure runs cipd binary's ensure funcationality over file. Fetched packages will be
-// saved to projectRoot directory. Parameter timeout is in minitues
+// saved to projectRoot directory. Parameter timeout is in minutes.
func Ensure(jirix *jiri.X, file, projectRoot string, timeout uint) error {
- cipdBinary, err := getCipdPath()
+ cipdPath, err := Bootstrap()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute)
defer cancel()
args := []string{"ensure", "-ensure-file", file, "-root", projectRoot, "-log-level", "warning"}
- jirix.Logger.Debugf("Invoke cipd with %v", args)
- command := exec.CommandContext(ctx, cipdBinary, args...)
+ // Walkaround so tests do not have to create a new fake jirix, which would
+ // result in an import cycle
+ if jirix != nil {
+ jirix.Logger.Debugf("Invoke cipd with %v", args)
+ }
+ // Construct arguments and invoke cipd for ensure file
+ command := exec.CommandContext(ctx, cipdPath, args...)
+ // Add User-Agent info for cipd
+ command.Env = append(os.Environ(), "CIPD_HTTP_USER_AGENT_PREFIX="+getUserAgent())
command.Stdin = os.Stdin
command.Stdout = os.Stdout
command.Stderr = os.Stderr
diff --git a/cipd/cipd_test.go b/cipd/cipd_test.go
new file mode 100644
index 0000000..fdff7ed
--- /dev/null
+++ b/cipd/cipd_test.go
@@ -0,0 +1,166 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cipd
+
+import (
+ "encoding/hex"
+ "io/ioutil"
+ "os"
+ "path"
+ "strings"
+ "testing"
+)
+
+const (
+ // Some random valid cipd version tags from infra/tools/cipd
+ cipdVersionForTestA = "git_revision:00e2d8b49a4e7505d1c71f19d15c9e7c5b9245a5"
+ cipdVersionForTestB = "git_revision:8fac632847b1ce0de3b57d16d0f2193625f4a4f0"
+)
+
+var (
+ // Digests generated by cipd selfupdate-roll ...
+ digestMapA = map[string]string{
+ "linux-amd64": "df37ffc2588e345a31ca790d773b6136fedbd2efbf9a34cb735dd34b6891c16c",
+ "linux-arm64": "650f2a045f8587062a16299a650aa24ba5c5c0652585a2d9bd56594369d5f99e",
+ "linux-armv6l": "61b657c860ddc39d3286ced073c843852b1dafc0222af0bdc22ad988b289d733",
+ "mac-amd64": "4d015791ed6f03f305cf6a5a673a447e5c47ff5fdb701f43f99fba9ca73e61f8",
+ }
+ digestMapB = map[string]string{
+ "linux-amd64": "bdc971fd2895c3771e0709d2a3ec5fcace69c59a3a9f9dc33ab76fbc2f777d40",
+ "linux-arm64": "e1d6aadc9bfc155e9088aa3de39b9d3311c7359f398f372b5ad1c308e25edfeb",
+ "linux-armv6l": "3ad97b47ecc1b358c8ebd1b0307087d354433d88f24bf8ece096fb05452837f9",
+ "mac-amd64": "167edadf7c7c019a40b9f7869a4c05b2d9834427dad68e295442ef9ebce88dba",
+ }
+)
+
+// TestFetchBinary tests fetchiBinary method by fetching a set of
+// cipd binaries. This test requires network access
+func TestFetchBinary(t *testing.T) {
+ tmpDir, err := ioutil.TempDir("", "jiri-test")
+ if err != nil {
+ t.Error("failed to create temp dir for testing")
+ }
+ defer os.RemoveAll(tmpDir)
+
+ tests := []struct {
+ version string
+ digest map[string]string
+ }{
+ {cipdVersionForTestA, digestMapA},
+ {cipdVersionForTestB, digestMapB},
+ }
+
+ for i, test := range tests {
+ for platform, digest := range test.digest {
+ cipdPath := path.Join(tmpDir, "cipd"+platform+test.version)
+ if err := fetchBinary(cipdPath, platform, test.version, digest); err != nil {
+ t.Errorf("test %d failed while retrieving cipd binary for platform %q on version %q with digest %q: %v", i, platform, test.version, digest, err)
+ }
+ }
+ }
+}
+
+func TestCipdVersion(t *testing.T) {
+ // Assume cipd version is always a git commit hash for now
+ versionStr := string(cipdVersion)
+ if len(versionStr) != len("git_revision:00e2d8b49a4e7505d1c71f19d15c9e7c5b9245a5") ||
+ !strings.HasPrefix(versionStr, "git_revision:") {
+ t.Errorf("unsupported cipd version tag: %q", versionStr)
+ }
+ versionHash := versionStr[len("git_revision:"):]
+ if _, err := hex.DecodeString(versionHash); err != nil {
+ t.Errorf("unsupported cipd version tag: %q", versionStr)
+ }
+}
+
+func TestFetchDigest(t *testing.T) {
+ tests := []string{
+ "linux-amd64",
+ "linux-arm64",
+ "linux-armv6l",
+ "mac-amd64",
+ }
+
+ for _, platform := range tests {
+ digest, _, err := fetchDigest(platform)
+ if err != nil {
+ t.Errorf("failed to retrieve cipd digest for platform %q due to error: %v", platform, err)
+ }
+ if _, err := hex.DecodeString(digest); err != nil {
+ t.Errorf("digest %q is not a valid hex string for platform %q", digest, platform)
+ }
+ }
+}
+
+func TestSelfUpdate(t *testing.T) {
+ tmpDir, err := ioutil.TempDir("", "jiri-test")
+ if err != nil {
+ t.Error("failed to create temp dir for testing")
+ }
+ defer os.RemoveAll(tmpDir)
+ // Bootstrap cipd to version A
+ cipdPath := path.Join(tmpDir, "cipd")
+ if err := fetchBinary(cipdPath, cipdPlatform, cipdVersionForTestA, digestMapA[cipdPlatform]); err != nil {
+ t.Errorf("failed to bootstrap cipd with version %q: %v", cipdVersionForTestA, err)
+ }
+ // Perform cipd self update to version B
+ if err := selfUpdate(cipdPath, cipdVersionForTestB); err != nil {
+ t.Errorf("failed to perform cipd self update: %v", err)
+ }
+ // Verify self updated cipd
+ cipdData, err := ioutil.ReadFile(cipdPath)
+ if err != nil {
+ t.Errorf("failed to read self-updated cipd binary: %v", err)
+ }
+ verified, err := verifyDigest(cipdData, digestMapB[cipdPlatform])
+ if err != nil {
+ t.Errorf("digest failed verification for platform %q on version %q", cipdPlatform, cipdVersionForTestB)
+ }
+ if !verified {
+ t.Errorf("self-updated cipd failed integrity test")
+ }
+}
+
+func TestEnsure(t *testing.T) {
+ cipdPath, err := Bootstrap()
+ if err != nil {
+ t.Errorf("bootstrap failed due to error: %v", err)
+ }
+ defer os.Remove(cipdPath)
+ // Write test ensure file
+ testEnsureFile, err := ioutil.TempFile("", "test_jiri*.ensure")
+ if err != nil {
+ t.Errorf("failed to create test ensure file: %v", err)
+ }
+ defer testEnsureFile.Close()
+ defer os.Remove(testEnsureFile.Name())
+ _, err = testEnsureFile.Write([]byte(`
+$ParanoidMode CheckPresence
+
+# GN
+gn/gn/${platform} git_revision:bdb0fd02324b120cacde634a9235405061c8ea06
+`))
+ if err != nil {
+ t.Errorf("failed to write test ensure file: %v", err)
+ }
+ testEnsureFile.Sync()
+ tmpDir, err := ioutil.TempDir("", "jiri-test")
+ if err != nil {
+ t.Error("failed to creat temp dir for testing")
+ }
+ defer os.RemoveAll(tmpDir)
+ // Invoke Ensure on test ensure file
+ if err := Ensure(nil, testEnsureFile.Name(), tmpDir, 30); err != nil {
+ t.Errorf("ensure failed due to error: %v", err)
+ }
+ // Check the existence downloaded package
+ gnPath := path.Join(tmpDir, "gn")
+ if _, err := os.Stat(gnPath); err != nil {
+ if os.IsNotExist(err) {
+ t.Errorf("fetched cipd package is not found at %q", gnPath)
+ }
+ t.Errorf("failed to execute os.Stat() on fetched cipd package due to error: %v", err)
+ }
+}
