src/testing/host-target-testing/artifacts/archive.go

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package artifacts

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"go.fuchsia.dev/fuchsia/src/testing/host-target-testing/util"
	"go.fuchsia.dev/fuchsia/tools/lib/logger"
	"go.fuchsia.dev/fuchsia/tools/lib/retry"
)

// Archive allows interacting with the build artifact repository.
type Archive struct {
	// lkg (typically found in $FUCHSIA_DIR/prebuilt/tools/lkg/lkg) is
	// used to look up the latest build id for a given builder.
	lkgPath string

	// artifacts (typically found in $FUCHSIA_DIR/prebuilt/tools/artifacts/artifacts)
	// is used to download artifacts for a given build id.
	artifactsPath string
}

// NewArchive creates a new Archive.
func NewArchive(lkgPath string, artifactsPath string) *Archive {
	return &Archive{
		lkgPath:       lkgPath,
		artifactsPath: artifactsPath,
	}
}

// GetBuilder returns a Builder with the given name and Archive.
func (a *Archive) GetBuilder(name string) *Builder {
	return &Builder{archive: a, name: name}
}

// GetBuildByID returns an ArtifactsBuild for fetching artifacts for the build
// with the given id.
func (a *Archive) GetBuildByID(
	ctx context.Context,
	id string,
	dir string,
	ffxPath string,
) (*ArtifactsBuild, error) {
	// Make sure the build exists.
	srcs, err := a.list(ctx, id)
	if err != nil {
		return nil, err
	}

	srcsMap := make(map[string]struct{})
	for _, src := range srcs {
		srcsMap[src] = struct{}{}
	}

	return &ArtifactsBuild{
		id:      id,
		archive: a,
		dir:     dir,
		srcs:    srcsMap,
		ffxPath: ffxPath,
	}, nil
}

// list artifacts that make up a build id `buildID`.
func (a *Archive) list(ctx context.Context, buildID string) ([]string, error) {
	args := []string{"ls", "-build", buildID}
	stdout, stderr, err := util.RunCommand(ctx, a.artifactsPath, args...)
	if err != nil {
		if len(stderr) != 0 {
			fmt.Printf("artifacts output: \n%s", stdout)
			return nil, fmt.Errorf("artifacts failed: %w: %s", err, string(stderr))
		}
		return nil, fmt.Errorf("artifacts failed: %w", err)
	}

	var lines []string
	sc := bufio.NewScanner(bytes.NewReader(stdout))
	for sc.Scan() {
		lines = append(lines, sc.Text())
	}

	return lines, nil
}

// Download artifacts from the build id `buildID` and write them to `dst`.
// If `srcs` contains only one source, it will copy the file or directory
// directly to `dst`. Otherwise, `dst` should be the directory under which to
// download the artifacts.
func (a *Archive) download(ctx context.Context, buildID string, fromRoot bool, dst string, srcs []string) error {
	tmpDir, err := os.MkdirTemp("", "download")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)

	// Filter out any duplicate sources.
	srcs = removeDuplicates(srcs)

	var src string
	var srcsFile string
	if len(srcs) > 1 {
		var filesToDownload []string
		var filesToSkip []string
		for _, src := range srcs {
			path := filepath.Join(dst, src)

			// We only need to download the file if it doesn't
			// exist locally, or the local path is a directory
			// (since we don't know what files exist in a directory
			// ahead of time).
			if st, err := os.Stat(path); err != nil || st.IsDir() {
				filesToDownload = append(filesToDownload, src)
			} else {
				filesToSkip = append(filesToSkip, src)
			}
		}

		logger.Infof(ctx, "skipping %d files to download", len(filesToSkip))
		if len(filesToDownload) == 0 {
			// Skip downloading if the files are already present in the build dir.
			logger.Infof(ctx, "no files to download")
			return nil
		}

		tmpFile, err := os.CreateTemp(tmpDir, "srcs-file")
		if err != nil {
			return err
		}
		tmpFile.Close()
		srcsFile = tmpFile.Name()
		if err := os.WriteFile(srcsFile, []byte(strings.Join(filesToDownload, "\n")), 0755); err != nil {
			return fmt.Errorf("failed to write srcs-file: %w", err)
		}
	} else {
		if st, err := os.Stat(dst); err == nil && !st.IsDir() {
			// Skip downloading if the file is already present in the build dir.
			return nil
		}
		src = srcs[0]
	}

	logger.Infof(ctx, "downloading %d artifacts to %s", len(srcs), dst)

	// The `artifacts` utility can occasionally run into transient issues. This implements a retry policy
	// that attempts to avoid such issues causing flakes.
	eb := retry.NewExponentialBackoff(100*time.Millisecond, 10*time.Second, 2)
	// ~12 seconds to hit backoff ceiling; 2.5 minutes of slack (given the above EB)
	retryCap := uint64(22)
	return retry.Retry(ctx, retry.WithMaxAttempts(eb, retryCap), func() error {
		// We don't want to leak files if we are interrupted during a download.
		// So we'll download all files to a temporary directory before moving
		// them to the specified destination, and we'll remove them in the case
		// of an error.
		tmpDst := filepath.Join(tmpDir, filepath.Base(dst))
		defer os.RemoveAll(tmpDst)
		args := []string{
			"cp",
			"-build", buildID,
			"-src", src,
			"-dst", tmpDst,
		}

		if fromRoot {
			args = append(args, "-root")
		}
		if srcsFile != "" {
			args = append(args, "-srcs-file", srcsFile)
		}

		stdout, stderr, err := util.RunCommand(ctx, a.artifactsPath, args...)
		if len(stdout) != 0 {
			logger.Infof(ctx, "artifacts stdout:\n%s", stdout)
		}
		if err != nil {
			if len(stderr) != 0 {
				logger.Infof(ctx, "artifacts stderr:\n%s", stderr)
				// Don't retry if the artifact we want to download does not exist.
				if bytes.Contains(stderr, []byte("nothing matched prefix")) {
					return retry.Fatal(os.ErrNotExist)
				}
				return fmt.Errorf("artifacts failed: %w: %s", err, string(stderr))
			}
			return fmt.Errorf("artifacts failed: %w", err)
		}
		return filepath.Walk(tmpDst, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if path == tmpDst && info.IsDir() {
				return nil
			}
			relPath, err := filepath.Rel(tmpDst, path)
			if err != nil {
				return err
			}
			dstPath := filepath.Join(dst, relPath)
			if fi, err := os.Stat(dstPath); err == nil {
				if fi.IsDir() {
					// If dstPath already exists and is a directory, then return nil so
					// we can walk the contents of the directory and move over the
					// individual files.
					return nil
				}
			}
			if err = os.MkdirAll(filepath.Dir(dstPath), os.ModePerm); err != nil {
				return err
			}
			if info.IsDir() {
				// Move/replace entire directory and skip walking contents.
				err = filepath.SkipDir
				os.RemoveAll(dstPath)
			}
			if moveErr := os.Rename(path, dstPath); moveErr != nil {
				return moveErr
			}
			return err
		})
	}, nil)
}

// removeDuplicates removes any duplicated items in the list.
func removeDuplicates(srcs []string) []string {
	var srcList []string
	seen := make(map[string]struct{})
	for _, src := range srcs {
		if _, ok := seen[src]; !ok {
			srcList = append(srcList, src)
			seen[src] = struct{}{}
		}
	}
	return srcList
}