blob: d4c182616f70955bf4836ebce47980d75a9b87c7 [file]
// Copyright 2026 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package project
import (
"context"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"go.fuchsia.dev/jiri"
"go.fuchsia.dev/jiri/cipd"
)
// FetchPackagePaths resolves one or more checkout-relative prebuilt paths
// (such as "prebuilt/third_party/jq/linux-x64/bin/jq" or
// "prebuilt/third_party/qemu/linux-x64") against the checkout's manifest and
// materializes the corresponding WorkspaceDestinations on demand.
func FetchPackagePaths(ctx context.Context, relPaths []string, fetchTimeout uint, opts PrebuiltProviderOptions) error {
checkout := jiri.CheckoutFromContext(ctx)
if checkout == nil || checkout.Root == "" {
return fmt.Errorf("checkout root is required to fetch prebuilt packages")
}
if len(relPaths) == 0 {
return fmt.Errorf("at least one prebuilt path is required")
}
if fetchTimeout == 0 {
fetchTimeout = DefaultPackageTimeout
}
cleanedPaths := make([]string, 0, len(relPaths))
seen := make(map[string]bool, len(relPaths))
for _, p := range relPaths {
cp := filepath.Clean(p)
if !seen[cp] {
seen[cp] = true
cleanedPaths = append(cleanedPaths, cp)
}
}
subsetPkgs, err := resolvePackagesForFetchPackage(ctx, checkout, cleanedPaths)
if err != nil {
return err
}
provider, err := NewPrebuiltProvider(checkout, opts, nil)
if err != nil {
return err
}
var isDestUpToDate func(WorkspaceDestination) bool
if provider.Name() == "rbe-casfs" {
isDestUpToDate = func(dest WorkspaceDestination) bool {
return isRBECasfsDestinationUpToDate(checkout.Root, checkout.PrebuiltTTLRefresh, dest)
}
} else {
cache := NewPackageCache(checkout.PackageCacheDir())
isDestUpToDate = func(dest WorkspaceDestination) bool {
return dest.IsUpToDate(checkout.Root, cache)
}
}
areAllDestsUpToDate := func(dests []WorkspaceDestination) bool {
if len(dests) == 0 {
return false
}
for _, dest := range dests {
if !isDestUpToDate(dest) {
return false
}
}
return true
}
hasInternal := false
for _, pkg := range subsetPkgs {
if pkg.Internal {
hasInternal = true
break
}
}
needsACLFilter := false
if hasInternal {
cachedAccess, ok := readCachedInternalAccess(ctx)
needsACLFilter = !ok || !cachedAccess
}
// Fast path: when an uncached CIPD ACL check would otherwise be required
// and all matched packages have pinned instance IDs, check whether their
// destinations are already materialized and up to date on disk so we can
// skip the network RPC in FilterACL.
if needsACLFilter && allPackagesPinned(subsetPkgs) {
if candidateDests, destErr := collectDestsWithRequiredPaths(ctx, subsetPkgs, cleanedPaths); destErr == nil && areAllDestsUpToDate(candidateDests) {
return finalizeUpToDateFetch(ctx, checkout.Root, provider.Name(), opts.RBEClient, cleanedPaths, false)
}
}
filteredInternal := false
if needsACLFilter {
preCount := len(subsetPkgs)
subsetPkgs, _, err = subsetPkgs.FilterACL(ctx)
if err != nil {
return err
}
filteredInternal = len(subsetPkgs) < preCount
}
dests, err := collectDestsWithRequiredPaths(ctx, subsetPkgs, cleanedPaths)
if err != nil {
return err
}
if len(dests) == 0 {
if filteredInternal {
return fmt.Errorf("no prebuilt package destinations resolved for %v (internal package(s) were skipped due to lack of CIPD access; try running 'cipd auth-login')", cleanedPaths)
}
return fmt.Errorf("no prebuilt package destinations resolved for %v", cleanedPaths)
}
if areAllDestsUpToDate(dests) {
return finalizeUpToDateFetch(ctx, checkout.Root, provider.Name(), opts.RBEClient, cleanedPaths, filteredInternal)
}
if err := provider.EnsureDestinations(ctx, checkout.Root, dests, fetchTimeout); err != nil {
return err
}
return verifyRequiredPaths(checkout.Root, cleanedPaths, true, filteredInternal)
}
func finalizeUpToDateFetch(ctx context.Context, wsRoot, providerName string, rbeClient RBEClient, cleanedPaths []string, filteredInternal bool) error {
if providerName == "rbe-casfs" {
if abfsCacheDir := resolveABFSCacheDir(ctx, rbeClient); abfsCacheDir != "" {
if err := ensureEmptyARIFile(abfsCacheDir); err != nil {
return err
}
}
}
return verifyRequiredPaths(wsRoot, cleanedPaths, false, filteredInternal)
}
func verifyRequiredPaths(targetRoot string, cleanedPaths []string, afterFetch, filteredInternal bool) error {
suffix := ""
if afterFetch {
suffix = " after fetching package(s)"
}
for _, reqPath := range cleanedPaths {
if _, err := os.Stat(filepath.Join(targetRoot, reqPath)); err != nil {
if filteredInternal {
return fmt.Errorf("prebuilt path %q does not exist%s (internal package(s) were skipped due to lack of CIPD access; try running 'cipd auth-login'): %w", reqPath, suffix, err)
}
return fmt.Errorf("prebuilt path %q does not exist%s: %w", reqPath, suffix, err)
}
}
return nil
}
func collectDestsWithRequiredPaths(ctx context.Context, pkgs Packages, cleanedPaths []string) ([]WorkspaceDestination, error) {
dests, err := collectWorkspaceDestinations(ctx, pkgs)
if err != nil {
return nil, err
}
sep := string(filepath.Separator)
for _, reqPath := range cleanedPaths {
bestIdx := -1
bestLen := -1
for i := range dests {
cleanDest := filepath.Clean(dests[i].RelPath)
if reqPath == cleanDest || strings.HasPrefix(reqPath, cleanDest+sep) {
if len(cleanDest) > bestLen {
bestLen = len(cleanDest)
bestIdx = i
}
}
}
if bestIdx >= 0 {
dests[bestIdx].RequiredRelPaths = append(dests[bestIdx].RequiredRelPaths, reqPath)
}
}
return dests, nil
}
func resolvePackagesForFetchPackage(ctx context.Context, checkout *jiri.Checkout, relPaths []string) (Packages, error) {
snapshotPath := checkout.UpdateHistoryLatestLink()
if _, err := os.Stat(snapshotPath); err != nil && checkout.WorktreeParentRoot != "" {
snapshotPath = filepath.Join(checkout.WorktreeParentRoot, jiri.RootMetaDir, "update_history", "latest")
}
manifestFile := checkout.JiriManifestFile()
_, manifestStatErr := os.Stat(manifestFile)
if _, err := os.Stat(snapshotPath); err == nil {
m, err := ManifestFromFile(ctx, snapshotPath)
if err != nil {
return nil, err
}
if m.Attributes == checkout.FetchingAttrs || manifestStatErr != nil {
pkgs := make(Packages, len(m.Packages))
for _, pkg := range m.Packages {
pkg.ComputedAttributes = newAttributes(pkg.Attributes)
pkg.Attributes = pkg.ComputedAttributes.String()
pkgs[pkg.Key()] = pkg
}
if err := FilterOptionalProjectsPackages(ctx, checkout.FetchingAttrs, nil, pkgs); err != nil {
return nil, err
}
matched, matchErr := matchPackagesForPaths(checkout.Root, pkgs, relPaths)
if matchErr == nil || manifestStatErr != nil {
return matched, matchErr
}
}
}
if manifestStatErr != nil {
return nil, fmt.Errorf("cannot resolve prebuilt packages: neither %s nor %s exists", checkout.UpdateHistoryLatestLink(), manifestFile)
}
localProjects, err := LocalProjects(ctx, FastScan)
if err != nil {
return nil, err
}
localManifestProjects, _ := getDefaultLocalManifestProjects(ctx)
manifestProjects, _, pkgs, err := LoadManifestFile(ctx, manifestFile, localProjects, localManifestProjects)
if err != nil {
return nil, err
}
if err := FilterOptionalProjectsPackages(ctx, checkout.FetchingAttrs, manifestProjects, pkgs); err != nil {
return nil, err
}
return matchPackagesForPaths(checkout.Root, pkgs, relPaths)
}
// matchPackagesForPaths matches each checkout-relative path in relPaths to one
// or more package destinations in pkgs. A package destination D matches a
// requested path P if:
// - D is the most-specific (longest-path) destination enclosing P
// (P == D or strings.HasPrefix(P, D+"/"))
// - D is inside P (strings.HasPrefix(D, P+"/"), e.g. P is an umbrella
// directory containing sub-package D)
//
// In addition, if any matched destination D has a nested child destination D'
// that is already materialized on disk under wsRoot, D' is included so that
// re-linking D in an earlier wave does not clobber D'.
func matchPackagesForPaths(wsRoot string, pkgs Packages, relPaths []string) (Packages, error) {
pkgsByDest := make(map[string][]Package)
for _, pkg := range pkgs {
plats, err := pkg.GetPlatforms()
if err != nil {
return nil, err
}
if len(plats) > 0 && !slices.Contains(plats, cipd.CurrentPlatform) {
continue
}
if strings.Contains(pkg.Name, "$") {
expandedNames, err := cipd.ResolvePlatforms(pkg.Name, []cipd.Platform{cipd.CurrentPlatform})
if err != nil {
return nil, err
}
if len(expandedNames) == 0 {
continue
}
}
resolvedPath, err := pkg.ResolvePath()
if err != nil {
return nil, err
}
cleanDest := filepath.Clean(resolvedPath)
pkgsByDest[cleanDest] = append(pkgsByDest[cleanDest], pkg)
}
sep := string(filepath.Separator)
matchedDests := make(map[string]bool)
for _, reqPath := range relPaths {
cleanReq := filepath.Clean(reqPath)
found := false
bestEnclosing := ""
var descendants []string
for destRel := range pkgsByDest {
if cleanReq == destRel || strings.HasPrefix(cleanReq, destRel+sep) {
if len(destRel) > len(bestEnclosing) {
bestEnclosing = destRel
}
found = true
} else if strings.HasPrefix(destRel, cleanReq+sep) {
descendants = append(descendants, destRel)
found = true
}
}
if bestEnclosing != "" {
matchedDests[bestEnclosing] = true
} else {
for _, desc := range descendants {
matchedDests[desc] = true
}
}
if !found {
return nil, fmt.Errorf("no prebuilt package found in manifest for path %q", reqPath)
}
}
// Include any nested child destination that already exists on disk under a
// matched parent destination so wave-ordered linking preserves the child.
var childDests []string
for parentDest := range matchedDests {
for candDest := range pkgsByDest {
if !matchedDests[candDest] && strings.HasPrefix(candDest, parentDest+sep) {
if NewCacheStamp(filepath.Join(wsRoot, candDest)).Exists() {
childDests = append(childDests, candDest)
}
}
}
}
for _, childDest := range childDests {
matchedDests[childDest] = true
}
subset := make(Packages)
for destRel := range matchedDests {
for _, pkg := range pkgsByDest[destRel] {
subset[pkg.Key()] = pkg
}
}
return subset, nil
}