| // Copyright 2026 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| package project | |
| import ( | |
| "context" | |
| "fmt" | |
| "os" | |
| "path/filepath" | |
| "slices" | |
| "strings" | |
| "go.fuchsia.dev/jiri" | |
| "go.fuchsia.dev/jiri/cipd" | |
| ) | |
| // FetchPackagePaths resolves one or more checkout-relative prebuilt paths | |
| // (such as "prebuilt/third_party/jq/linux-x64/bin/jq" or | |
| // "prebuilt/third_party/qemu/linux-x64") against the checkout's manifest and | |
| // materializes the corresponding WorkspaceDestinations on demand. | |
| func FetchPackagePaths(ctx context.Context, relPaths []string, fetchTimeout uint, opts PrebuiltProviderOptions) error { | |
| checkout := jiri.CheckoutFromContext(ctx) | |
| if checkout == nil || checkout.Root == "" { | |
| return fmt.Errorf("checkout root is required to fetch prebuilt packages") | |
| } | |
| if len(relPaths) == 0 { | |
| return fmt.Errorf("at least one prebuilt path is required") | |
| } | |
| if fetchTimeout == 0 { | |
| fetchTimeout = DefaultPackageTimeout | |
| } | |
| cleanedPaths := make([]string, 0, len(relPaths)) | |
| seen := make(map[string]bool, len(relPaths)) | |
| for _, p := range relPaths { | |
| cp := filepath.Clean(p) | |
| if !seen[cp] { | |
| seen[cp] = true | |
| cleanedPaths = append(cleanedPaths, cp) | |
| } | |
| } | |
| subsetPkgs, err := resolvePackagesForFetchPackage(ctx, checkout, cleanedPaths) | |
| if err != nil { | |
| return err | |
| } | |
| provider, err := NewPrebuiltProvider(checkout, opts, nil) | |
| if err != nil { | |
| return err | |
| } | |
| var isDestUpToDate func(WorkspaceDestination) bool | |
| if provider.Name() == "rbe-casfs" { | |
| isDestUpToDate = func(dest WorkspaceDestination) bool { | |
| return isRBECasfsDestinationUpToDate(checkout.Root, checkout.PrebuiltTTLRefresh, dest) | |
| } | |
| } else { | |
| cache := NewPackageCache(checkout.PackageCacheDir()) | |
| isDestUpToDate = func(dest WorkspaceDestination) bool { | |
| return dest.IsUpToDate(checkout.Root, cache) | |
| } | |
| } | |
| areAllDestsUpToDate := func(dests []WorkspaceDestination) bool { | |
| if len(dests) == 0 { | |
| return false | |
| } | |
| for _, dest := range dests { | |
| if !isDestUpToDate(dest) { | |
| return false | |
| } | |
| } | |
| return true | |
| } | |
| hasInternal := false | |
| for _, pkg := range subsetPkgs { | |
| if pkg.Internal { | |
| hasInternal = true | |
| break | |
| } | |
| } | |
| needsACLFilter := false | |
| if hasInternal { | |
| cachedAccess, ok := readCachedInternalAccess(ctx) | |
| needsACLFilter = !ok || !cachedAccess | |
| } | |
| // Fast path: when an uncached CIPD ACL check would otherwise be required | |
| // and all matched packages have pinned instance IDs, check whether their | |
| // destinations are already materialized and up to date on disk so we can | |
| // skip the network RPC in FilterACL. | |
| if needsACLFilter && allPackagesPinned(subsetPkgs) { | |
| if candidateDests, destErr := collectDestsWithRequiredPaths(ctx, subsetPkgs, cleanedPaths); destErr == nil && areAllDestsUpToDate(candidateDests) { | |
| return finalizeUpToDateFetch(ctx, checkout.Root, provider.Name(), opts.RBEClient, cleanedPaths, false) | |
| } | |
| } | |
| filteredInternal := false | |
| if needsACLFilter { | |
| preCount := len(subsetPkgs) | |
| subsetPkgs, _, err = subsetPkgs.FilterACL(ctx) | |
| if err != nil { | |
| return err | |
| } | |
| filteredInternal = len(subsetPkgs) < preCount | |
| } | |
| dests, err := collectDestsWithRequiredPaths(ctx, subsetPkgs, cleanedPaths) | |
| if err != nil { | |
| return err | |
| } | |
| if len(dests) == 0 { | |
| if filteredInternal { | |
| return fmt.Errorf("no prebuilt package destinations resolved for %v (internal package(s) were skipped due to lack of CIPD access; try running 'cipd auth-login')", cleanedPaths) | |
| } | |
| return fmt.Errorf("no prebuilt package destinations resolved for %v", cleanedPaths) | |
| } | |
| if areAllDestsUpToDate(dests) { | |
| return finalizeUpToDateFetch(ctx, checkout.Root, provider.Name(), opts.RBEClient, cleanedPaths, filteredInternal) | |
| } | |
| if err := provider.EnsureDestinations(ctx, checkout.Root, dests, fetchTimeout); err != nil { | |
| return err | |
| } | |
| return verifyRequiredPaths(checkout.Root, cleanedPaths, true, filteredInternal) | |
| } | |
| func finalizeUpToDateFetch(ctx context.Context, wsRoot, providerName string, rbeClient RBEClient, cleanedPaths []string, filteredInternal bool) error { | |
| if providerName == "rbe-casfs" { | |
| if abfsCacheDir := resolveABFSCacheDir(ctx, rbeClient); abfsCacheDir != "" { | |
| if err := ensureEmptyARIFile(abfsCacheDir); err != nil { | |
| return err | |
| } | |
| } | |
| } | |
| return verifyRequiredPaths(wsRoot, cleanedPaths, false, filteredInternal) | |
| } | |
| func verifyRequiredPaths(targetRoot string, cleanedPaths []string, afterFetch, filteredInternal bool) error { | |
| suffix := "" | |
| if afterFetch { | |
| suffix = " after fetching package(s)" | |
| } | |
| for _, reqPath := range cleanedPaths { | |
| if _, err := os.Stat(filepath.Join(targetRoot, reqPath)); err != nil { | |
| if filteredInternal { | |
| return fmt.Errorf("prebuilt path %q does not exist%s (internal package(s) were skipped due to lack of CIPD access; try running 'cipd auth-login'): %w", reqPath, suffix, err) | |
| } | |
| return fmt.Errorf("prebuilt path %q does not exist%s: %w", reqPath, suffix, err) | |
| } | |
| } | |
| return nil | |
| } | |
| func collectDestsWithRequiredPaths(ctx context.Context, pkgs Packages, cleanedPaths []string) ([]WorkspaceDestination, error) { | |
| dests, err := collectWorkspaceDestinations(ctx, pkgs) | |
| if err != nil { | |
| return nil, err | |
| } | |
| sep := string(filepath.Separator) | |
| for _, reqPath := range cleanedPaths { | |
| bestIdx := -1 | |
| bestLen := -1 | |
| for i := range dests { | |
| cleanDest := filepath.Clean(dests[i].RelPath) | |
| if reqPath == cleanDest || strings.HasPrefix(reqPath, cleanDest+sep) { | |
| if len(cleanDest) > bestLen { | |
| bestLen = len(cleanDest) | |
| bestIdx = i | |
| } | |
| } | |
| } | |
| if bestIdx >= 0 { | |
| dests[bestIdx].RequiredRelPaths = append(dests[bestIdx].RequiredRelPaths, reqPath) | |
| } | |
| } | |
| return dests, nil | |
| } | |
| func resolvePackagesForFetchPackage(ctx context.Context, checkout *jiri.Checkout, relPaths []string) (Packages, error) { | |
| snapshotPath := checkout.UpdateHistoryLatestLink() | |
| if _, err := os.Stat(snapshotPath); err != nil && checkout.WorktreeParentRoot != "" { | |
| snapshotPath = filepath.Join(checkout.WorktreeParentRoot, jiri.RootMetaDir, "update_history", "latest") | |
| } | |
| manifestFile := checkout.JiriManifestFile() | |
| _, manifestStatErr := os.Stat(manifestFile) | |
| if _, err := os.Stat(snapshotPath); err == nil { | |
| m, err := ManifestFromFile(ctx, snapshotPath) | |
| if err != nil { | |
| return nil, err | |
| } | |
| if m.Attributes == checkout.FetchingAttrs || manifestStatErr != nil { | |
| pkgs := make(Packages, len(m.Packages)) | |
| for _, pkg := range m.Packages { | |
| pkg.ComputedAttributes = newAttributes(pkg.Attributes) | |
| pkg.Attributes = pkg.ComputedAttributes.String() | |
| pkgs[pkg.Key()] = pkg | |
| } | |
| if err := FilterOptionalProjectsPackages(ctx, checkout.FetchingAttrs, nil, pkgs); err != nil { | |
| return nil, err | |
| } | |
| matched, matchErr := matchPackagesForPaths(checkout.Root, pkgs, relPaths) | |
| if matchErr == nil || manifestStatErr != nil { | |
| return matched, matchErr | |
| } | |
| } | |
| } | |
| if manifestStatErr != nil { | |
| return nil, fmt.Errorf("cannot resolve prebuilt packages: neither %s nor %s exists", checkout.UpdateHistoryLatestLink(), manifestFile) | |
| } | |
| localProjects, err := LocalProjects(ctx, FastScan) | |
| if err != nil { | |
| return nil, err | |
| } | |
| localManifestProjects, _ := getDefaultLocalManifestProjects(ctx) | |
| manifestProjects, _, pkgs, err := LoadManifestFile(ctx, manifestFile, localProjects, localManifestProjects) | |
| if err != nil { | |
| return nil, err | |
| } | |
| if err := FilterOptionalProjectsPackages(ctx, checkout.FetchingAttrs, manifestProjects, pkgs); err != nil { | |
| return nil, err | |
| } | |
| return matchPackagesForPaths(checkout.Root, pkgs, relPaths) | |
| } | |
| // matchPackagesForPaths matches each checkout-relative path in relPaths to one | |
| // or more package destinations in pkgs. A package destination D matches a | |
| // requested path P if: | |
| // - D is the most-specific (longest-path) destination enclosing P | |
| // (P == D or strings.HasPrefix(P, D+"/")) | |
| // - D is inside P (strings.HasPrefix(D, P+"/"), e.g. P is an umbrella | |
| // directory containing sub-package D) | |
| // | |
| // In addition, if any matched destination D has a nested child destination D' | |
| // that is already materialized on disk under wsRoot, D' is included so that | |
| // re-linking D in an earlier wave does not clobber D'. | |
| func matchPackagesForPaths(wsRoot string, pkgs Packages, relPaths []string) (Packages, error) { | |
| pkgsByDest := make(map[string][]Package) | |
| for _, pkg := range pkgs { | |
| plats, err := pkg.GetPlatforms() | |
| if err != nil { | |
| return nil, err | |
| } | |
| if len(plats) > 0 && !slices.Contains(plats, cipd.CurrentPlatform) { | |
| continue | |
| } | |
| if strings.Contains(pkg.Name, "$") { | |
| expandedNames, err := cipd.ResolvePlatforms(pkg.Name, []cipd.Platform{cipd.CurrentPlatform}) | |
| if err != nil { | |
| return nil, err | |
| } | |
| if len(expandedNames) == 0 { | |
| continue | |
| } | |
| } | |
| resolvedPath, err := pkg.ResolvePath() | |
| if err != nil { | |
| return nil, err | |
| } | |
| cleanDest := filepath.Clean(resolvedPath) | |
| pkgsByDest[cleanDest] = append(pkgsByDest[cleanDest], pkg) | |
| } | |
| sep := string(filepath.Separator) | |
| matchedDests := make(map[string]bool) | |
| for _, reqPath := range relPaths { | |
| cleanReq := filepath.Clean(reqPath) | |
| found := false | |
| bestEnclosing := "" | |
| var descendants []string | |
| for destRel := range pkgsByDest { | |
| if cleanReq == destRel || strings.HasPrefix(cleanReq, destRel+sep) { | |
| if len(destRel) > len(bestEnclosing) { | |
| bestEnclosing = destRel | |
| } | |
| found = true | |
| } else if strings.HasPrefix(destRel, cleanReq+sep) { | |
| descendants = append(descendants, destRel) | |
| found = true | |
| } | |
| } | |
| if bestEnclosing != "" { | |
| matchedDests[bestEnclosing] = true | |
| } else { | |
| for _, desc := range descendants { | |
| matchedDests[desc] = true | |
| } | |
| } | |
| if !found { | |
| return nil, fmt.Errorf("no prebuilt package found in manifest for path %q", reqPath) | |
| } | |
| } | |
| // Include any nested child destination that already exists on disk under a | |
| // matched parent destination so wave-ordered linking preserves the child. | |
| var childDests []string | |
| for parentDest := range matchedDests { | |
| for candDest := range pkgsByDest { | |
| if !matchedDests[candDest] && strings.HasPrefix(candDest, parentDest+sep) { | |
| if NewCacheStamp(filepath.Join(wsRoot, candDest)).Exists() { | |
| childDests = append(childDests, candDest) | |
| } | |
| } | |
| } | |
| } | |
| for _, childDest := range childDests { | |
| matchedDests[childDest] = true | |
| } | |
| subset := make(Packages) | |
| for destRel := range matchedDests { | |
| for _, pkg := range pkgsByDest[destRel] { | |
| subset[pkg.Key()] = pkg | |
| } | |
| } | |
| return subset, nil | |
| } |