project/package_cache.go - jiri - Git at Google

 // Copyright 2026 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package project

 import (
 	"context"
 	"encoding/json"
 	"errors"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"

 	"go.fuchsia.dev/jiri"
 	"go.fuchsia.dev/jiri/cipd"
 	"go.fuchsia.dev/jiri/log"
 )

 // PackageCacheFormatVersionFileName is the filename in the package cache root
 // that records the schema version of the central package cache.
 const PackageCacheFormatVersionFileName = ".jiri_cache_format_version"

 // CurrentPackageCacheFormatVersion is the current format version of the package cache.
 // Incrementing this version triggers a one-time sanitization pass across cached instances.
 const CurrentPackageCacheFormatVersion = 2

 // PackageCache manages the central package cache directory (`.jiri_root/packages`).
 //
 // In Jiri, the central package cache serves as the single source of truth for
 // downloaded CIPD package payloads across all active workspaces and worktrees.
 // It stores immutable package instances in content-addressed directories under
 // `instances/<instance_id>`, manages the global non-blocking synchronization lock
 // (`.cache.lock`), and coordinates CIPD ensure synchronization and pruning.
 type PackageCache struct {
 	root string
 }

 // NewPackageCache constructs a new PackageCache manager for the specified cache root.
 func NewPackageCache(root string) *PackageCache {
 	return &PackageCache{root: root}
 }

 // Root returns the root directory of the package cache.
 func (c *PackageCache) Root() string {
 	return c.root
 }

 // InstanceDir returns the absolute directory path where a package instance is installed in cache.
 func (c *PackageCache) InstanceDir(inst PackageCacheInstance) string {
 	return filepath.Join(c.root, filepath.FromSlash(inst.Subdir()))
 }

 // InstanceExists checks if the instance directory exists in the cache.
 func (c *PackageCache) InstanceExists(inst PackageCacheInstance) bool {
 	fi, err := os.Stat(c.InstanceDir(inst))
 	return err == nil && fi.IsDir()
 }

 // HasAllInstances reports whether all specified instances exist in the cache.
 // It returns false if instances is empty or any instance is missing.
 func (c *PackageCache) HasAllInstances(instances []PackageCacheInstance) bool {
 	if len(instances) == 0 {
 		return false
 	}
 	for _, inst := range instances {
 		if !c.InstanceExists(inst) {
 			return false
 		}
 	}
 	return true
 }

 // Lock acquires an exclusive, non-blocking lock on `.cache.lock` within the cache root.
 // This guarantees fail-fast concurrency control across concurrent Jiri processes
 // or active worktrees running package synchronization.
 // Returns an unlock function to release the lock, or an error if acquisition fails.
 func (c *PackageCache) Lock() (unlock func(), err error) {
 	return AcquireCacheLock(c.root)
 }

 // Sync runs CIPD cache synchronization using the provided CIPDEnsureFile.
 // It writes the ensure specification to `instance_master.ensure` and invokes CIPD ensure,
 // which validates, repairs, downloads, and natively prunes unreferenced instances
 // under `instances/<instance_id>`. The caller must hold cache.Lock() before calling Sync.
 func (c *PackageCache) Sync(ctx context.Context, ensureFile *CIPDEnsureFile, timeout uint) error {
 	checkout := jiri.CheckoutFromContext(ctx)
 	if err := os.MkdirAll(c.root, 0755); err != nil {
 		return err
 	}

 	// Perform a one-time migration check to purge any cached instances that were
 	// previously contaminated by multi-package standalone adoption.
 	if !c.isFormatVersionCurrent() {
 		c.sanitizeContaminatedInstances(ctx)
 		_ = c.writeFormatVersion()
 	}

 	// In non-paranoid mode, CIPD assumes packages are intact if their metadata
 	// slot in .cipd/pkgs exists and does not check the filesystem. If an instance
 	// directory is missing or corrupted, clean up any corrupted file and remove the
 	// corresponding .cipd/pkgs metadata slot so CIPD notices and redeploys it.
 	for _, inst := range ensureFile.Instances() {
 		instDir := c.InstanceDir(inst)
 		fi, err := os.Lstat(instDir)
 		if err != nil || !fi.IsDir() {
 			if err == nil {
 				_ = os.RemoveAll(instDir)
 			}
 			c.removeCIPDPkgForInstance(inst.InstanceID)
 		}
 	}

 	masterEnsurePath := filepath.Join(c.root, "instance_master.ensure")
 	if err := ensureFile.Write(masterEnsurePath, checkout.CipdParanoidMode); err != nil {
 		return err
 	}
 	defer os.Remove(masterEnsurePath)

 	jiri.TimerPush(ctx, "cipd package cache sync")
 	defer jiri.TimerPop(ctx)

 	return cipd.Ensure(ctx, masterEnsurePath, c.root, timeout)
 }

 func (c *PackageCache) isFormatVersionCurrent() bool {
 	versionPath := filepath.Join(c.root, PackageCacheFormatVersionFileName)
 	data, err := os.ReadFile(versionPath)
 	if err != nil {
 		return false
 	}
 	v, err := strconv.Atoi(strings.TrimSpace(string(data)))
 	return err == nil && v >= CurrentPackageCacheFormatVersion
 }

 func (c *PackageCache) writeFormatVersion() error {
 	versionPath := filepath.Join(c.root, PackageCacheFormatVersionFileName)
 	return os.WriteFile(versionPath, []byte(strconv.Itoa(CurrentPackageCacheFormatVersion)+"\n"), 0644)
 }

 func (c *PackageCache) sanitizeContaminatedInstances(ctx context.Context) {
 	instancesDir := filepath.Join(c.root, "instances")
 	entries, err := os.ReadDir(instancesDir)
 	if err != nil {
 		return
 	}
 	manifestMap := c.loadAllCIPDManifestPaths()
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
 		}
 		instID := entry.Name()
 		instDir := filepath.Join(instancesDir, instID)
 		if c.isInstanceContaminated(instID, instDir, manifestMap) {
 			log.Warningf(ctx, "Detected contaminated package instance %s; purging cache and CIPD metadata slot to force re-download", instID)
 			_ = os.RemoveAll(instDir)
 			c.removeCIPDPkgForInstance(instID)
 		}
 	}
 }

 // removeCIPDPkgForInstance removes the .cipd/pkgs metadata directory associated
 // with instID. When an instance directory is missing on disk, cleaning up the
 // corresponding CIPD package slot forces CIPD to re-deploy it during ensure.
 func (c *PackageCache) removeCIPDPkgForInstance(instID string) {
 	pkgsDir := filepath.Join(c.root, ".cipd", "pkgs")
 	entries, err := os.ReadDir(pkgsDir)
 	if err != nil {
 		return
 	}
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
 		}
 		pDir := filepath.Join(pkgsDir, entry.Name())
 		descBytes, err := os.ReadFile(filepath.Join(pDir, "description.json"))
 		if err == nil && strings.Contains(string(descBytes), instID) {
 			_ = os.RemoveAll(pDir)
 			continue
 		}
 		if _, err := os.Stat(filepath.Join(pDir, instID)); err == nil {
 			_ = os.RemoveAll(pDir)
 			continue
 		}
 	}
 }

 // loadAllCIPDManifestPaths scans .cipd/pkgs to map instance IDs to their .cipdpkg/manifest.json paths.
 func (c *PackageCache) loadAllCIPDManifestPaths() map[string]string {
 	manifests := make(map[string]string)
 	pkgsDir := filepath.Join(c.root, ".cipd", "pkgs")
 	entries, err := os.ReadDir(pkgsDir)
 	if err != nil {
 		return manifests
 	}
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
 		}
 		pDir := filepath.Join(pkgsDir, entry.Name())
 		subEntries, err := os.ReadDir(pDir)
 		if err != nil {
 			continue
 		}
 		for _, sub := range subEntries {
 			if !sub.IsDir() || len(sub.Name()) < 20 {
 				continue
 			}
 			mPath := filepath.Join(pDir, sub.Name(), ".cipdpkg", "manifest.json")
 			if fi, err := os.Stat(mPath); err == nil && !fi.IsDir() {
 				manifests[sub.Name()] = mPath
 			}
 		}
 	}
 	return manifests
 }

 type cipdPackageManifest struct {
 	FormatVersion string `json:"format_version"`
 	PackageName   string `json:"package_name"`
 	VersionFile   string `json:"version_file"`
 	Files         []struct {
 		Name string `json:"name"`
 	} `json:"files"`
 }

 // isInstanceContaminated checks if an existing instance directory contains extraneous
 // files that are not tracked in its CIPD package manifest.
 func (c *PackageCache) isInstanceContaminated(instID, instDir string, manifestMap map[string]string) bool {
 	manifestPath, ok := manifestMap[instID]
 	if !ok {
 		return false
 	}

 	manifestBytes, err := os.ReadFile(manifestPath)
 	if err != nil {
 		// If CIPD metadata exists for this instance but its manifest cannot be read,
 		// treat the instance as corrupted to force a clean re-download.
 		return true
 	}
 	var manifest cipdPackageManifest
 	if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
 		return true
 	}

 	knownFiles := make(map[string]bool, len(manifest.Files)+2)
 	for _, f := range manifest.Files {
 		cleanName := filepath.ToSlash(filepath.Clean(f.Name))
 		knownFiles[cleanName] = true
 	}
 	if manifest.VersionFile != "" {
 		cleanName := filepath.ToSlash(filepath.Clean(manifest.VersionFile))
 		knownFiles[cleanName] = true
 	}

 	var fileCount int
 	walkErr := filepath.WalkDir(instDir, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
 		if d.IsDir() {
 			if d.Name() == ".cipdpkg" || d.Name() == ".cipd" {
 				return filepath.SkipDir
 			}
 			return nil
 		}
 		rel, err := filepath.Rel(instDir, path)
 		if err != nil {
 			return err
 		}
 		rel = filepath.ToSlash(filepath.Clean(rel))
 		if rel == CacheStampFileName {
 			return nil
 		}
 		fileCount++
 		if !knownFiles[rel] {
 			return errors.New("contaminated")
 		}
 		return nil
 	})

 	if walkErr != nil || (fileCount == 0 && len(knownFiles) > 0) {
 		return true
 	}

 	return false
 }
	// Copyright 2026 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package project

	import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"go.fuchsia.dev/jiri"
	"go.fuchsia.dev/jiri/cipd"
	"go.fuchsia.dev/jiri/log"
	)

	// PackageCacheFormatVersionFileName is the filename in the package cache root
	// that records the schema version of the central package cache.
	const PackageCacheFormatVersionFileName = ".jiri_cache_format_version"

	// CurrentPackageCacheFormatVersion is the current format version of the package cache.
	// Incrementing this version triggers a one-time sanitization pass across cached instances.
	const CurrentPackageCacheFormatVersion = 2

	// PackageCache manages the central package cache directory (`.jiri_root/packages`).
	//
	// In Jiri, the central package cache serves as the single source of truth for
	// downloaded CIPD package payloads across all active workspaces and worktrees.
	// It stores immutable package instances in content-addressed directories under
	// `instances/<instance_id>`, manages the global non-blocking synchronization lock
	// (`.cache.lock`), and coordinates CIPD ensure synchronization and pruning.
	type PackageCache struct {
	root string
	}

	// NewPackageCache constructs a new PackageCache manager for the specified cache root.
	func NewPackageCache(root string) *PackageCache {
	return &PackageCache{root: root}
	}

	// Root returns the root directory of the package cache.
	func (c *PackageCache) Root() string {
	return c.root
	}

	// InstanceDir returns the absolute directory path where a package instance is installed in cache.
	func (c *PackageCache) InstanceDir(inst PackageCacheInstance) string {
	return filepath.Join(c.root, filepath.FromSlash(inst.Subdir()))
	}

	// InstanceExists checks if the instance directory exists in the cache.
	func (c *PackageCache) InstanceExists(inst PackageCacheInstance) bool {
	fi, err := os.Stat(c.InstanceDir(inst))
	return err == nil && fi.IsDir()
	}

	// HasAllInstances reports whether all specified instances exist in the cache.
	// It returns false if instances is empty or any instance is missing.
	func (c *PackageCache) HasAllInstances(instances []PackageCacheInstance) bool {
	if len(instances) == 0 {
	return false
	}
	for _, inst := range instances {
	if !c.InstanceExists(inst) {
	return false
	}
	}
	return true
	}

	// Lock acquires an exclusive, non-blocking lock on `.cache.lock` within the cache root.
	// This guarantees fail-fast concurrency control across concurrent Jiri processes
	// or active worktrees running package synchronization.
	// Returns an unlock function to release the lock, or an error if acquisition fails.
	func (c *PackageCache) Lock() (unlock func(), err error) {
	return AcquireCacheLock(c.root)
	}

	// Sync runs CIPD cache synchronization using the provided CIPDEnsureFile.
	// It writes the ensure specification to `instance_master.ensure` and invokes CIPD ensure,
	// which validates, repairs, downloads, and natively prunes unreferenced instances
	// under `instances/<instance_id>`. The caller must hold cache.Lock() before calling Sync.
	func (c PackageCache) Sync(ctx context.Context, ensureFile CIPDEnsureFile, timeout uint) error {
	checkout := jiri.CheckoutFromContext(ctx)
	if err := os.MkdirAll(c.root, 0755); err != nil {
	return err
	}

	// Perform a one-time migration check to purge any cached instances that were
	// previously contaminated by multi-package standalone adoption.
	if !c.isFormatVersionCurrent() {
	c.sanitizeContaminatedInstances(ctx)
	_ = c.writeFormatVersion()
	}

	// In non-paranoid mode, CIPD assumes packages are intact if their metadata
	// slot in .cipd/pkgs exists and does not check the filesystem. If an instance
	// directory is missing or corrupted, clean up any corrupted file and remove the
	// corresponding .cipd/pkgs metadata slot so CIPD notices and redeploys it.
	for _, inst := range ensureFile.Instances() {
	instDir := c.InstanceDir(inst)
	fi, err := os.Lstat(instDir)
	if err != nil \|\| !fi.IsDir() {
	if err == nil {
	_ = os.RemoveAll(instDir)
	}
	c.removeCIPDPkgForInstance(inst.InstanceID)
	}
	}

	masterEnsurePath := filepath.Join(c.root, "instance_master.ensure")
	if err := ensureFile.Write(masterEnsurePath, checkout.CipdParanoidMode); err != nil {
	return err
	}
	defer os.Remove(masterEnsurePath)

	jiri.TimerPush(ctx, "cipd package cache sync")
	defer jiri.TimerPop(ctx)

	return cipd.Ensure(ctx, masterEnsurePath, c.root, timeout)
	}

	func (c *PackageCache) isFormatVersionCurrent() bool {
	versionPath := filepath.Join(c.root, PackageCacheFormatVersionFileName)
	data, err := os.ReadFile(versionPath)
	if err != nil {
	return false
	}
	v, err := strconv.Atoi(strings.TrimSpace(string(data)))
	return err == nil && v >= CurrentPackageCacheFormatVersion
	}

	func (c *PackageCache) writeFormatVersion() error {
	versionPath := filepath.Join(c.root, PackageCacheFormatVersionFileName)
	return os.WriteFile(versionPath, []byte(strconv.Itoa(CurrentPackageCacheFormatVersion)+"\n"), 0644)
	}

	func (c *PackageCache) sanitizeContaminatedInstances(ctx context.Context) {
	instancesDir := filepath.Join(c.root, "instances")
	entries, err := os.ReadDir(instancesDir)
	if err != nil {
	return
	}
	manifestMap := c.loadAllCIPDManifestPaths()
	for _, entry := range entries {
	if !entry.IsDir() {
	continue
	}
	instID := entry.Name()
	instDir := filepath.Join(instancesDir, instID)
	if c.isInstanceContaminated(instID, instDir, manifestMap) {
	log.Warningf(ctx, "Detected contaminated package instance %s; purging cache and CIPD metadata slot to force re-download", instID)
	_ = os.RemoveAll(instDir)
	c.removeCIPDPkgForInstance(instID)
	}
	}
	}

	// removeCIPDPkgForInstance removes the .cipd/pkgs metadata directory associated
	// with instID. When an instance directory is missing on disk, cleaning up the
	// corresponding CIPD package slot forces CIPD to re-deploy it during ensure.
	func (c *PackageCache) removeCIPDPkgForInstance(instID string) {
	pkgsDir := filepath.Join(c.root, ".cipd", "pkgs")
	entries, err := os.ReadDir(pkgsDir)
	if err != nil {
	return
	}
	for _, entry := range entries {
	if !entry.IsDir() {
	continue
	}
	pDir := filepath.Join(pkgsDir, entry.Name())
	descBytes, err := os.ReadFile(filepath.Join(pDir, "description.json"))
	if err == nil && strings.Contains(string(descBytes), instID) {
	_ = os.RemoveAll(pDir)
	continue
	}
	if _, err := os.Stat(filepath.Join(pDir, instID)); err == nil {
	_ = os.RemoveAll(pDir)
	continue
	}
	}
	}

	// loadAllCIPDManifestPaths scans .cipd/pkgs to map instance IDs to their .cipdpkg/manifest.json paths.
	func (c *PackageCache) loadAllCIPDManifestPaths() map[string]string {
	manifests := make(map[string]string)
	pkgsDir := filepath.Join(c.root, ".cipd", "pkgs")
	entries, err := os.ReadDir(pkgsDir)
	if err != nil {
	return manifests
	}
	for _, entry := range entries {
	if !entry.IsDir() {
	continue
	}
	pDir := filepath.Join(pkgsDir, entry.Name())
	subEntries, err := os.ReadDir(pDir)
	if err != nil {
	continue
	}
	for _, sub := range subEntries {
	if !sub.IsDir() \|\| len(sub.Name()) < 20 {
	continue
	}
	mPath := filepath.Join(pDir, sub.Name(), ".cipdpkg", "manifest.json")
	if fi, err := os.Stat(mPath); err == nil && !fi.IsDir() {
	manifests[sub.Name()] = mPath
	}
	}
	}
	return manifests
	}

	type cipdPackageManifest struct {
	FormatVersion string `json:"format_version"`
	PackageName string `json:"package_name"`
	VersionFile string `json:"version_file"`
	Files []struct {
	Name string `json:"name"`
	} `json:"files"`
	}

	// isInstanceContaminated checks if an existing instance directory contains extraneous
	// files that are not tracked in its CIPD package manifest.
	func (c *PackageCache) isInstanceContaminated(instID, instDir string, manifestMap map[string]string) bool {
	manifestPath, ok := manifestMap[instID]
	if !ok {
	return false
	}

	manifestBytes, err := os.ReadFile(manifestPath)
	if err != nil {
	// If CIPD metadata exists for this instance but its manifest cannot be read,
	// treat the instance as corrupted to force a clean re-download.
	return true
	}
	var manifest cipdPackageManifest
	if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
	return true
	}

	knownFiles := make(map[string]bool, len(manifest.Files)+2)
	for _, f := range manifest.Files {
	cleanName := filepath.ToSlash(filepath.Clean(f.Name))
	knownFiles[cleanName] = true
	}
	if manifest.VersionFile != "" {
	cleanName := filepath.ToSlash(filepath.Clean(manifest.VersionFile))
	knownFiles[cleanName] = true
	}

	var fileCount int
	walkErr := filepath.WalkDir(instDir, func(path string, d os.DirEntry, err error) error {
	if err != nil {
	return err
	}
	if d.IsDir() {
	if d.Name() == ".cipdpkg" \|\| d.Name() == ".cipd" {
	return filepath.SkipDir
	}
	return nil
	}
	rel, err := filepath.Rel(instDir, path)
	if err != nil {
	return err
	}
	rel = filepath.ToSlash(filepath.Clean(rel))
	if rel == CacheStampFileName {
	return nil
	}
	fileCount++
	if !knownFiles[rel] {
	return errors.New("contaminated")
	}
	return nil
	})

	if walkErr != nil \|\| (fileCount == 0 && len(knownFiles) > 0) {
	return true
	}

	return false
	}