project/cache_migration.go

// Copyright 2026 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package project

import (
	"context"
	"encoding/json"
	"fmt"
	"maps"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"

	"go.fuchsia.dev/jiri"
	"go.fuchsia.dev/jiri/log"
)

// MigrateLegacyGroupCache inspects legacy package-group cache entries (.jiri_root/packages/<hex64>),
// adopts individual package instances into instances/<instance_id>, and immediately removes each legacy
// directory upon adoption.
// This completes the migration and retires the old schema in ~1-2s with 0 bytes of network traffic.
func MigrateLegacyGroupCache(ctx context.Context, cache *PackageCache) error {
	cacheRoot := cache.Root()
	entries, err := os.ReadDir(cacheRoot)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}

	instancesDir := filepath.Join(cacheRoot, "instances")
	cipdPkgsDir := filepath.Join(cacheRoot, ".cipd", "pkgs")

	// Fast check: return immediately if no legacy directories exist
	hasLegacy := false
	for _, entry := range entries {
		if len(entry.Name()) == 64 && isHex(entry.Name()) && entry.IsDir() {
			hasLegacy = true
			break
		}
	}
	if !hasLegacy {
		return nil
	}

	if err := os.MkdirAll(instancesDir, 0755); err != nil {
		return err
	}
	if err := os.MkdirAll(cipdPkgsDir, 0755); err != nil {
		return err
	}

	pkgIndex := nextCIPDPkgIndex(cipdPkgsDir)
	adoptedInstances := make(map[string]bool)

	for _, entry := range entries {
		name := entry.Name()
		if len(name) != 64 || !isHex(name) || !entry.IsDir() {
			continue
		}
		legacyDir := filepath.Join(cacheRoot, name)
		legacyCipdPkgs := filepath.Join(legacyDir, ".cipd", "pkgs")
		var allSuccess bool
		pkgIndex, allSuccess = adoptCIPDPackages(ctx, legacyCipdPkgs, instancesDir, cipdPkgsDir, pkgIndex, adoptedInstances)

		// Immediately remove the legacy directory after adopting its packages
		if allSuccess {
			log.Debugf(ctx, "Retiring legacy group cache entry: %s", name)
			if err := os.RemoveAll(legacyDir); err != nil {
				log.Warningf(ctx, "Failed to remove legacy directory %s: %v", legacyDir, err)
			}
		} else {
			log.Warningf(ctx, "Preserving legacy directory %s due to adoption errors", name)
		}
	}
	return nil
}

// MigrateStandaloneToPackageCache adopts package instances from a standalone CIPD checkout (<root>/.cipd)
// into the package cache (instances/<id>), replicates metadata into .cipd/pkgs, and removes <root>/.cipd.
func MigrateStandaloneToPackageCache(ctx context.Context, cache *PackageCache) error {
	checkout := jiri.CheckoutFromContext(ctx)
	standaloneCipd := filepath.Join(checkout.Root, ".cipd")
	standalonePkgs := filepath.Join(standaloneCipd, "pkgs")
	if fi, err := os.Stat(standalonePkgs); err != nil || !fi.IsDir() {
		return nil
	}

	cacheRoot := cache.Root()
	instancesDir := filepath.Join(cacheRoot, "instances")
	cipdPkgsDir := filepath.Join(cacheRoot, ".cipd", "pkgs")
	if err := os.MkdirAll(instancesDir, 0755); err != nil {
		return err
	}
	if err := os.MkdirAll(cipdPkgsDir, 0755); err != nil {
		return err
	}

	pkgIndex := nextCIPDPkgIndex(cipdPkgsDir)
	adoptedInstances := make(map[string]bool)
	pkgIndex, allSuccess := adoptCIPDPackages(ctx, standalonePkgs, instancesDir, cipdPkgsDir, pkgIndex, adoptedInstances)

	if allSuccess {
		log.Debugf(ctx, "Retiring standalone .cipd directory after adoption")
		_ = os.RemoveAll(standaloneCipd)
	} else {
		log.Warningf(ctx, "Preserving standalone .cipd directory due to adoption errors")
	}
	return nil
}

// MigratePackageCacheToStandalone migrates a checkout from package-cache=true to package-cache=false.
//
// For directories whose cache stamp matches the target instances, it detaches the directory
// in-place by removing .jiri_cache_hash.
// For missing destination directories whose instances exist in cache, it restores the directory
// via local hardlinks without downloading.
// It returns the subset of packages requiring a direct CIPD fetch (e.g. version rolls or uncached packages).
func MigratePackageCacheToStandalone(ctx context.Context, activePkgs Packages, cache *PackageCache) (Packages, error) {
	checkout := jiri.CheckoutFromContext(ctx)
	fetchPkgs := make(Packages)
	index, err := LoadWorktreePackageIndex(ctx, activePkgs)
	if err != nil {
		return nil, err
	}
	dests := index.DestinationsFor(checkout.Root)

	pkgsByRel := make(map[string][]Package)
	for key, pkg := range activePkgs {
		rel, err := pkg.ResolvePath()
		if err != nil {
			fetchPkgs[key] = pkg
			continue
		}
		pkgsByRel[rel] = append(pkgsByRel[rel], pkg)
	}

	// Process destinations in deterministic parent-first order so parent checks and
	// restorations occur before nested child destinations.
	for _, rel := range slices.Sorted(maps.Keys(pkgsByRel)) {
		wsDir := filepath.Join(checkout.Root, rel)
		stamp := NewCacheStamp(wsDir)

		var dest WorkspaceDestination
		for _, d := range dests {
			if d.RelPath == rel {
				dest = d
				break
			}
		}

		// In-place detachment: if matching target is already present, detach stamp.
		// Older stamps are also detached in-place if canAttachInPlace confirms the files match the cache.
		if stamp.MatchesDestination(dest) || (stamp.MatchesInstances(dest) && dest.canAttachInPlace(checkout.Root, cache)) {
			stamp.Remove()
			continue
		}

		// Cache-assisted restoration: if instances exist in cache, restore clean files locally
		// without network traffic when the directory is missing or contained mismatched files.
		if cache.HasAllInstances(dest.Instances) {
			if err := dest.validatePath(checkout.Root); err != nil {
				return nil, err
			}
			_ = os.RemoveAll(wsDir)
			restoreErr := func() error {
				if err := os.MkdirAll(filepath.Dir(wsDir), 0755); err != nil {
					return err
				}
				for _, inst := range dest.Instances {
					instDir := cache.InstanceDir(inst)
					if err := linkDirectoryTreeWithOverwrite(instDir, wsDir); err != nil {
						return err
					}
				}
				if err := fixSymlinksAfterRestoration(ctx, wsDir, cache); err != nil && err != errBrokenSymlink {
					return err
				}
				return nil
			}()
			if restoreErr != nil {
				log.Warningf(ctx, "Failed to restore destination %s from cache to %s: %v", rel, wsDir, restoreErr)
				os.RemoveAll(wsDir)
			} else {
				continue
			}
		}

		// Version roll or missing cache entry: remove stale stamp and queue for CIPD download
		stamp.Remove()
		for _, pkg := range pkgsByRel[rel] {
			fetchPkgs[pkg.Key()] = pkg
		}
	}

	return fetchPkgs, nil
}

// nextCIPDPkgIndex calculates the next available integer index for CIPD package metadata
// directories in `.cipd/pkgs/`. CIPD stores installed package metadata in consecutively
// numbered directories ("0", "1", "2", ...).
func nextCIPDPkgIndex(cipdPkgsDir string) int {
	pkgIndex := 0
	if existing, err := os.ReadDir(cipdPkgsDir); err == nil {
		for _, e := range existing {
			if idx, err := strconv.Atoi(e.Name()); err == nil && idx >= pkgIndex {
				pkgIndex = idx + 1
			}
		}
	}
	return pkgIndex
}

// adoptCIPDPackages scans a source CIPD package directory (such as from a legacy
// package-group cache or a standalone `.cipd` directory) and adopts its packages into
// the central package cache `instances/<instance_id>` and `.cipd/pkgs/<index>`.
// By deduplicating on instanceID, each instance gets a single clean slot in `.cipd/pkgs`,
// completely avoiding duplicate directory warnings and eliminating the ~3 minute verification overhead.
func adoptCIPDPackages(ctx context.Context, sourceCipdPkgsDir, instancesDir, cipdPkgsDir string, pkgIndex int, adoptedInstances map[string]bool) (int, bool) {
	allSuccess := true
	pkgEntries, err := os.ReadDir(sourceCipdPkgsDir)
	if err != nil {
		return pkgIndex, false
	}

	type cipdPkgInfo struct {
		pDir        string
		packageName string
		subdir      string
		cleanSubdir string
	}
	var validPkgs []cipdPkgInfo
	subdirCounts := make(map[string]int)

	// Parse description.json once and count how many packages share each destination subdirectory.
	// When multiple packages share the same destination (or point to root "."), the deployed files
	// on disk are merged together.
	for _, pEntry := range pkgEntries {
		if !pEntry.IsDir() {
			continue
		}
		pDir := filepath.Join(sourceCipdPkgsDir, pEntry.Name())
		descBytes, err := os.ReadFile(filepath.Join(pDir, "description.json"))
		if err != nil {
			allSuccess = false
			continue
		}
		var desc struct {
			PackageName string `json:"package_name"`
			Subdir      string `json:"subdir"`
		}
		if err := json.Unmarshal(descBytes, &desc); err != nil || desc.PackageName == "" {
			allSuccess = false
			continue
		}
		cleanSubdir := filepath.Clean(desc.Subdir)
		subdirCounts[cleanSubdir]++
		validPkgs = append(validPkgs, cipdPkgInfo{
			pDir:        pDir,
			packageName: desc.PackageName,
			subdir:      desc.Subdir,
			cleanSubdir: cleanSubdir,
		})
	}

	sourceRootDir := filepath.Dir(filepath.Dir(sourceCipdPkgsDir))
	for _, pkg := range validPkgs {
		pDir := pkg.pDir
		cleanSubdir := pkg.cleanSubdir

		instEntries, _ := os.ReadDir(pDir)
		for _, instEntry := range instEntries {
			instID := instEntry.Name()
			if len(instID) < 20 || !instEntry.IsDir() {
				continue
			}

			// Deduplicate: if this exact instance ID has already been adopted into a slot, skip creating a duplicate slot.
			if adoptedInstances[instID] {
				continue
			}

			srcInstDir := filepath.Join(pDir, instID)
			targetInstDir := filepath.Join(instancesDir, instID)

			// If the destination subdirectory is shared by multiple packages, encloses a nested package
			// destination, or points to the root directory ".", skip local adoption entirely. Adopting
			// from a shared, parent, or root directory would contaminate the instance cache with foreign
			// files from other packages or the workspace. CIPD will download clean, isolated instances
			// during cache sync.
			hasNestedSubdir := false
			for sub := range subdirCounts {
				if isNestedUnder(sub, cleanSubdir) {
					hasNestedSubdir = true
					break
				}
			}
			if subdirCounts[cleanSubdir] > 1 || hasNestedSubdir || cleanSubdir == "." {
				log.Debugf(ctx, "Skipping standalone adoption for %s (shares or encloses destination %s with multiple packages); CIPD will install cleanly", pkg.packageName, pkg.subdir)
				continue
			}

			// Single-package destination: adopt from the deployed subdirectory, falling back to srcInstDir.
			actualSourceDir := filepath.Join(sourceRootDir, pkg.subdir)
			if fi, err := os.Stat(actualSourceDir); err == nil && fi.IsDir() {
				if _, err := os.Stat(targetInstDir); os.IsNotExist(err) {
					if err := linkDirectoryTree(actualSourceDir, targetInstDir, ".cipd"); err != nil {
						log.Warningf(ctx, "Skipping adoption for %s: %v", instID, err)
						allSuccess = false
						continue
					}
					_ = os.Remove(filepath.Join(targetInstDir, ".jiri_cache_hash"))
				}
			} else if _, err := os.Stat(targetInstDir); os.IsNotExist(err) {
				// Fall back to linking from the CIPD package slot directory if the deployed
				// package directory does not exist, preserving any available package metadata.
				if err := linkDirectoryTree(srcInstDir, targetInstDir, ".cipdpkg"); err != nil {
					log.Warningf(ctx, "Skipping adoption for %s: %v", instID, err)
					allSuccess = false
					continue
				}
			}

			targetPkgDir := filepath.Join(cipdPkgsDir, strconv.Itoa(pkgIndex))
			if _, err := os.Stat(targetPkgDir); os.IsNotExist(err) {
				if err := os.MkdirAll(targetPkgDir, 0755); err != nil {
					log.Warningf(ctx, "Failed to create metadata dir for %s: %v", instID, err)
					allSuccess = false
					continue
				}
				newDesc := fmt.Sprintf("{\n  \"subdir\": \"instances/%s\",\n  \"package_name\": %q\n}\n", instID, pkg.packageName)
				if err := os.WriteFile(filepath.Join(targetPkgDir, "description.json"), []byte(newDesc), 0644); err != nil {
					log.Warningf(ctx, "Failed to write description.json for %s: %v", instID, err)
					allSuccess = false
					continue
				}
				if err := os.WriteFile(filepath.Join(targetPkgDir, ".lock"), nil, 0644); err != nil {
					log.Warningf(ctx, "Failed to write .lock for %s: %v", instID, err)
					allSuccess = false
					continue
				}
				if err := os.Symlink(instID, filepath.Join(targetPkgDir, "_current")); err != nil {
					log.Warningf(ctx, "Failed to symlink _current for %s: %v", instID, err)
					allSuccess = false
					continue
				}
				// Copy only the small .cipdpkg metadata manifest into targetPkgDir/<instID>/.cipdpkg/
				srcCipdPkg := filepath.Join(srcInstDir, ".cipdpkg")
				if _, err := os.Stat(srcCipdPkg); err == nil {
					targetCipdPkg := filepath.Join(targetPkgDir, instID, ".cipdpkg")
					_ = os.MkdirAll(targetCipdPkg, 0755)
					_ = copyTreeWithHardlinks(srcCipdPkg, targetCipdPkg)
				}
			}

			adoptedInstances[instID] = true
			pkgIndex++
		}
	}
	return pkgIndex, allSuccess
}

func isNestedUnder(child, parent string) bool {
	rel, err := filepath.Rel(parent, child)
	return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}

// isHex checks whether a string contains only valid hexadecimal characters.
// It is used to identify legacy 64-character SHA-256 package group directories.
func isHex(s string) bool {
	for i := 0; i < len(s); i++ {
		b := s[i]
		if !(b >= '0' && b <= '9' || b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F') {
			return false
		}
	}
	return true
}