feat: select a toolchain-scoped shared Lake cache with private fallback
diff --git a/src/lake/Lake/Build/Common.lean b/src/lake/Lake/Build/Common.lean index 8100ffc..2fc13c4 100644 --- a/src/lake/Lake/Build/Common.lean +++ b/src/lake/Lake/Build/Common.lean
@@ -454,6 +454,16 @@ clearFileHash file setTrace (← fetchFileTrace file text) +def sharePrivateArtifact (cached file : FilePath) : IO Unit := do + try + if (← file.metadata).numLinks == 1 then + withTempFileFor file fun tmp => do + IO.FS.removeFile tmp + IO.FS.hardLink cached tmp + IO.FS.rename tmp file + catch _ => pure () + + /-- Copies `file` to the Lake cache with the file extension `ext`, and saves its hash in its `.hash` file. @@ -482,6 +492,7 @@ createParentDirs cacheFile writeFileAtomic cacheFile normalized (overwrite := false) (readOnly := true) IO.setAccessRights cacheFile ⟨r, r, r⟩ + if useLocalFile then sharePrivateArtifact cacheFile file writeFileHash file hash let mtime ← getMTime cacheFile let path := if useLocalFile then file else cacheFile @@ -502,6 +513,7 @@ unless e matches .alreadyExists .. do writeBinFileAtomic cacheFile contents (overwrite := false) (readOnly := true) (exe := exe) IO.setAccessRights cacheFile ⟨r, r, r⟩ + if useLocalFile then sharePrivateArtifact cacheFile file writeFileHash file hash let mtime ← getMTime cacheFile let path := if useLocalFile then file else cacheFile @@ -656,7 +668,12 @@ **For internal use.** -/ public def restoreArtifact (file : FilePath) (art : Artifact) (exe := false) : LogIO Artifact := do - unless (← file.pathExists) do + if (← file.pathExists) then + sharePrivateArtifact art.path file + if exe then + let rights := {read := true, write := false, execution := true} + IO.setAccessRights file ⟨rights, rights, rights⟩ + else logVerbose s!"found artifact in cache: {art.path}" createParentDirs file if let .error e ← (IO.FS.hardLink art.path file).toBaseIO then
diff --git a/src/lake/Lake/CLI/Help.lean b/src/lake/Lake/CLI/Help.lean index d1d2af4..dc39c7e 100644 --- a/src/lake/Lake/CLI/Help.lean +++ b/src/lake/Lake/CLI/Help.lean
@@ -48,6 +48,7 @@ BASIC OPTIONS: --version print version and exit + --shared-cache-dir print the compatible native shared cache, or an empty line --help, -h print help of the program or a command and exit --dir, -d=file use the package configuration in a specific directory --file, -f=file use a specific file for the package configuration
diff --git a/src/lake/Lake/CLI/Main.lean b/src/lake/Lake/CLI/Main.lean index 963b865..4c8dd2b 100644 --- a/src/lake/Lake/CLI/Main.lean +++ b/src/lake/Lake/CLI/Main.lean
@@ -528,19 +528,23 @@ def endpointDeprecation : String := "configuring the cache service via environment variables is deprecated; use --service instead" -protected def get : CliM PUnit := do - processOptions lakeOption - let opts ← getThe LakeOptions - let mappings? ← takeArg? - noArgsRem do - let cfg ← mkLoadConfig opts - let ws ← loadWorkspace cfg - let cache := ws.lakeCache +def withPrivateCacheFallback (ws : Workspace) (action : Cache → LoggerIO α) : LoggerIO α := do + if ws.lakeEnv.sharedLakeCache?.any (·.dir == ws.lakeCache.dir) then + if let some fallback := ws.lakeEnv.privateLakeCache? then + let (value?, log) ← (action ws.lakeCache).captureLog + if let some value := value? then + log.replay + return value + for entry in log.entries do + logEntry {entry with level := min entry.level .warning} + logWarning "shared local cache unavailable; using the private cache" + return ← action fallback + action ws.lakeCache + +def getCore (cache : Cache) (ws : Workspace) (opts : LakeOptions) + (pkg? : Option Package) (mappings? : Option String) +: LoggerIO Unit := do let overwrite := opts.overwrite?.getD true - unless overwrite do - -- artifacts of skipped mappings with `--no-overwrite` cannot be cleanly handled - error "`--no-overwrite` is not supported for `cache get`" - let pkg? ← opts.package?.bindM (liftM <| parsePackageSpec ws ·) if let some file := mappings? then liftM (m := LoggerIO) do if opts.mappingsOnly then error "`--mappings-only` is not supported with a mappings file; use `lake cache add` instead" @@ -660,6 +664,19 @@ service.downloadArtifacts descrs cache remoteScope opts.forceDownload cache.writeMap pkg.cacheScope map service.name? (some remoteScope) overwrite +protected def get : CliM PUnit := do + processOptions lakeOption + let opts ← getThe LakeOptions + let mappings? ← takeArg? + noArgsRem do + let cfg ← mkLoadConfig opts + let ws ← loadWorkspace cfg + unless opts.overwrite?.getD true do + error "`--no-overwrite` is not supported for `cache get`" + let pkg? ← opts.package?.bindM (liftM <| parsePackageSpec ws ·) + liftM (m := LoggerIO) <| withPrivateCacheFallback ws fun cache => + getCore cache ws opts pkg? mappings? + def computeUploadService (service? : Option String) (lakeEnv : Env) (lakeCfg : LoadedLakeConfig) : CliStateM CacheService := do @@ -762,7 +779,8 @@ return some (.ofString service) let map ← CacheMap.load file let overwrite := opts.overwrite?.getD true - ws.lakeCache.writeMap localScope map service? opts.scope? overwrite + liftM (m := LoggerIO) <| withPrivateCacheFallback ws fun cache => + liftM <| cache.writeMap localScope map service? opts.scope? overwrite def stagingOutputsFile := "outputs.jsonl" @@ -825,29 +843,28 @@ return some (.ofString service) let map ← CacheMap.load (stagingDir / stagingOutputsFile) let descrs ← map.collectOutputDescrs - let artDir := ws.lakeCache.artifactDir - IO.FS.createDirAll artDir - let overwrite := opts.overwrite?.getD false - let ok ← descrs.foldlM (init := true) fun ok descr => do - let cachePath := artDir/ descr.relPath - let stagingPath := stagingDir / descr.relPath - if (← cachePath.pathExists) && !overwrite then return ok - let publish := do - let contents ← IO.FS.readBinFile stagingPath - writeBinFileAtomic cachePath contents overwrite (readOnly := true) - match (← publish |>.toBaseIO) with - | .ok _ => - return ok - | .error (.noFileOrDirectory ..) => - logError s!"output artifact not found in staging directory: {stagingPath}" - return false - | .error e => - logError s!"failed to copy artifact: {e}" - return false - unless ok do - logError "failed to copy all outputs to the staging directory" - exit 1 - ws.lakeCache.writeMap localScope map service? opts.scope? overwrite + liftM (m := LoggerIO) <| withPrivateCacheFallback ws fun cache => do + IO.FS.createDirAll cache.artifactDir + let overwrite := opts.overwrite?.getD false + let ok ← descrs.foldlM (init := true) fun ok descr => do + let cachePath := cache.artifactDir / descr.relPath + let stagingPath := stagingDir / descr.relPath + if (← cachePath.pathExists) && !overwrite then return ok + let publish := do + let contents ← IO.FS.readBinFile stagingPath + if ws.lakeEnv.sharedLakeCache?.any (fun (shared : Cache) => shared.dir == cache.dir) && Hash.ofByteArray contents != descr.hash then + throw <| IO.userError s!"{stagingPath}: staged artifact hash mismatch" + writeBinFileAtomic cachePath contents overwrite (readOnly := true) + match (← publish |>.toBaseIO) with + | .ok _ => return ok + | .error (.noFileOrDirectory ..) => + logError s!"output artifact not found in staging directory: {stagingPath}" + return false + | .error e => + logError s!"failed to copy artifact: {e}" + return false + unless ok do error "failed to copy all outputs to the staging directory" + cache.writeMap localScope map service? opts.scope? overwrite protected def putStaged : CliM PUnit := do processOptions lakeOption @@ -1445,6 +1462,9 @@ match (← getArgs) with | [] => IO.println usage | ["--version"] => IO.println uiVersionString + | ["--shared-cache-dir"] => + let env ← (← getThe LakeOptions).computeEnv + if let some cache := env.sharedLakeCache? then IO.println cache.dir else IO.println "" | _ => -- normal CLI processLeadingOptions lakeOption -- between `lake` and command if let some cmd ← takeArg? then
diff --git a/src/lake/Lake/Config/Env.lean b/src/lake/Lake/Config/Env.lean index 50f6408..e580071 100644 --- a/src/lake/Lake/Config/Env.lean +++ b/src/lake/Lake/Config/Env.lean
@@ -8,6 +8,7 @@ prelude public import Lake.Config.Cache public import Lake.Config.InstallPath +import Lake.Util.IO import Init.System.Platform open System @@ -158,6 +159,52 @@ else return none +/-- Recognize the native shared namespace while preserving the public environment layout. -/ +public def sharedLakeCache? (env : Env) : Option Cache := do + let active ← env.lakeCache? + let fallback ← env.lakeSystemCache? + guard (active.dir != fallback.dir) + guard (active.dir.parent.bind FilePath.fileName == some "toolchains") + return active + +/-- The unchanged system-cache field retains the explicitly configured private fallback. -/ +public def privateLakeCache? (env : Env) : Option Cache := + if env.sharedLakeCache?.isSome then env.lakeSystemCache? else none + +/-- Select shared storage only for supported release toolchains with restored artifacts. -/ +def sharedCache? (env : Env) : IO (Option Cache) := do + let some root ← IO.getEnv "LAKE_SHARED_CACHE_ROOT" | return none + unless env.lakeCache?.isSome && env.lakeCache?.map (·.dir) == env.lakeSystemCache?.map (·.dir) do + return none + if let some fallback ← IO.getEnv "LAKE_SHARED_CACHE_FALLBACK" then + unless (← IO.getEnv "LAKE_CACHE_DIR") == some fallback do return none + if root.isEmpty || env.noSystemCache || env.restoreAllArtifacts? != some true || + env.enableArtifactCache? == some false || + (!env.githashOverride.isEmpty && env.githashOverride != env.lean.githash) then + return none + unless env.toolchain.startsWith "leanprover/lean4:v" || + env.toolchain.startsWith "leanprover/lean4-nightly:nightly-" do + return none + let some elan := env.elan? | return none + let install := elan.toolchainDir env.toolchain + if (← install.symlinkMetadata).type == .symlink then return none + let sysroot ← IO.FS.realPath install + unless (← IO.FS.realPath env.lean.leanLibDir) == (← IO.FS.realPath (sysroot / "lib" / "lean")) do + return none + let actual ← env.lean.sharedDynlibs.mapM fun lib => IO.FS.realPath lib.path + let expected ← (leanSharedDynlibs sysroot).mapM fun lib => IO.FS.realPath lib.path + unless actual == expected do return none + let identity := Hash.ofString (sysroot.toString ++ env.lean.githash) + let cache : Cache := ⟨FilePath.mk root / "toolchains" / identity.toString⟩ + for name in #["artifacts", "outputs", "revisions"] do + withTempFileFor (cache.dir / name / "probe") fun _ => pure () + let cwd ← IO.currentDir + withTempFileFor (cwd / ".lake" / "probe") fun source => + withTempFileFor (cache.artifactDir / "probe") fun target => do + IO.FS.removeFile target + IO.FS.hardLink source target + return some cache + /-- Compute a `Lake.Env` object from the given installs and the set environment variables. @@ -169,7 +216,7 @@ let reservoirBaseUrl ← getUrlD "RESERVOIR_API_BASE_URL" "https://reservoir.lean-lang.org/api" let toolchain ← computeToolchain let userHome? ← getUserHome? - addCacheDirs userHome? toolchain { + let env : Env ← addCacheDirs userHome? toolchain { lake, lean, elan?, pkgUrlMap := ← computePkgUrlMap reservoirApiUrl := ← getUrlD "RESERVOIR_API_URL" s!"{reservoirBaseUrl}/v1" @@ -188,6 +235,10 @@ initSharedLibPath := ← getSearchPath sharedLibPathEnvVar, initPath := ← getSearchPath "PATH" } + match (← sharedCache? env |>.toBaseIO) with + | .ok (some cache) => + return {env with lakeCache? := some cache} + | _ => return env where addCacheDirs userHome? toolchain env := do if let some dir ← IO.getEnv "LAKE_CACHE_DIR" then @@ -307,7 +358,8 @@ /-- Environment variable settings for the `Lake.Env`. -/ public def vars (env : Env) : Array (String × Option String) := let vars := env.baseVars ++ #[ - ("LAKE_CACHE_DIR", if let some cache := env.lakeCache? then cache.dir.toString else ""), + ("LAKE_CACHE_DIR", if let some cache := + (if env.sharedLakeCache?.isSome then env.privateLakeCache? else env.lakeCache?) then cache.dir.toString else ""), ("LAKE_ARTIFACT_CACHE", if let some b := env.enableArtifactCache? then toString b else ""), ("LAKE_RESTORE_ARTIFACTS", if let some b := env.restoreAllArtifacts? then toString b else ""), ("LEAN_PATH", some env.leanPath.toString),
diff --git a/src/lake/Lake/Config/Workspace.lean b/src/lake/Lake/Config/Workspace.lean index 3bbed35..cadb8cf 100644 --- a/src/lake/Lake/Config/Workspace.lean +++ b/src/lake/Lake/Config/Workspace.lean
@@ -29,7 +29,9 @@ Computes the cache to use for the package based on the environment. -/ public def computeLakeCache (pkg : Package) (lakeEnv : Lake.Env) : Cache := - if pkg.bootstrap then + if lakeEnv.sharedLakeCache?.isSome && pkg.restoreAllArtifacts? == some false then + lakeEnv.privateLakeCache?.getD ⟨pkg.lakeDir / "cache"⟩ + else if pkg.bootstrap then lakeEnv.lakeSystemCache?.getD ⟨pkg.lakeDir / "cache"⟩ else lakeEnv.lakeCache?.getD ⟨pkg.lakeDir / "cache"⟩ @@ -416,7 +418,8 @@ -/ public def augmentedEnvVars (self : Workspace) : Array (String × Option String) := let vars := self.lakeEnv.baseVars ++ #[ - ("LAKE_CACHE_DIR", some self.lakeCache.dir.toString), + ("LAKE_CACHE_DIR", some <| if self.lakeEnv.sharedLakeCache?.isSome then + self.lakeEnv.privateLakeCache?.map (·.dir.toString) |>.getD "" else self.lakeCache.dir.toString), ("LAKE_ARTIFACT_CACHE", if let some b := self.enableArtifactCache? then toString b else ""), ("LAKE_RESTORE_ARTIFACTS", if let some b := self.restoreAllArtifacts? then toString b else ""), ("LEAN_PATH", some self.augmentedLeanPath.toString),
diff --git a/src/lake/Lake/Load/Workspace.lean b/src/lake/Lake/Load/Workspace.lean index 1a42a44..48ebe78 100644 --- a/src/lake/Lake/Load/Workspace.lean +++ b/src/lake/Lake/Load/Workspace.lean
@@ -13,6 +13,7 @@ import Lake.Load.Lean.Eval import Lake.Load.Toml import Lake.Build.InitFacets +import Lake.Util.IO /-! # Workspace Loader @@ -61,12 +62,25 @@ public def loadWorkspace (config : LoadConfig) : LoggerIO Workspace := do let {reconfigure, leanOpts, updateDeps, updateToolchain, packageOverrides, ..} := config let ws ← loadWorkspaceRoot config - if updateDeps then + let ws ← if updateDeps then ws.updateAndMaterialize {} leanOpts updateToolchain else if let some manifest ← Manifest.load? ws.manifestFile then ws.materializeDeps manifest leanOpts reconfigure packageOverrides else ws.updateAndMaterialize {} leanOpts updateToolchain + if ws.lakeEnv.sharedLakeCache?.isSome && ws.packages.any (·.restoreAllArtifacts? == some false) then + return {ws with lakeCache := ws.lakeEnv.privateLakeCache?.getD ⟨ws.root.lakeDir / "cache"⟩} + else if ws.lakeEnv.sharedLakeCache?.isSome then + let probe : IO Unit := do + for pkg in ws.packages do + withTempFileFor (ws.lakeCache.outputsFile pkg.cacheScope .nil) fun _ => pure () + withTempFileFor (ws.lakeCache.revisionPath pkg.cacheScope "probe") fun _ => pure () + let available ← probe.toBaseIO + match available with + | .ok _ => return ws + | .error _ => + return {ws with lakeCache := ws.lakeEnv.privateLakeCache?.getD ⟨ws.root.lakeDir / "cache"⟩} + else return ws /-- Updates the manifest for the loaded Lake workspace (see `updateAndMaterialize`). -/ public def updateManifest
diff --git a/src/lake/README.md b/src/lake/README.md index 2bc9c78..b31a5fa 100644 --- a/src/lake/README.md +++ b/src/lake/README.md
@@ -575,6 +575,26 @@ Greet the entity with the given name. Otherwise, greet the whole world. ``` +## Sharing a local artifact store + +`LAKE_SHARED_CACHE_ROOT` opts a compatible Lake executable into a local store +under `toolchains/<identity>`. It requires restored artifacts, a recognized +release toolchain, and working hard links between the store and workspace. +Forwarding sysroots can use the same namespace when their Lean libraries resolve +to the installed release's libraries. Compiler command wrappers remain in use. +Otherwise the ordinary `LAKE_CACHE_DIR` remains in use. The native store publishes +complete artifacts and mappings atomically and treats eviction during restoration +as a cache miss. + +Keep `LAKE_CACHE_DIR` pointing to a private fallback when older Lake executables +may run in the same environment: they ignore the shared-cache opt-in. Setting +`LAKE_SHARED_CACHE_FALLBACK` to that private path lets an explicit command-local +`LAKE_CACHE_DIR` override disable shared selection. Children retain the private +`LAKE_CACHE_DIR`; each compatible executable selects its own shared store. +`lake --shared-cache-dir` reports the initial selection without loading project +configuration. Package overrides requiring non-restored paths keep the workspace +private. Retention policy belongs to the caller; no age or size cap is implied. + ## Building and Running Lake from the Source If you already have a Lean installation with `lake` packaged with it, you can build a new `lake` by just running `lake build`.
diff --git a/tests/lake/tests/sharedCache/Probe.lean b/tests/lake/tests/sharedCache/Probe.lean new file mode 100644 index 0000000..209bac0 --- /dev/null +++ b/tests/lake/tests/sharedCache/Probe.lean
@@ -0,0 +1,19 @@ +import Lake.Config.Cache +import Lake.Build.Common + +/-! Exercise shared-cache integration through existing Lake artifact APIs. -/ + +open System Lake + +def main (args : List String) : IO UInt32 := do + match args with + | ["restore-exe", root, source, target] => + let artifact ← (Cache.mk root).saveArtifact source "exe" + let result ← (restoreArtifact target artifact (exe := true)).toBaseIO + return if result.isSome then 0 else 1 + | ["map", path, contents] => + let artifact := artifactWithExt (Hash.ofByteArray contents.toUTF8) "art" + let map := ({} : CacheMap).insert (Hash.ofString "input") artifact + let result ← (map.writeFile path).toBaseIO + return if result.isSome then 0 else 1 + | _ => throw <| IO.userError "expected restore-exe CACHE SOURCE TARGET or map PATH CONTENTS"
diff --git a/tests/lake/tests/sharedCache/test.py b/tests/lake/tests/sharedCache/test.py new file mode 100644 index 0000000..645775a --- /dev/null +++ b/tests/lake/tests/sharedCache/test.py
@@ -0,0 +1,374 @@ +"""Exercise opt-in shared-cache selection, compatibility and reuse.""" + +import http.server +import json +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest + + +@unittest.skipUnless(os.name == "posix", "native workspace fixtures use POSIX permissions") +class NativeWorkspaces(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = pathlib.Path(self.temp.name) + executable = shutil.which(os.environ.get("LAKE", "lake")) + self.assertIsNotNone(executable, "the test harness must provide Lake on PATH or through LAKE") + self.lake = pathlib.Path(executable).resolve() + prefix = pathlib.Path(subprocess.check_output( + [os.environ.get("LEAN", "lean"), "--print-prefix"], text=True).strip()) + self.original = prefix / "bin/lake" + elan = self.root / "elan" + install = elan / "toolchains/leanprover--lean4---v0.0.0-cache-fixture" + install.mkdir(parents=True) + # A directory-based fixture distribution uses the current test compiler's + # files without requiring an independently downloaded release toolchain. + for name in ("bin", "lib", "src", "include", "share"): + if (prefix / name).exists(): + if name == "bin": + (install / name).mkdir() + for binary in (prefix / name).iterdir(): + (install / name / binary.name).symlink_to(binary) + else: + (install / name).symlink_to(prefix / name, target_is_directory=True) + self.install = install + self.env = dict(os.environ, ELAN_HOME=str(elan), + ELAN_TOOLCHAIN="leanprover/lean4:v0.0.0-cache-fixture", + LEAN_SYSROOT=str(install), LEAN=str(install / "bin/lean"), + LAKE_OVERRIDE_LEAN="1", + LAKE_SHARED_CACHE_ROOT=str(self.root / "shared"), + LAKE_CACHE_DIR=str(self.root / "private"), + LAKE_SHARED_CACHE_FALLBACK=str(self.root / "private"), + LAKE_ARTIFACT_CACHE="1", LAKE_RESTORE_ARTIFACTS="1") + self.env.pop("LEAN_GITHASH", None) + self.first = self.project("first") + self.second = self.project("second") + + + def project(self, name): + project = self.root / name + project.mkdir() + (project / "lakefile.toml").write_text('name = "shared_fixture"\n[[lean_lib]]\nname = "Example"\n') + (project / "Example.lean").write_text("module\npublic def answer : Nat := 42\n") + return project + + + def run_lake(self, project, *args, env=None): + result = subprocess.run([str(self.lake), *args], cwd=project, env=env or self.env, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + self.assertEqual(result.returncode, 0, result.stdout) + return result.stdout + + + def test_explicit_sysroot_with_colocated_lake(self): + prefix = self.root / "colocated" + (prefix / "bin").mkdir(parents=True) + shutil.copy2(self.lake, prefix / "bin/lake") + compiler = self.original.parent.parent + (prefix / "bin/lean").symlink_to(compiler / "bin/lean") + (prefix / "lib").symlink_to(compiler / "lib", target_is_directory=True) + selections = [] + for override in ("0", "1"): + result = subprocess.run([str(prefix / "bin/lake"), "--shared-cache-dir"], cwd=self.first, + env=dict(self.env, LAKE_OVERRIDE_LEAN=override), + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + selections.append(result.stdout.strip()) + self.assertEqual(selections[0], selections[1]) + self.assertTrue(selections[1].startswith(str(self.root / "shared/toolchains"))) + + + def test_toolchain_revision_namespaces_are_separate(self): + first = self.run_lake(self.first, "--shared-cache-dir").strip() + install = self.install.with_name("leanprover--lean4---v0.0.1-cache-fixture") + install.mkdir() + for name in ("bin", "lib", "src", "include", "share"): + if (self.install / name).exists(): + (install / name).symlink_to(self.install / name, target_is_directory=True) + env = dict(self.env, ELAN_TOOLCHAIN="leanprover/lean4:v0.0.1-cache-fixture", + LEAN_SYSROOT=str(install), LEAN=str(install / "bin/lean")) + second = self.run_lake(self.first, "--shared-cache-dir", env=env).strip() + self.assertTrue(second) + self.assertNotEqual(first, second) + self.run_lake(self.first, "build", "Example", env=env) + + + def test_forwarding_sysroot_shares_without_bypassing_compiler_wrapper(self): + expected = self.run_lake(self.first, "--shared-cache-dir").strip() + proxy = self.root / "forwarding" + (proxy / "bin").mkdir(parents=True) + for name in ("lib", "src", "include", "share"): + if (self.install / name).exists(): + (proxy / name).symlink_to(self.install / name, target_is_directory=True) + for binary in (self.install / "bin").iterdir(): + if binary.name != "lean": + (proxy / "bin" / binary.name).symlink_to(binary) + marker = self.root / "wrapper-used" + compiler = self.original.with_name("lean") + wrapper = proxy / "bin/lean" + wrapper.write_text(f'''#!{sys.executable} +import os, pathlib, sys +if sys.argv[1:] == ["--print-prefix"]: + print({str(proxy)!r}) + raise SystemExit(0) +if any(arg.endswith("Example.lean") for arg in sys.argv[1:]): + pathlib.Path({str(marker)!r}).touch() +os.execv({str(compiler)!r}, [{str(compiler)!r}, *sys.argv[1:]]) +''') + wrapper.chmod(0o755) + env = dict(self.env, LEAN=str(wrapper)) + env.pop("LEAN_SYSROOT") + selected = self.run_lake(self.first, "--shared-cache-dir", env=env).strip() + self.assertEqual(selected, expected) + self.run_lake(self.first, "build", "Example", env=env) + self.assertTrue(marker.exists(), "cache sharing bypassed the compiler wrapper") + self.run_lake(self.second, "build", "Example") + self.assertEqual((self.first / ".lake/build/lib/lean/Example.olean").stat().st_ino, + (self.second / ".lake/build/lib/lean/Example.olean").stat().st_ino) + + + def test_linked_and_overridden_toolchains_keep_private_storage(self): + self.assertEqual(self.run_lake(self.first, "--shared-cache-dir", + env=dict(self.env, LEAN_GITHASH="local-edit")).strip(), "") + target = self.install.with_name("local-build") + self.install.rename(target) + self.install.symlink_to(target, target_is_directory=True) + self.assertEqual(self.run_lake(self.first, "--shared-cache-dir").strip(), "") + self.run_lake(self.first, "build", "Example") + + + def test_reuse_rebuild_and_private_rollback(self): + selected = self.run_lake(self.first, "--shared-cache-dir").strip() + self.assertTrue(selected.startswith(str(self.root / "shared/toolchains"))) + self.run_lake(self.first, "build", "Example") + result = self.run_lake(self.second, "build", "Example") + self.assertNotIn("Built Example", result) + first = self.first / ".lake/build/lib/lean/Example.olean" + second = self.second / ".lake/build/lib/lean/Example.olean" + self.assertEqual(first.stat().st_ino, second.stat().st_ino) + before = first.read_bytes() + (self.second / "Example.lean").write_text("module\npublic def changed : Nat := 43\n") + self.run_lake(self.second, "build", "Example") + self.assertEqual(first.read_bytes(), before) + self.assertNotEqual(second.read_bytes(), before) + self.run_lake(self.second, "build", "Example", env=dict(self.env, LAKE_SHARED_CACHE_ROOT="")) + self.assertTrue(list((self.root / "private/artifacts").glob("*.olean"))) + + + def test_old_child_retains_private_fallback(self): + if self.original.resolve() == self.lake: + self.skipTest("a distinct stock Lake executable is not available") + old = self.original + result = self.run_lake(self.first, "env", str(old), "env", "printenv", "LAKE_CACHE_DIR") + self.assertEqual(result.strip().splitlines()[-1], str(self.root / "private")) + private = self.run_lake(self.first, "--shared-cache-dir", env=dict(self.env, LAKE_RESTORE_ARTIFACTS="0")) + self.assertEqual(private.strip(), "") + override = self.run_lake(self.first, "--shared-cache-dir", env=dict(self.env, LAKE_CACHE_DIR=str(self.root / "override"))) + self.assertEqual(override.strip(), "") + (self.first / "lakefile.toml").unlink() + (self.first / "lakefile.lean").write_text( + 'import Lake\nopen Lake DSL\npackage shared_fixture\nlean_lib Example\n') + self.run_lake(self.first, "env", str(old), "build", "Example") + + + def test_cached_configuration_survives_stock_patched_alternation(self): + if self.original.resolve() == self.lake: + self.skipTest("a distinct stock Lake executable is not available") + config = '''import Lake +open Lake DSL +initialize do + let (elan, lean, lake) ← findInstall? + let env ← (Env.compute lake.get! lean.get! elan).toIO IO.userError + unless env.cacheKey? == some "abi-fixture" do + throw <| IO.userError "cached Lake.Env field layout changed" + unless env.cacheArtifactEndpoint? == some "https://fixture.invalid" do + throw <| IO.userError "cached Lake.Env endpoint field layout changed" +package shared_fixture +lean_lib Example +''' + env = dict(self.env, LAKE_CACHE_KEY="abi-fixture", + LAKE_CACHE_ARTIFACT_ENDPOINT="https://fixture.invalid") + for project, binaries in ((self.first, (self.original, self.lake, self.original)), + (self.second, (self.lake, self.original, self.lake))): + (project / "lakefile.toml").unlink() + (project / "lakefile.lean").write_text(config) + timestamp = None + for binary in binaries: + result = subprocess.run([str(binary), "env", "true"], cwd=project, env=env, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + installed, = (project / ".lake").rglob("lakefile.olean") + if timestamp is not None: + self.assertEqual(installed.stat().st_mtime_ns, timestamp) + timestamp = installed.stat().st_mtime_ns + + + def test_existing_executable_keeps_execute_permission_after_sharing(self): + source = self.root / "downloaded-executable" + target = self.root / "existing-executable" + source.write_bytes(b"#!/bin/sh\nexit 0\n") + target.write_bytes(source.read_bytes()) + target.chmod(0o755) + probe = pathlib.Path(__file__).with_name("Probe.lean").resolve() + self.run_lake(self.first, "env", "lean", "--run", str(probe), "restore-exe", + str(self.root / "exe-cache"), str(source), str(target)) + self.assertTrue(os.access(target, os.X_OK)) + canonical = next((self.root / "exe-cache/artifacts").glob("*.exe")) + self.assertEqual(target.stat().st_ino, canonical.stat().st_ino) + + + def test_up_to_date_private_build_seeds_and_joins_native_store(self): + private = dict(self.env, LAKE_SHARED_CACHE_ROOT="", LAKE_ARTIFACT_CACHE="0") + self.run_lake(self.first, "build", "Example", env=private) + self.run_lake(self.second, "build", "Example", env=private) + first = self.first / ".lake/build/lib/lean/Example.olean" + second = self.second / ".lake/build/lib/lean/Example.olean" + self.assertNotEqual(first.stat().st_ino, second.stat().st_ino) + self.run_lake(self.first, "build", "Example") + self.run_lake(self.second, "build", "Example") + self.assertEqual(first.stat().st_ino, second.stat().st_ino) + + + def test_unwritable_shared_metadata_uses_private_cache(self): + body = b"native fallback artifact" + mapping = self.root / "mapping.jsonl" + probe = pathlib.Path(__file__).with_name("Probe.lean").resolve() + self.run_lake(self.first, "env", "lean", "--run", str(probe), "map", str(mapping), body.decode()) + selected = pathlib.Path(self.run_lake(self.first, "--shared-cache-dir").strip()) + outputs = selected / "outputs" + outputs.mkdir(exist_ok=True) + def permissions(mode): + outputs.chmod(mode) + for directory in outputs.rglob("*"): + if directory.is_dir(): + directory.chmod(mode) + self.addCleanup(lambda: permissions(0o755)) + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + permissions(0o555) + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + config = self.root / "services.toml" + config.write_text('[[cache.service]]\nname = "fixture"\nkind = "s3"\n' + f'artifactEndpoint = "http://127.0.0.1:{server.server_port}"\n' + f'revisionEndpoint = "http://127.0.0.1:{server.server_port}"\n') + try: + self.run_lake(self.first, "cache", "get", str(mapping), "--service", "fixture", "--scope", "fixture", "--wfail", + env=dict(self.env, LAKE_CONFIG=str(config))) + finally: + server.shutdown() + server.server_close() + thread.join() + payloads = list((self.root / "private/artifacts").glob("*.art")) + self.assertEqual(len(payloads), 1) + self.assertEqual(payloads[0].read_bytes(), body) + self.assertEqual(self.run_lake(self.first, "--shared-cache-dir").strip(), "") + self.run_lake(self.first, "build", "Example") + + + def test_leantar_overwrite_does_not_modify_a_shared_inode(self): + env = dict(self.env, LAKE_SHARED_CACHE_ROOT="", LAKE_ARTIFACT_CACHE="0") + self.run_lake(self.first, "build", "+Example:ltar", env=env) + original = self.first / ".lake/build/lib/lean/Example.olean" + canonical = self.root / "canonical.olean" + os.link(original, canonical) + before = canonical.read_bytes() + (self.second / "Example.lean").write_text("module\npublic def changed : Nat := 43\n") + self.run_lake(self.second, "build", "+Example:ltar", env=env) + archive = self.second / ".lake/build/ir/Example.ltar" + result = subprocess.run([str(self.original.with_name("leantar")), "-f", "-x", "-j", "-"], + input=json.dumps([dict(file=str(archive), hash="0123456789abcdef", + base=[str(self.first / ".lake/build/lib/lean"), + str(self.first / ".lake/build/ir")])]), + cwd=self.first, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(canonical.read_bytes(), before) + self.assertNotEqual(original.read_bytes(), before) + self.assertNotEqual(original.stat().st_ino, canonical.stat().st_ino) + + + def test_archive_disappearance_before_unpack_is_a_recoverable_miss(self): + mapping = self.root / "outputs.jsonl" + self.run_lake(self.first, "build", "+Example", "-o", str(mapping)) + self.run_lake(self.first, "cache", "add", str(mapping)) + marker = self.root / "archive-removed" + wrapper = self.install / "bin/leantar" + wrapper.unlink() + wrapper.write_text(f'''#!{sys.executable} +import json, os, pathlib, subprocess, sys +original = {str(self.original.with_name("leantar"))!r} +if "-x" in sys.argv and "-j" in sys.argv: + data = sys.stdin.buffer.read() + for entry in json.loads(data): + path = pathlib.Path(entry["file"]) + if path.is_relative_to({str(self.root / "shared")!r}): + path.unlink() + pathlib.Path({str(marker)!r}).touch() + raise SystemExit(subprocess.run([original, *sys.argv[1:]], input=data).returncode) +os.execv(original, [original, *sys.argv[1:]]) +''') + wrapper.chmod(0o755) + self.run_lake(self.second, "build", "Example") + self.assertTrue(marker.exists(), "build did not exercise shared archive unpack") + self.assertTrue((self.second / ".lake/build/lib/lean/Example.olean").is_file()) + + + @unittest.skipUnless(sys.platform.startswith("linux") and shutil.which("cc"), "link interposition needs Linux and cc") + def test_eviction_after_lookup_is_a_recoverable_miss(self): + self.eviction_after_lookup(clear_store=False) + + + @unittest.skipUnless(sys.platform.startswith("linux") and shutil.which("cc"), "link interposition needs Linux and cc") + def test_clearing_store_during_restore_is_a_recoverable_miss(self): + self.eviction_after_lookup(clear_store=True) + + + def eviction_after_lookup(self, *, clear_store): + self.run_lake(self.first, "build", "Example") + pause = self.root / "pause" + pause.mkdir() + library = self.root / "pause.so" + compile_result = subprocess.run(["cc", "-shared", "-fPIC", "-o", str(library), + str(pathlib.Path(__file__).resolve().parent.parent / "cacheEviction/pause_link.c"), "-ldl"], capture_output=True, text=True, + env={k: v for k, v in os.environ.items() if k != "LD_LIBRARY_PATH"}) + self.assertEqual(compile_result.returncode, 0, compile_result.stderr) + env = dict(self.env, LD_PRELOAD=str(library), TEST_CACHE_PREFIX=str(self.root / "shared"), + TEST_LINK_PAUSE_DIR=str(pause)) + child = subprocess.Popen([str(self.lake), "build", "Example"], cwd=self.second, + env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + deadline = time.monotonic() + 60 + while not (pause / "source").exists() and child.poll() is None and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue((pause / "source").exists(), "restore did not enter the controlled race window") + source = pathlib.Path((pause / "source").read_text()) + if clear_store: + shutil.rmtree(self.root / "shared") + else: + source.unlink() + finally: + (pause / "release").touch() + output, _ = child.communicate(timeout=60) + self.assertEqual(child.returncode, 0, output) + self.assertTrue((self.second / ".lake/build/lib/lean/Example.olean").is_file()) + + +if __name__ == "__main__": + unittest.main()
diff --git a/tests/lake/tests/sharedCache/test.sh b/tests/lake/tests/sharedCache/test.sh new file mode 100755 index 0000000..0df4cf8 --- /dev/null +++ b/tests/lake/tests/sharedCache/test.sh
@@ -0,0 +1,3 @@ +#!/usr/bin/env bash +source ../common.sh +python3 test.py