remote-cache: add file:// backend for sharing compiled state between worktrees

bleep-cli/src/scala/bleep/Main.scala

@@ -635,12 +635,12 @@ object Main {
               commands.Fmt(check, projects)
             }
           },
-          Opts.subcommand("remote-cache", "push and pull compiled classes to/from a remote S3-compatible cache")(
+          Opts.subcommand("remote-cache", "push and pull compiled classes to/from a build cache (S3-compatible service or local directory)")(
             List(
-              Opts.subcommand("pull", "pull cached compiled classes from remote cache")(
+              Opts.subcommand("pull", "pull cached compiled classes from the cache")(
                 projectNames.map(names => commands.RemoteCache.Pull(names))
               ),
-              Opts.subcommand("push", "push compiled classes to remote cache")(
+              Opts.subcommand("push", "push compiled classes to the cache")(
                 (projectNames, Opts.flag("force", "overwrite existing cache entries").orFalse).mapN(commands.RemoteCache.Push.apply)
               )
             ).foldK

bleep-core/src/scala/bleep/CacheStore.scala

@@ -0,0 +1,70 @@
+package bleep
+
+import ryddig.Logger
+
+import java.nio.file.{Files, Path, StandardCopyOption}
+
+/** Storage backend for the build cache: content-addressed blobs under string keys.
+  *
+  * Implementations: [[S3Client]] (S3-compatible HTTP services) and [[LocalDirStore]] (a directory on the local filesystem, for sharing compiled state between
+  * checkouts/worktrees on one machine).
+  */
+trait CacheStore {
+
+  /** Check if an object exists. */
+  def headObject(key: String): Boolean
+
+  /** Download an object. Throws if missing. */
+  def getObject(key: String): Array[Byte]
+
+  /** Upload an object. Throws on failure. */
+  def putObject(key: String, content: Array[Byte]): Unit
+}
+
+/** Cache backend backed by a local directory. Keys map directly to file paths under `root`.
+  *
+  * Writes are atomic: content goes to a temp file in the target directory, then an atomic move to the final name. Concurrent pushes of the same key (e.g. two
+  * worktrees compiling the same digest) both succeed; content is identical because keys are content digests.
+  */
+class LocalDirStore(logger: Logger, root: Path) extends CacheStore {
+
+  private def pathFor(key: String): Path = {
+    val resolved = root.resolve(key).normalize()
+    if (!resolved.startsWith(root)) throw new BleepException.Text(s"Cache key '$key' escapes cache root $root")
+    resolved
+  }
+
+  override def headObject(key: String): Boolean =
+    Files.isRegularFile(pathFor(key))
+
+  override def getObject(key: String): Array[Byte] = {
+    val path = pathFor(key)
+    if (!Files.isRegularFile(path)) throw new BleepException.Text(s"Cache object not found: $path")
+    Files.readAllBytes(path)
+  }
+
+  override def putObject(key: String, content: Array[Byte]): Unit = {
+    val path = pathFor(key)
+    Files.createDirectories(path.getParent)
+    val temp = Files.createTempFile(path.getParent, s".${path.getFileName.toString}", ".tmp")
+    try {
+      Files.write(temp, content)
+      Files.move(temp, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)
+      logger.debug(s"cache PUT $path (${content.length} bytes)")
+    } finally Files.deleteIfExists(temp): Unit
+  }
+}
+
+object LocalDirStore {
+
+  /** Interpret a `file://` cache URI as an absolute directory. */
+  def fromUri(logger: Logger, uri: java.net.URI): LocalDirStore = {
+    val root =
+      try Path.of(uri).toAbsolutePath.normalize()
+      catch {
+        case e: IllegalArgumentException =>
+          throw new BleepException.Text(s"Invalid file cache uri '$uri': ${e.getMessage}. Expected an absolute path like file:///path/to/cache")
+      }
+    new LocalDirStore(logger, root)
+  }
+}

bleep-core/src/scala/bleep/S3Client.scala

@@ -22,7 +22,7 @@ class S3Client(
     endpoint: URI,
     accessKeyId: String,
     secretAccessKey: String
-) {
+) extends CacheStore {
   private val httpClient: HttpClient = HttpClient.newBuilder().build()
   private val service = "s3"
 

bleep-core/src/scala/bleep/commands/RemoteCache.scala

@@ -8,7 +8,8 @@ import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.{Executors, Future as JFuture}
 import scala.jdk.StreamConverters.*
 
-/** Remote build cache: pull pre-compiled classes from S3, push after building.
+/** Build cache: pull pre-compiled classes from a cache backend, push after building. Backends: S3-compatible services and local directories (see
+  * [[CacheStore]]).
   *
   * Cache entries are per-project tar.gz archives keyed by content digest. The digest captures project config, source content, resource content, and transitive
   * dependency digests.
@@ -34,11 +35,13 @@ object RemoteCache {
       val cacheConfig = started.build.remoteCache
       cacheConfig match {
         case None =>
-          Left(new BleepException.Text("No remote-cache configured in bleep.yaml. Add:\n  remote-cache:\n    uri: s3://bucket/prefix\n    region: us-east-1"))
+          Left(
+            new BleepException.Text(
+              "No remote-cache configured in bleep.yaml. Add:\n  remote-cache:\n    uri: s3://bucket/prefix\n    region: us-east-1\nor for a local directory (shared between checkouts/worktrees on this machine):\n  remote-cache:\n    uri: file:///path/to/cache"
+            )
+          )
         case Some(config) =>
-          val credentials = resolveCredentials(started)
-          val client = S3Client.fromConfig(started.logger, config, credentials)
-          val prefix = S3Client.keyPrefix(config)
+          val (client, prefix) = storeFor(started, config)
 
           val digests = ProjectDigest.computeAll(started.build, started.buildPaths)
           val projectsToPull = if (projects.nonEmpty) projects.toSet else digests.keySet
@@ -102,9 +105,7 @@ object RemoteCache {
         case None =>
           Left(new BleepException.Text("No remote-cache configured in bleep.yaml"))
         case Some(config) =>
-          val credentials = resolveCredentials(started)
-          val client = S3Client.fromConfig(started.logger, config, credentials)
-          val prefix = S3Client.keyPrefix(config)
+          val (client, prefix) = storeFor(started, config)
 
           val digests = ProjectDigest.computeAll(started.build, started.buildPaths)
           val projectsToPush = if (projects.nonEmpty) projects.toSet else digests.keySet
@@ -177,6 +178,17 @@ object RemoteCache {
     }
   }
 
+  /** Select the cache backend from the configured uri scheme.
+    *
+    * `file://` is a directory on the local filesystem — no credentials, the uri path is the cache root and the key prefix is empty. Anything else goes through
+    * [[S3Client]] (s3:// or an S3-compatible HTTP endpoint) and requires credentials.
+    */
+  private def storeFor(started: Started, config: model.RemoteCacheConfig): (CacheStore, String) =
+    config.uri.getScheme match {
+      case "file" => (LocalDirStore.fromUri(started.logger, config.uri), "")
+      case _      => (S3Client.fromConfig(started.logger, config, resolveCredentials(started)), S3Client.keyPrefix(config))
+    }
+
   private def cacheKey(prefix: String, crossName: model.CrossProjectName, digest: String): String = {
     val projectKey = crossName.value.replace('/', '-')
     if (prefix.isEmpty) s"$projectKey/$digest.tar.gz"

bleep-tests/src/scala/bleep/LocalDirCacheIT.scala

@@ -0,0 +1,105 @@
+package bleep
+
+import bleep.analysis.NoopManifestStore
+import bleep.commands.RemoteCache
+import bleep.internal.FileUtils
+
+import java.nio.file.{Files, Path}
+import scala.jdk.StreamConverters.*
+
+/** The `file://` cache backend: same push/pull semantics as the S3 backend, but keys are files under a local directory and no credentials are required.
+  *
+  * Deliberately does NOT override `testConfig`: the harness default has no `remoteCacheCredentials`, so these tests prove the local backend never asks for them
+  * (the S3 path fails hard without credentials).
+  */
+class LocalDirCacheIT extends IntegrationTestHarness {
+
+  private def listRelativeFiles(dir: Path): List[String] =
+    if (!Files.isDirectory(dir)) Nil
+    else
+      scala.util
+        .Using(Files.walk(dir)) { stream =>
+          stream
+            .toScala(List)
+            .filter(Files.isRegularFile(_))
+            .map(p => dir.relativize(p).toString.replace('\\', '/'))
+            .sorted
+        }
+        .getOrElse(Nil)
+
+  integrationTest("local dir cache: push writes archive, pull restores classes + regenerates manifest, no credentials involved") { ws =>
+    val cacheDir = Files.createTempDirectory("bleep-local-cache-")
+    try {
+      ws.yaml(
+        s"""remote-cache:
+           |  uri: ${cacheDir.toUri}
+           |
+           |projects:
+           |  greeter:
+           |    platform:
+           |      name: jvm
+           |    scala:
+           |      version: 3.3.3
+           |""".stripMargin
+      )
+      ws.file(
+        "greeter/src/scala/com/test/Greeter.scala",
+        """package com.test
+          |object Greeter { def hello: String = "hi" }
+          |""".stripMargin
+      )
+
+      val (started, _, _) = ws.start()
+      val greeter = model.CrossProjectName(model.ProjectName("greeter"), None)
+      ws.compileAll()
+
+      val projectPaths = started.projectPaths(greeter)
+      val analysisFile = projectPaths.targetDir.resolve(".zinc/analysis.zip")
+      val classFile = projectPaths.classes.resolve("com/test/Greeter.class")
+      val noopManifest = NoopManifestStore.manifestPath(analysisFile)
+
+      assert(Files.exists(classFile), s"compile should have produced $classFile")
+      assert(Files.exists(analysisFile), s"compile should have produced $analysisFile")
+
+      // === PUSH ===
+      RemoteCache.Push(projects = Array.empty, force = false).run(started).fold(e => fail(s"push failed: ${e.getMessage}"), identity)
+
+      // Cache dir should contain exactly one object: <project>/<digest>.tar.gz (no prefix for file backend).
+      val cached = listRelativeFiles(cacheDir)
+      assert(cached.size == 1, s"expected 1 cached object, got $cached")
+      val cacheKey = cached.head
+      assert(cacheKey.startsWith("greeter/"), s"unexpected cache key: $cacheKey")
+      assert(cacheKey.endsWith(".tar.gz"), s"unexpected cache key extension: $cacheKey")
+      assert(!cached.exists(_.endsWith(".tmp")), s"temp files must not survive an atomic put: $cached")
+
+      // Archive must ship classes + analysis but never the per-machine noop manifest.
+      val unpackDir = Files.createTempDirectory("bleep-local-cache-inspect-")
+      try {
+        TarGz.unpack(Files.readAllBytes(cacheDir.resolve(cacheKey)), unpackDir)
+        val archived = listRelativeFiles(unpackDir)
+        assert(archived.exists(_.endsWith("Greeter.class")), s"archive missing classes: $archived")
+        assert(archived.exists(_ == ".zinc/analysis.zip"), s"archive missing analysis: $archived")
+        assert(!archived.exists(_.endsWith("noop-manifest.bin")), s"noop-manifest.bin must not be shipped, got $archived")
+      } finally FileUtils.deleteDirectory(unpackDir)
+
+      // Second push without --force skips: headObject sees the existing file.
+      RemoteCache.Push(projects = Array.empty, force = false).run(started).fold(e => fail(s"second push failed: ${e.getMessage}"), identity)
+      assert(listRelativeFiles(cacheDir) == cached, "second push should not have written new keys")
+
+      // === WIPE LOCAL, PULL ===
+      FileUtils.deleteDirectory(projectPaths.targetDir)
+      assert(!Files.exists(classFile), "wipe should have removed local classes")
+
+      RemoteCache.Pull(projects = Array.empty).run(started).fold(e => fail(s"pull failed: ${e.getMessage}"), identity)
+
+      assert(Files.exists(classFile), s"pull should have restored $classFile")
+      assert(Files.exists(analysisFile), s"pull should have restored $analysisFile")
+      if (NoopManifestStore.ctimeAvailable) {
+        assert(Files.exists(noopManifest), s"pull should have regenerated $noopManifest")
+        val loaded = NoopManifestStore.load(analysisFile).getOrElse(fail("regenerated manifest must load"))
+        assert(loaded.cachedResult.outputDir == projectPaths.classes, s"manifest outputDir wrong: ${loaded.cachedResult.outputDir}")
+      }
+      succeed
+    } finally FileUtils.deleteDirectory(cacheDir)
+  }
+}

docs/usage/remote-cache.mdx

@@ -1,10 +1,10 @@
 ---
-title: Remote build cache
+title: Build cache
 ---
 
-# Remote build cache
+# Build cache
 
-Bleep can cache compiled classes in S3 (or any S3-compatible storage like MinIO, Cloudflare R2) so CI pipelines skip compilation for unchanged projects.
+Bleep can cache compiled classes in S3 (or any S3-compatible storage like MinIO, Cloudflare R2) so CI pipelines skip compilation for unchanged projects, or in a local directory so multiple checkouts and git worktrees on one machine share compiled state.
 
 ## Setup
 
@@ -106,6 +106,25 @@ build:
     - bleep remote-cache push
 ```
 
+## Local directory cache
+
+Point the cache at a directory instead of a bucket:
+
+```yaml
+remote-cache:
+  uri: file:///Users/me/.cache/my-project-bleep-cache
+```
+
+No credentials or region are needed. Cache entries land at `<dir>/<project>/<digest>.tar.gz`, written atomically so concurrent pushes from different checkouts are safe.
+
+This is built for git worktrees: because cache keys are content digests and the shipped zinc analysis is path-portable, a freshly created worktree can `bleep remote-cache pull` and skip compiling everything its sibling already built:
+
+```bash
+git worktree add ../feature-x
+cd ../feature-x
+bleep remote-cache pull   # restores classes + analysis for all unchanged projects
+```
+
 ## S3-compatible services
 
 The remote cache works with any S3-compatible service: