From 94e28c730803fb0e5b302974419cedce8edcc6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Raddum=20Berg?= Date: Sat, 25 Jul 2026 02:24:00 +0200 Subject: [PATCH] remote-cache: add file:// backend for sharing compiled state between worktrees The build cache only spoke the S3 wire protocol: s3:// went to AWS and any other uri was treated as an S3-compatible HTTP endpoint, with credentials required unconditionally. Pointing the cache at a local directory - the natural setup for sharing compiled state between git worktrees on one machine - was impossible. Push/pull logic was already backend-agnostic (content-digest keys, portable zinc analysis, noop-manifest regeneration on pull), so the backend surface is three operations. Extract that as CacheStore (headObject/getObject/ putObject), implemented by the existing S3Client and a new LocalDirStore that maps keys to files under a root directory. Writes are atomic (temp file + ATOMIC_MOVE) so concurrent pushes of the same digest from different worktrees are safe, and keys are checked against escaping the root. file:// uris skip credential resolution entirely; everything else keeps the existing fail-hard requirement. The new integration test deliberately runs with the harness default config - no remoteCacheCredentials - proving the local backend never asks for them, and covers push layout, noop-manifest exclusion, skip-on-second-push, and pull restoring classes + analysis + regenerated manifest. Co-Authored-By: Claude Fable 5 --- bleep-cli/src/scala/bleep/Main.scala | 6 +- bleep-core/src/scala/bleep/CacheStore.scala | 70 ++++++++++++ bleep-core/src/scala/bleep/S3Client.scala | 2 +- .../scala/bleep/commands/RemoteCache.scala | 28 +++-- .../src/scala/bleep/LocalDirCacheIT.scala | 105 ++++++++++++++++++ docs/usage/remote-cache.mdx | 25 ++++- 6 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 bleep-core/src/scala/bleep/CacheStore.scala create mode 100644 bleep-tests/src/scala/bleep/LocalDirCacheIT.scala diff --git a/bleep-cli/src/scala/bleep/Main.scala b/bleep-cli/src/scala/bleep/Main.scala index a7cb6bb15..e9b8fcc05 100644 --- a/bleep-cli/src/scala/bleep/Main.scala +++ b/bleep-cli/src/scala/bleep/Main.scala @@ -635,12 +635,12 @@ object Main { commands.Fmt(check, projects) } }, - Opts.subcommand("remote-cache", "push and pull compiled classes to/from a remote S3-compatible cache")( + Opts.subcommand("remote-cache", "push and pull compiled classes to/from a build cache (S3-compatible service or local directory)")( List( - Opts.subcommand("pull", "pull cached compiled classes from remote cache")( + Opts.subcommand("pull", "pull cached compiled classes from the cache")( projectNames.map(names => commands.RemoteCache.Pull(names)) ), - Opts.subcommand("push", "push compiled classes to remote cache")( + Opts.subcommand("push", "push compiled classes to the cache")( (projectNames, Opts.flag("force", "overwrite existing cache entries").orFalse).mapN(commands.RemoteCache.Push.apply) ) ).foldK diff --git a/bleep-core/src/scala/bleep/CacheStore.scala b/bleep-core/src/scala/bleep/CacheStore.scala new file mode 100644 index 000000000..f14c44f67 --- /dev/null +++ b/bleep-core/src/scala/bleep/CacheStore.scala @@ -0,0 +1,70 @@ +package bleep + +import ryddig.Logger + +import java.nio.file.{Files, Path, StandardCopyOption} + +/** Storage backend for the build cache: content-addressed blobs under string keys. + * + * Implementations: [[S3Client]] (S3-compatible HTTP services) and [[LocalDirStore]] (a directory on the local filesystem, for sharing compiled state between + * checkouts/worktrees on one machine). + */ +trait CacheStore { + + /** Check if an object exists. */ + def headObject(key: String): Boolean + + /** Download an object. Throws if missing. */ + def getObject(key: String): Array[Byte] + + /** Upload an object. Throws on failure. */ + def putObject(key: String, content: Array[Byte]): Unit +} + +/** Cache backend backed by a local directory. Keys map directly to file paths under `root`. + * + * Writes are atomic: content goes to a temp file in the target directory, then an atomic move to the final name. Concurrent pushes of the same key (e.g. two + * worktrees compiling the same digest) both succeed; content is identical because keys are content digests. + */ +class LocalDirStore(logger: Logger, root: Path) extends CacheStore { + + private def pathFor(key: String): Path = { + val resolved = root.resolve(key).normalize() + if (!resolved.startsWith(root)) throw new BleepException.Text(s"Cache key '$key' escapes cache root $root") + resolved + } + + override def headObject(key: String): Boolean = + Files.isRegularFile(pathFor(key)) + + override def getObject(key: String): Array[Byte] = { + val path = pathFor(key) + if (!Files.isRegularFile(path)) throw new BleepException.Text(s"Cache object not found: $path") + Files.readAllBytes(path) + } + + override def putObject(key: String, content: Array[Byte]): Unit = { + val path = pathFor(key) + Files.createDirectories(path.getParent) + val temp = Files.createTempFile(path.getParent, s".${path.getFileName.toString}", ".tmp") + try { + Files.write(temp, content) + Files.move(temp, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) + logger.debug(s"cache PUT $path (${content.length} bytes)") + } finally Files.deleteIfExists(temp): Unit + } +} + +object LocalDirStore { + + /** Interpret a `file://` cache URI as an absolute directory. */ + def fromUri(logger: Logger, uri: java.net.URI): LocalDirStore = { + val root = + try Path.of(uri).toAbsolutePath.normalize() + catch { + case e: IllegalArgumentException => + throw new BleepException.Text(s"Invalid file cache uri '$uri': ${e.getMessage}. Expected an absolute path like file:///path/to/cache") + } + new LocalDirStore(logger, root) + } +} diff --git a/bleep-core/src/scala/bleep/S3Client.scala b/bleep-core/src/scala/bleep/S3Client.scala index ca78a1403..dfe6db361 100644 --- a/bleep-core/src/scala/bleep/S3Client.scala +++ b/bleep-core/src/scala/bleep/S3Client.scala @@ -22,7 +22,7 @@ class S3Client( endpoint: URI, accessKeyId: String, secretAccessKey: String -) { +) extends CacheStore { private val httpClient: HttpClient = HttpClient.newBuilder().build() private val service = "s3" diff --git a/bleep-core/src/scala/bleep/commands/RemoteCache.scala b/bleep-core/src/scala/bleep/commands/RemoteCache.scala index f43648829..78ba317fd 100644 --- a/bleep-core/src/scala/bleep/commands/RemoteCache.scala +++ b/bleep-core/src/scala/bleep/commands/RemoteCache.scala @@ -8,7 +8,8 @@ import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.{Executors, Future as JFuture} import scala.jdk.StreamConverters.* -/** Remote build cache: pull pre-compiled classes from S3, push after building. +/** Build cache: pull pre-compiled classes from a cache backend, push after building. Backends: S3-compatible services and local directories (see + * [[CacheStore]]). * * Cache entries are per-project tar.gz archives keyed by content digest. The digest captures project config, source content, resource content, and transitive * dependency digests. @@ -34,11 +35,13 @@ object RemoteCache { val cacheConfig = started.build.remoteCache cacheConfig match { case None => - Left(new BleepException.Text("No remote-cache configured in bleep.yaml. Add:\n remote-cache:\n uri: s3://bucket/prefix\n region: us-east-1")) + Left( + new BleepException.Text( + "No remote-cache configured in bleep.yaml. Add:\n remote-cache:\n uri: s3://bucket/prefix\n region: us-east-1\nor for a local directory (shared between checkouts/worktrees on this machine):\n remote-cache:\n uri: file:///path/to/cache" + ) + ) case Some(config) => - val credentials = resolveCredentials(started) - val client = S3Client.fromConfig(started.logger, config, credentials) - val prefix = S3Client.keyPrefix(config) + val (client, prefix) = storeFor(started, config) val digests = ProjectDigest.computeAll(started.build, started.buildPaths) val projectsToPull = if (projects.nonEmpty) projects.toSet else digests.keySet @@ -102,9 +105,7 @@ object RemoteCache { case None => Left(new BleepException.Text("No remote-cache configured in bleep.yaml")) case Some(config) => - val credentials = resolveCredentials(started) - val client = S3Client.fromConfig(started.logger, config, credentials) - val prefix = S3Client.keyPrefix(config) + val (client, prefix) = storeFor(started, config) val digests = ProjectDigest.computeAll(started.build, started.buildPaths) val projectsToPush = if (projects.nonEmpty) projects.toSet else digests.keySet @@ -177,6 +178,17 @@ object RemoteCache { } } + /** Select the cache backend from the configured uri scheme. + * + * `file://` is a directory on the local filesystem — no credentials, the uri path is the cache root and the key prefix is empty. Anything else goes through + * [[S3Client]] (s3:// or an S3-compatible HTTP endpoint) and requires credentials. + */ + private def storeFor(started: Started, config: model.RemoteCacheConfig): (CacheStore, String) = + config.uri.getScheme match { + case "file" => (LocalDirStore.fromUri(started.logger, config.uri), "") + case _ => (S3Client.fromConfig(started.logger, config, resolveCredentials(started)), S3Client.keyPrefix(config)) + } + private def cacheKey(prefix: String, crossName: model.CrossProjectName, digest: String): String = { val projectKey = crossName.value.replace('/', '-') if (prefix.isEmpty) s"$projectKey/$digest.tar.gz" diff --git a/bleep-tests/src/scala/bleep/LocalDirCacheIT.scala b/bleep-tests/src/scala/bleep/LocalDirCacheIT.scala new file mode 100644 index 000000000..e7d0b9a1f --- /dev/null +++ b/bleep-tests/src/scala/bleep/LocalDirCacheIT.scala @@ -0,0 +1,105 @@ +package bleep + +import bleep.analysis.NoopManifestStore +import bleep.commands.RemoteCache +import bleep.internal.FileUtils + +import java.nio.file.{Files, Path} +import scala.jdk.StreamConverters.* + +/** The `file://` cache backend: same push/pull semantics as the S3 backend, but keys are files under a local directory and no credentials are required. + * + * Deliberately does NOT override `testConfig`: the harness default has no `remoteCacheCredentials`, so these tests prove the local backend never asks for them + * (the S3 path fails hard without credentials). + */ +class LocalDirCacheIT extends IntegrationTestHarness { + + private def listRelativeFiles(dir: Path): List[String] = + if (!Files.isDirectory(dir)) Nil + else + scala.util + .Using(Files.walk(dir)) { stream => + stream + .toScala(List) + .filter(Files.isRegularFile(_)) + .map(p => dir.relativize(p).toString.replace('\\', '/')) + .sorted + } + .getOrElse(Nil) + + integrationTest("local dir cache: push writes archive, pull restores classes + regenerates manifest, no credentials involved") { ws => + val cacheDir = Files.createTempDirectory("bleep-local-cache-") + try { + ws.yaml( + s"""remote-cache: + | uri: ${cacheDir.toUri} + | + |projects: + | greeter: + | platform: + | name: jvm + | scala: + | version: 3.3.3 + |""".stripMargin + ) + ws.file( + "greeter/src/scala/com/test/Greeter.scala", + """package com.test + |object Greeter { def hello: String = "hi" } + |""".stripMargin + ) + + val (started, _, _) = ws.start() + val greeter = model.CrossProjectName(model.ProjectName("greeter"), None) + ws.compileAll() + + val projectPaths = started.projectPaths(greeter) + val analysisFile = projectPaths.targetDir.resolve(".zinc/analysis.zip") + val classFile = projectPaths.classes.resolve("com/test/Greeter.class") + val noopManifest = NoopManifestStore.manifestPath(analysisFile) + + assert(Files.exists(classFile), s"compile should have produced $classFile") + assert(Files.exists(analysisFile), s"compile should have produced $analysisFile") + + // === PUSH === + RemoteCache.Push(projects = Array.empty, force = false).run(started).fold(e => fail(s"push failed: ${e.getMessage}"), identity) + + // Cache dir should contain exactly one object: /.tar.gz (no prefix for file backend). + val cached = listRelativeFiles(cacheDir) + assert(cached.size == 1, s"expected 1 cached object, got $cached") + val cacheKey = cached.head + assert(cacheKey.startsWith("greeter/"), s"unexpected cache key: $cacheKey") + assert(cacheKey.endsWith(".tar.gz"), s"unexpected cache key extension: $cacheKey") + assert(!cached.exists(_.endsWith(".tmp")), s"temp files must not survive an atomic put: $cached") + + // Archive must ship classes + analysis but never the per-machine noop manifest. + val unpackDir = Files.createTempDirectory("bleep-local-cache-inspect-") + try { + TarGz.unpack(Files.readAllBytes(cacheDir.resolve(cacheKey)), unpackDir) + val archived = listRelativeFiles(unpackDir) + assert(archived.exists(_.endsWith("Greeter.class")), s"archive missing classes: $archived") + assert(archived.exists(_ == ".zinc/analysis.zip"), s"archive missing analysis: $archived") + assert(!archived.exists(_.endsWith("noop-manifest.bin")), s"noop-manifest.bin must not be shipped, got $archived") + } finally FileUtils.deleteDirectory(unpackDir) + + // Second push without --force skips: headObject sees the existing file. + RemoteCache.Push(projects = Array.empty, force = false).run(started).fold(e => fail(s"second push failed: ${e.getMessage}"), identity) + assert(listRelativeFiles(cacheDir) == cached, "second push should not have written new keys") + + // === WIPE LOCAL, PULL === + FileUtils.deleteDirectory(projectPaths.targetDir) + assert(!Files.exists(classFile), "wipe should have removed local classes") + + RemoteCache.Pull(projects = Array.empty).run(started).fold(e => fail(s"pull failed: ${e.getMessage}"), identity) + + assert(Files.exists(classFile), s"pull should have restored $classFile") + assert(Files.exists(analysisFile), s"pull should have restored $analysisFile") + if (NoopManifestStore.ctimeAvailable) { + assert(Files.exists(noopManifest), s"pull should have regenerated $noopManifest") + val loaded = NoopManifestStore.load(analysisFile).getOrElse(fail("regenerated manifest must load")) + assert(loaded.cachedResult.outputDir == projectPaths.classes, s"manifest outputDir wrong: ${loaded.cachedResult.outputDir}") + } + succeed + } finally FileUtils.deleteDirectory(cacheDir) + } +} diff --git a/docs/usage/remote-cache.mdx b/docs/usage/remote-cache.mdx index 330e94252..04a17767e 100644 --- a/docs/usage/remote-cache.mdx +++ b/docs/usage/remote-cache.mdx @@ -1,10 +1,10 @@ --- -title: Remote build cache +title: Build cache --- -# Remote build cache +# Build cache -Bleep can cache compiled classes in S3 (or any S3-compatible storage like MinIO, Cloudflare R2) so CI pipelines skip compilation for unchanged projects. +Bleep can cache compiled classes in S3 (or any S3-compatible storage like MinIO, Cloudflare R2) so CI pipelines skip compilation for unchanged projects, or in a local directory so multiple checkouts and git worktrees on one machine share compiled state. ## Setup @@ -106,6 +106,25 @@ build: - bleep remote-cache push ``` +## Local directory cache + +Point the cache at a directory instead of a bucket: + +```yaml +remote-cache: + uri: file:///Users/me/.cache/my-project-bleep-cache +``` + +No credentials or region are needed. Cache entries land at `//.tar.gz`, written atomically so concurrent pushes from different checkouts are safe. + +This is built for git worktrees: because cache keys are content digests and the shipped zinc analysis is path-portable, a freshly created worktree can `bleep remote-cache pull` and skip compiling everything its sibling already built: + +```bash +git worktree add ../feature-x +cd ../feature-x +bleep remote-cache pull # restores classes + analysis for all unchanged projects +``` + ## S3-compatible services The remote cache works with any S3-compatible service: