diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75084f126..afd0bccdf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,15 @@ concurrency: jobs: build: - timeout-minutes: 15 + # Measured, not guessed: this job does sourcegen, a full compile, publish-local AND the whole + # test suite. On master it takes 12-14 minutes against a 15-minute cap — 82-92% of budget, so it + # was one slow runner away from red before anything changed. The zinc 2 upgrade and per-analysis + # interning pushed it over: two consecutive runs were cancelled at exactly 15m17s and 15m18s. + # + # Not hiding a hang. The log shows suites finishing 4 seconds apart right up to the cut, three + # running concurrently — steady throughput into the wall, not a stall. The native-image jobs + # already get 40 minutes for less work than this. + timeout-minutes: 30 # need to be newer than `ubuntu-20.04` because of scalafmt native binary runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'ci skip')" diff --git a/bleep-bsp-tests/src/scala/bleep/MachineResourcesTest.scala b/bleep-bsp-tests/src/scala/bleep/MachineResourcesTest.scala index 3436d64c1..a6bc9a77f 100644 --- a/bleep-bsp-tests/src/scala/bleep/MachineResourcesTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/MachineResourcesTest.scala @@ -19,6 +19,24 @@ class MachineResourcesTest extends AnyFunSuite with Matchers { private def machine(cpu: Int, memMb: Long): MachineResources = MachineResources.create(totalCpu = cpu, totalMemoryMb = memMb, logger = TypedLogger.DevNull, longWaitWarnMs = 200L) + test("activeCompiles is what the heap gate reads, and counts every client's compiles") { + // Per connection this figure was always 1 for a client running a single compile, so a dozen + // such clients each concluded they were alone and skipped the stagger entirely. + val m = machine(cpu = 8, memMb = 8192) + val prog = for { + running <- CountDownLatch[IO](3) + release <- CountDownLatch[IO](1) + held <- List("a", "b", "c").parTraverse(n => m.reserve(Compile, n, cpu = 1, memoryMb = 0L).use(_ => running.release *> release.await).start) + _ <- running.await + n <- m.activeCompiles + _ <- IO(n shouldBe 3) + _ <- release.release + _ <- held.traverse_(_.join) + afterCount <- m.activeCompiles + } yield afterCount shouldBe 0 + prog.timeout(20.seconds).unsafeRunSync() + } + test("a reservation within budget is granted immediately and released") { val m = machine(cpu = 4, memMb = 8192) val prog = for { diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/AnalysisCacheOwnershipTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/AnalysisCacheOwnershipTest.scala new file mode 100644 index 000000000..3349ece4b --- /dev/null +++ b/bleep-bsp-tests/src/scala/bleep/analysis/AnalysisCacheOwnershipTest.scala @@ -0,0 +1,95 @@ +package bleep.analysis + +import bleep.model +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import xsbti.compile.CompileAnalysis +import xsbti.compile.analysis.{ReadCompilations, ReadSourceInfos, ReadStamps} + +import java.nio.file.{Files, Path} + +/** Analyses belong to a workspace, and the consequences of that: they are partitioned by it and dropped with it. + * + * The point of the ownership is not tidiness. An unowned global pool could neither say which build was holding the heap nor free it when that build was + * evicted — so a `BuildCache` eviction freed the resolved build (hundreds of MB) and left its analyses (gigabytes) behind. + */ +class AnalysisCacheOwnershipTest extends AnyFunSuite with Matchers { + + /** The cache stores analyses and reports their file size; it never looks inside one. */ + private object StubAnalysis extends CompileAnalysis { + def readStamps(): ReadStamps = ??? + def readSourceInfos(): ReadSourceInfos = ??? + def readCompilations(): ReadCompilations = ??? + } + + private def key(name: String): model.WorkspaceKey = + model.WorkspaceKey(Path.of(s"/tmp/$name"), model.BuildVariant.Normal) + + /** A real file, because `put` sizes the entry from disk. */ + private def analysisFile(bytes: Int): Path = { + val f = Files.createTempFile("analysis", ".zip") + Files.write(f, Array.fill(bytes)(0: Byte)) + f.toFile.deleteOnExit() + f + } + + private def cache(): AnalysisCache = new AnalysisCache + + test("analyses are partitioned by workspace: one workspace cannot see another's") { + val c = cache() + val f = analysisFile(64) + val mtime = Files.getLastModifiedTime(f).toMillis + c.put(key("alpha"), f, mtime, StubAnalysis): Unit + + c.get(key("alpha"), f, mtime) shouldBe defined + // Same analysis FILE, different workspace. Nothing is shared across workspaces on disk, and + // nothing is shared here either — otherwise one workspace's entries would be charged to another. + c.get(key("beta"), f, mtime) shouldBe empty + } + + test("a changed file on disk invalidates the entry rather than serving a stale analysis") { + val c = cache() + val f = analysisFile(64) + val mtime = Files.getLastModifiedTime(f).toMillis + c.put(key("alpha"), f, mtime, StubAnalysis): Unit + + c.get(key("alpha"), f, mtime) shouldBe defined + c.get(key("alpha"), f, mtime + 1) shouldBe empty // e.g. after `remote-cache pull` + } + + test("evicting a workspace frees only its own analyses, and reports what it freed") { + val c = cache() + val a1 = analysisFile(100) + val a2 = analysisFile(200) + val b1 = analysisFile(400) + c.put(key("alpha"), a1, Files.getLastModifiedTime(a1).toMillis, StubAnalysis): Unit + c.put(key("alpha"), a2, Files.getLastModifiedTime(a2).toMillis, StubAnalysis): Unit + c.put(key("beta"), b1, Files.getLastModifiedTime(b1).toMillis, StubAnalysis): Unit + + val freed = c.evictWorkspace(key("alpha")) + freed.entries shouldBe 2 + freed.fileBytes shouldBe 300L + + c.get(key("alpha"), a1, Files.getLastModifiedTime(a1).toMillis) shouldBe empty + c.get(key("beta"), b1, Files.getLastModifiedTime(b1).toMillis) shouldBe defined + } + + test("evicting a workspace that holds nothing is a no-op, not an error") { + val c = cache() + c.evictWorkspace(key("never-seen")) shouldBe AnalysisCache.Freed(0, 0L) + } + + test("stats total across workspaces and are ordered by what they hold") { + val c = cache() + val small = analysisFile(10) + val large = analysisFile(900) + c.put(key("small-ws"), small, Files.getLastModifiedTime(small).toMillis, StubAnalysis): Unit + c.put(key("large-ws"), large, Files.getLastModifiedTime(large).toMillis, StubAnalysis): Unit + + val stats = c.stats + stats.entries shouldBe 2 + stats.fileBytes shouldBe 910L + // Biggest first: the whole point of per-workspace stats is answering "who is holding the heap". + stats.perWorkspace.head.key shouldBe key("large-ws") + } +} diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/AnalysisDedupMeasurementTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/AnalysisDedupMeasurementTest.scala new file mode 100644 index 000000000..f5e753e06 --- /dev/null +++ b/bleep-bsp-tests/src/scala/bleep/analysis/AnalysisDedupMeasurementTest.scala @@ -0,0 +1,134 @@ +package bleep.analysis + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import xsbti.api.AnalyzedClass + +import java.nio.file.{Files, Path, Paths} +import java.security.MessageDigest +import scala.jdk.CollectionConverters.* +import scala.jdk.OptionConverters.* + +/** How much of the compile server's heap would a leaf interner give back? + * + * Not a unit test of behaviour — a measurement, kept in the suite so the number can be reproduced rather than quoted from a session that has scrolled away. It + * loads real `analysis.zip` files and reports how many `AnalyzedClass` instances are duplicates of each other. + * + * ==Why AnalyzedClass is the number that matters== + * + * A class histogram of a live daemon at 7.2GB live set found ~4.5GB in `xsbti.api.*`: 31.7M `NameHash` (1.0GB), 31.1M `Id` (498MB), 7.4M `PathComponent[]` + * (428MB), 839K `AnalyzedClass`, 839K `NameHash[]` (267MB). `AnalyzedClass` count matches `NameHash[]` count exactly, and each one owns the lazy `Companions` + * tree holding all of the above — so the duplication factor measured here is the factor by which that 4.5GB shrinks. + * + * ==Reading the output== + * + * `distinct` counts by the proposed intern key — everything `AnalyzedClass` carries EXCEPT `compilationTimestamp`, which zinc 1.12 reads in one place as a + * fast-path gate that falls back to a full structural diff, and which zinc 2.0.0 stopped reading altogether. The run reports both keys so the cost of + * including the timestamp is visible rather than assumed: with it, two worktrees that compiled the same code independently do not merge. + * + * Point it at more workspaces with `BLEEP_DEDUP_ROOTS` (colon-separated workspace roots). With none set it measures this build alone, which shows only the + * within-workspace sharing (library stamps, shared dependencies) and not the cross-worktree case that motivates the work. + */ +class AnalysisDedupMeasurementTest extends AnyFunSuite with Matchers { + + /** ScalaTest's `info` is swallowed by the runner's reporter, so the measurement is written where it can be read afterwards. */ + private val report = Paths.get(sys.props.getOrElse("user.dir", ".")).resolve("dedup-report.txt").toAbsolutePath + private val lines = scala.collection.mutable.ListBuffer.empty[String] + private def say(s: String): Unit = { lines += s; println(s"[dedup] $s") } + private def flush(): Unit = Files.write(report, (s"report: $report\n" + lines.mkString("\n")).getBytes("UTF-8")): Unit + + private def analysisFilesUnder(root: Path): List[Path] = + if (!Files.isDirectory(root)) Nil + else { + val stream = Files.walk(root) + try stream.iterator().asScala.filter(p => p.getFileName.toString == "analysis.zip" && Files.isRegularFile(p)).toList + finally stream.close() + } + + /** Roots to measure, one per line in `dedup-roots.txt` beside the build, else this build's own `.bleep`. + * + * A file rather than an env var because the test runs in a forked JVM that does not inherit one. + */ + private def roots: List[Path] = { + val cwd = Paths.get(sys.props.getOrElse("user.dir", ".")) + val listing = cwd.resolve("dedup-roots.txt") + if (Files.isRegularFile(listing)) + Files.readAllLines(listing).asScala.toList.map(_.trim).filter(_.nonEmpty).map(Paths.get(_)) + else List(cwd.resolve(".bleep")) + } + + /** The proposed intern key. `withTimestamp` shows what including `compilationTimestamp` would cost in hit rate. */ + private def internKey(ac: AnalyzedClass, withTimestamp: Boolean): String = { + val md = MessageDigest.getInstance("SHA-256") + md.update(ac.name().getBytes("UTF-8")) + md.update(java.nio.ByteBuffer.allocate(4).putInt(ac.apiHash()).array()) + md.update(java.nio.ByteBuffer.allocate(4).putInt(ac.extraHash()).array()) + md.update(if (ac.hasMacro()) Array[Byte](1) else Array[Byte](0)) + md.update(ac.provenance().getBytes("UTF-8")) + // nameHashes is an array of (name, scope, hash) — digest it rather than holding it in the key. + ac.nameHashes().sortBy(nh => (nh.name(), nh.scope().ordinal())).foreach { nh => + md.update(nh.name().getBytes("UTF-8")) + md.update(java.nio.ByteBuffer.allocate(4).putInt(nh.scope().ordinal()).array()) + md.update(java.nio.ByteBuffer.allocate(4).putInt(nh.hash()).array()) + } + if (withTimestamp) md.update(java.nio.ByteBuffer.allocate(8).putLong(ac.compilationTimestamp()).array()) + md.digest().map("%02x".format(_)).mkString + } + + test("MEASUREMENT: duplicate AnalyzedClass across real analysis files") { + val files = roots.flatMap(analysisFilesUnder) + if (files.isEmpty) { + say(s"no analysis.zip found under ${roots.mkString(", ")} — compile something first, or set BLEEP_DEDUP_ROOTS") + flush() + cancel("nothing to measure") + } + + val totalBytes = files.map(Files.size).sum + say(f"${files.size}%d analysis files, ${totalBytes / (1024.0 * 1024)}%.1f MB on disk, across ${roots.size}%d root(s)") + + var total = 0L + val distinctNoTs = scala.collection.mutable.HashSet.empty[String] + val distinctWithTs = scala.collection.mutable.HashSet.empty[String] + var unreadable = 0 + + files.foreach { f => + // The same store production uses, so this measures what bleep actually writes. Files left + // over from zinc 1.x are in the old protobuf format and simply do not read here — they count + // as unreadable and the run reports them. + val store = sbt.internal.inc.consistent.ConsistentFileAnalysisStore.binary( + f.toFile, + ZincBridge.analysisMappers(f), + reproducible = true, + parallelism = 4 + ) + store.get().toScala match { + case None => unreadable += 1 + case Some(contents) => + contents.getAnalysis match { + case a: sbt.internal.inc.Analysis => + val apis = a.apis + (apis.internal.values ++ apis.external.values).foreach { ac => + total += 1 + distinctNoTs += internKey(ac, withTimestamp = false) + distinctWithTs += internKey(ac, withTimestamp = true) + } + case other => say(s"unexpected analysis type ${other.getClass.getName}") + } + } + } + + if (unreadable > 0) say(s"$unreadable file(s) could not be read") + + val ratioNoTs = if (distinctNoTs.isEmpty) 0.0 else total.toDouble / distinctNoTs.size + val ratioWithTs = if (distinctWithTs.isEmpty) 0.0 else total.toDouble / distinctWithTs.size + say(f"AnalyzedClass total=$total%d distinct(no timestamp)=${distinctNoTs.size}%d → ${ratioNoTs}%.2fx sharing") + say(f"AnalyzedClass total=$total%d distinct(with timestamp)=${distinctWithTs.size}%d → ${ratioWithTs}%.2fx sharing") + say(f"cost of keeping timestamp in the key: ${distinctWithTs.size - distinctNoTs.size}%d extra retained instances") + + flush() + // A measurement with no data is not a failure — a fresh checkout, or the window right after an + // analysis-format change, legitimately has nothing readable. Say so and skip. + if (total == 0L) cancel(s"no readable analyses (${files.size} file(s) found, $unreadable unreadable)") + distinctNoTs.size should be <= total.toInt + } +} diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/BspTestHarness.scala b/bleep-bsp-tests/src/scala/bleep/analysis/BspTestHarness.scala index 662c1090f..9e9b6ab8a 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/BspTestHarness.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/BspTestHarness.scala @@ -199,6 +199,7 @@ class BspTestHarness(workspaceRoot: Path, projectConfigs: Option[List[BspTestHar // The production server. It has no way to be handed build state directly — it compiles the // build its client sends — so the configs are lowered into the same payload a real bleep // client would send, and delivered through build/initialize below. + val harnessAnalysisCache = new bleep.analysis.AnalysisCache val server = new MultiWorkspaceBspServer( serverInput, serverToClient, @@ -207,7 +208,8 @@ class BspTestHarness(workspaceRoot: Path, projectConfigs: Option[List[BspTestHar heapMonitor = HeapMonitor.system, // One server per harness, so fresh daemon-scoped state is the right scope here. kspMutexes = new KspMutexes, - buildCache = new BuildCache + buildCache = new BuildCache(bleep.model.BspServerConfig.default.effectiveMaxCachedWorkspaces, harnessAnalysisCache), + analysisCache = harnessAnalysisCache ) val buildPayload: Option[BspBuildData.Payload] = diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/CompilerIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/CompilerIntegrationTest.scala index a080bab76..39290852f 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/CompilerIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/CompilerIntegrationTest.scala @@ -1506,7 +1506,8 @@ class CompilerVersionIsolationTest extends AnyFunSuite with Matchers { DiagnosticListener.noop, CancellationToken.never, Map.empty, - ProgressListener.noop + ProgressListener.noop, + AnalysisCache.standalone(config.buildDir) ) .unsafeRunSync() diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/IncrementalCompilationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/IncrementalCompilationTest.scala index 1e79f77df..35f2e280e 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/IncrementalCompilationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/IncrementalCompilationTest.scala @@ -205,7 +205,9 @@ class IncrementalTrackingTest extends AnyFunSuite with Matchers { diagnosticListener = listener, cancellationToken = CancellationToken.never, dependencyAnalyses = Map.empty, - progressListener = ProgressListener.noop + progressListener = ProgressListener.noop, + ecjVersion = None, + analyses = AnalysisCache.standalone(config.buildDir) ) .unsafeRunSync() @@ -265,7 +267,9 @@ class IncrementalTrackingTest extends AnyFunSuite with Matchers { diagnosticListener = listener1, cancellationToken = CancellationToken.never, dependencyAnalyses = Map.empty, - progressListener = ProgressListener.noop + progressListener = ProgressListener.noop, + ecjVersion = None, + analyses = AnalysisCache.standalone(config.buildDir) ) .unsafeRunSync() @@ -287,7 +291,9 @@ class IncrementalTrackingTest extends AnyFunSuite with Matchers { diagnosticListener = listener2, cancellationToken = CancellationToken.never, dependencyAnalyses = Map.empty, - progressListener = ProgressListener.noop + progressListener = ProgressListener.noop, + ecjVersion = None, + analyses = AnalysisCache.standalone(config.buildDir) ) .unsafeRunSync() diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala index ae46ad9fb..b5893dc33 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala @@ -20,6 +20,10 @@ import org.scalatest.matchers.should.Matchers */ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { + /** A machine for tests: enough CPU for the parallelism a case asks for, ample fork memory so admission never turns on it. */ + private def testMachine(cpu: Int): bleep.MachineResources = + bleep.MachineResources.create(totalCpu = cpu, totalMemoryMb = 64 * 1024, logger = ryddig.TypedLogger.DevNull, longWaitWarnMs = 60000L) + /** Helper to create CrossProjectName from a simple string. */ private def projectName(name: String): CrossProjectName = CrossProjectName(ProjectName(name), None) @@ -264,7 +268,7 @@ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { val result = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - finalDag <- executor.execute(dag, 4, eventQueue, killSignal) + finalDag <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield finalDag).unsafeRunSync() linkCalled shouldBe true @@ -309,7 +313,7 @@ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { val events = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - _ <- executor.execute(dag, 4, eventQueue, killSignal) + _ <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) allEvents <- drainQueue(eventQueue) } yield allEvents).unsafeRunSync() @@ -354,7 +358,7 @@ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { val result = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - finalDag <- executor.execute(dag, 4, eventQueue, killSignal) + finalDag <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield finalDag).unsafeRunSync() result.failed should contain(TaskId.Link(project)) diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/PlatformTestHelper.scala b/bleep-bsp-tests/src/scala/bleep/analysis/PlatformTestHelper.scala index 5537f752a..cecf9ccc2 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/PlatformTestHelper.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/PlatformTestHelper.scala @@ -144,7 +144,9 @@ trait PlatformTestHelper { DiagnosticListener.noop, CancellationToken.never, Map.empty, - ProgressListener.noop + ProgressListener.noop, + None, + AnalysisCache.standalone(config.buildDir) ) .unsafeRunSync() @@ -213,7 +215,9 @@ trait PlatformTestHelper { DiagnosticListener.noop, CancellationToken.never, Map.empty, - ProgressListener.noop + ProgressListener.noop, + None, + AnalysisCache.standalone(config.buildDir) ) .unsafeRunSync() diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala index f5ea59574..3ba20fdb0 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala @@ -32,6 +32,10 @@ import scala.jdk.CollectionConverters.* */ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { + /** A machine for tests: enough CPU for the parallelism a case asks for, ample fork memory so admission never turns on it. */ + private def testMachine(cpu: Int): bleep.MachineResources = + bleep.MachineResources.create(totalCpu = cpu, totalMemoryMb = 64 * 1024, logger = ryddig.TypedLogger.DevNull, longWaitWarnMs = 60000L) + private def projectName(name: String): CrossProjectName = CrossProjectName(ProjectName(name), None) @@ -306,7 +310,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - _ <- executor.execute(dag, 4, eventQueue, killSignal) + _ <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield ()).unsafeRunSync() val events = order.asScala.toList @@ -355,7 +359,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val finalDag = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - d <- executor.execute(dag, 4, eventQueue, killSignal) + d <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield d).unsafeRunSync() sourcegenCalled.get() shouldBe false @@ -404,7 +408,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val finalDag = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - d <- executor.execute(dag, 4, eventQueue, killSignal) + d <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield d).unsafeRunSync() targetCompileCalled.get() shouldBe false @@ -452,7 +456,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val finalDag = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - d <- executor.execute(dag, 4, eventQueue, killSignal) + d <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield d).unsafeRunSync() targetCompileCalled.get() shouldBe true @@ -498,7 +502,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val events = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - _ <- executor.execute(dag, 4, eventQueue, killSignal) + _ <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) _ <- eventQueue.offer(None) drained <- drainQueue(eventQueue) } yield drained).unsafeRunSync() @@ -550,7 +554,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val events = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - _ <- executor.execute(dag, 4, eventQueue, killSignal) + _ <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) _ <- eventQueue.offer(None) drained <- drainQueue(eventQueue) } yield drained).unsafeRunSync() @@ -601,7 +605,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Deferred[IO, KillReason] _ <- (IO.sleep(scala.concurrent.duration.DurationInt(50).millis) >> killSignal.complete(KillReason.UserRequest)).start - d <- executor.execute(dag, 4, eventQueue, killSignal) + d <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield d).unsafeRunSync() targetCompileCalled.get() shouldBe false @@ -650,7 +654,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - _ <- executor.execute(dag, 4, eventQueue, killSignal) + _ <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) } yield ()).unsafeRunSync() val tags = timeline.asScala.toList.map(_._1) @@ -693,7 +697,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val (finalDag, events) = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - d <- executor.execute(dag, 4, eventQueue, killSignal) + d <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) _ <- eventQueue.offer(None) drained <- drainQueue(eventQueue) } yield (d, drained)).unsafeRunSync() @@ -745,7 +749,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { val (finalDag, events) = (for { eventQueue <- Queue.unbounded[IO, Option[DagEvent]] killSignal <- Outcome.neverKillSignal - d <- executor.execute(dag, 4, eventQueue, killSignal) + d <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) _ <- eventQueue.offer(None) drained <- drainQueue(eventQueue) } yield (d, drained)).unsafeRunSync() diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala index d6914ff93..201684144 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala @@ -30,6 +30,10 @@ import scala.jdk.CollectionConverters.* */ class SymbolProcessorDagIntegrationTest extends AnyFunSuite with Matchers { + /** A machine for tests: enough CPU for the parallelism a case asks for, ample fork memory so admission never turns on it. */ + private def testMachine(cpu: Int): bleep.MachineResources = + bleep.MachineResources.create(totalCpu = cpu, totalMemoryMb = 64 * 1024, logger = ryddig.TypedLogger.DevNull, longWaitWarnMs = 60000L) + private def projectName(name: String): CrossProjectName = CrossProjectName(ProjectName(name), None) @@ -111,7 +115,7 @@ class SymbolProcessorDagIntegrationTest extends AnyFunSuite with Matchers { symbolProcessor = (kspt, _) => IO { timeline.add(s"ksp:${kspt.project.value}"); (TaskResult.Success, 2) } ) executor = TaskDag.executor(handlers) - _ <- executor.execute(dag, maxParallelism = 4, eventQueue, killSignal).flatTap(_ => eventQueue.offer(None)) + _ <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal).flatTap(_ => eventQueue.offer(None)) } yield () program.unsafeRunSync() @@ -143,7 +147,7 @@ class SymbolProcessorDagIntegrationTest extends AnyFunSuite with Matchers { symbolProcessor = (_, _) => IO((TaskResult.Failure("simulated KSP misconfig", Nil), 0)) ) executor = TaskDag.executor(handlers) - finalDag <- executor.execute(dag, maxParallelism = 4, eventQueue, killSignal).flatTap(_ => eventQueue.offer(None)) + finalDag <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal).flatTap(_ => eventQueue.offer(None)) } yield finalDag val finalDag = program.unsafeRunSync() diff --git a/bleep-bsp-tests/src/scala/bleep/bsp/BuildCacheEvictionTest.scala b/bleep-bsp-tests/src/scala/bleep/bsp/BuildCacheEvictionTest.scala new file mode 100644 index 000000000..1b97219bc --- /dev/null +++ b/bleep-bsp-tests/src/scala/bleep/bsp/BuildCacheEvictionTest.scala @@ -0,0 +1,54 @@ +package bleep.bsp + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +/** The eviction policy that bounds how much resolved-build state one daemon retains. + * + * Written against the pure decision function rather than the cache, because what is worth pinning down is the policy — LRU order, the two exemptions, and what + * happens when they conflict with the bound — and exercising it through the cache would mean resolving real builds to observe it. + */ +class BuildCacheEvictionTest extends AnyFunSuite with Matchers { + + private def select(present: Vector[(String, Long)], keep: String, bound: Int, busy: Set[String] = Set.empty): Vector[String] = + BuildCache.selectEvictions(present, keep, bound, busy.contains).map(_._1) + + test("under the bound nothing is evicted") { + select(Vector("a" -> 1L, "b" -> 2L), keep = "b", bound = 4) shouldBe empty + } + + test("at the bound nothing is evicted — the bound is a ceiling, not a target") { + select(Vector("a" -> 1L, "b" -> 2L, "c" -> 3L), keep = "c", bound = 3) shouldBe empty + } + + test("over the bound the least recently used go first, and only as many as the overage") { + // Five entries, bound of three: the two oldest idle ones go, the rest stay. + val present = Vector("oldest" -> 10L, "older" -> 20L, "middle" -> 30L, "recent" -> 40L, "newest" -> 50L) + select(present, keep = "newest", bound = 3) shouldBe Vector("oldest", "older") + } + + test("the entry just loaded is never evicted, even when it is the least recently used of them all") { + // A pathological clock reading for the new entry must not make it the victim: it was loaded + // because someone is about to use it, and evicting it would mean reloading it immediately. + val present = Vector("fresh" -> 0L, "a" -> 10L, "b" -> 20L, "c" -> 30L) + select(present, keep = "fresh", bound = 2) shouldBe Vector("a", "b") + } + + test("a workspace with operations in flight is never evicted, even when it is the oldest") { + val present = Vector("busy" -> 1L, "idle1" -> 2L, "idle2" -> 3L, "keep" -> 4L) + select(present, keep = "keep", bound = 2, busy = Set("busy")) shouldBe Vector("idle1", "idle2") + } + + test("when too many entries are busy the cache exceeds its bound rather than evicting live work") { + // Four entries, bound of one, and everything but the kept entry is compiling. Nothing is + // evictable, so the bound yields. A daemon can serve more workspaces at once than it caches. + val present = Vector("busy1" -> 1L, "busy2" -> 2L, "busy3" -> 3L, "keep" -> 4L) + select(present, keep = "keep", bound = 1, busy = Set("busy1", "busy2", "busy3")) shouldBe empty + } + + test("busy entries still count towards the bound, so idle ones are evicted to make room for them") { + // Three busy + two idle, bound of three. The overage is two, and both idle entries pay it. + val present = Vector("busy1" -> 1L, "busy2" -> 2L, "busy3" -> 3L, "idle1" -> 4L, "keep" -> 5L) + select(present, keep = "keep", bound = 3, busy = Set("busy1", "busy2", "busy3")) shouldBe Vector("idle1") + } +} diff --git a/bleep-bsp-tests/src/scala/bleep/bsp/ServerLogRetentionTest.scala b/bleep-bsp-tests/src/scala/bleep/bsp/ServerLogRetentionTest.scala new file mode 100644 index 000000000..214acf7ed --- /dev/null +++ b/bleep-bsp-tests/src/scala/bleep/bsp/ServerLogRetentionTest.scala @@ -0,0 +1,96 @@ +package bleep.bsp + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} + +/** What survives a crash, and what a socket directory is allowed to cost. + * + * The motivating failure: a client retries a crashed server once, so a systematic crash produces `crash → restart → crash` and two rotations within seconds. + * With one generation of history the second rotation overwrote the first crash's log — and `cleanup` deleted the live one outright before the restart, so a + * crash that was not an OutOfMemoryError left nothing at all. Reported from the field as "crashed twice … no OutOfMemoryError recorded", log already replaced. + */ +class ServerLogRetentionTest extends AnyFunSuite with Matchers { + + private def socketDir(): Path = { + val d = Files.createTempDirectory("bleep-socket") + d.toFile.deleteOnExit() + d + } + + private def write(p: Path, content: String): Unit = Files.write(p, content.getBytes("UTF-8")): Unit + private def read(p: Path): String = new String(Files.readAllBytes(p), "UTF-8") + + /** startServer's rotation, reached the way the server reaches it. */ + private def rotate(dir: Path): Unit = { + val m = BspServerOperations.getClass.getDeclaredMethod("rotateOutput", classOf[Path]) + m.setAccessible(true) + m.invoke(BspServerOperations, dir): Unit + } + + test("cleanup keeps the log — a crash must not delete its own evidence") { + val dir = socketDir() + write(dir.resolve("output"), "the crash trace") + write(dir.resolve("pid"), "12345") + write(dir.resolve("lock"), "") + + BspServerOperations.cleanup(dir).unsafeRunSyncCompat() + + // runtime state goes + Files.exists(dir.resolve("pid")) shouldBe false + Files.exists(dir.resolve("lock")) shouldBe false + // evidence stays + read(dir.resolve("output")) shouldBe "the crash trace" + } + + test("the original crash survives the retry that was meant to diagnose it") { + val dir = socketDir() + // boot 1 crashes + write(dir.resolve("output"), "CRASH ONE — the interesting one") + // client restarts: rotation happens at spawn + rotate(dir) + write(dir.resolve("output"), "boot 2, short and uninformative") + // boot 2 crashes too; client restarts again before giving up + rotate(dir) + + val kept = (1 to BspServerOperations.OutputGenerationsKept).map(n => dir.resolve(s"output.$n")).filter(Files.exists(_)).map(read) + kept should contain("CRASH ONE — the interesting one") + } + + test("history is bounded: generations beyond the kept count are dropped") { + val dir = socketDir() + (1 to BspServerOperations.OutputGenerationsKept + 3).foreach { i => + write(dir.resolve("output"), s"boot $i") + rotate(dir) + } + val present = (1 to 20).map(n => dir.resolve(s"output.$n")).count(Files.exists(_)) + present should be <= BspServerOperations.OutputGenerationsKept + } + + test("the legacy single-generation name is cleaned up rather than left to rot") { + val dir = socketDir() + write(dir.resolve("output.prev"), "written by an older bleep") + write(dir.resolve("output"), "current") + rotate(dir) + Files.exists(dir.resolve("output.prev")) shouldBe false + } + + test("retained logs stay within the byte budget") { + val dir = socketDir() + val big = "x" * (BspServerOperations.MaxRetainedLogBytes.toInt / 2 + 1024) + (1 to BspServerOperations.OutputGenerationsKept).foreach { _ => + write(dir.resolve("output"), big) + rotate(dir) + } + val retained = (1 to BspServerOperations.OutputGenerationsKept).map(n => dir.resolve(s"output.$n")).filter(Files.exists(_)).map(Files.size).sum + retained should be <= BspServerOperations.MaxRetainedLogBytes + } + + extension (io: cats.effect.IO[Unit]) { + def unsafeRunSyncCompat(): Unit = { + import cats.effect.unsafe.implicits.global + io.unsafeRunSync() + } + } +} diff --git a/bleep-bsp/src/scala/bleep/analysis/AnalysisCache.scala b/bleep-bsp/src/scala/bleep/analysis/AnalysisCache.scala new file mode 100644 index 000000000..2ebce5d49 --- /dev/null +++ b/bleep-bsp/src/scala/bleep/analysis/AnalysisCache.scala @@ -0,0 +1,223 @@ +package bleep.analysis + +import bleep.model +import xsbti.api.AnalyzedClass +import xsbti.compile.CompileAnalysis + +import java.lang.ref.{ReferenceQueue, WeakReference} +import java.nio.file.{Files, Path} +import java.security.MessageDigest +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.{AtomicLong, LongAdder} +import scala.jdk.CollectionConverters.* + +/** Zinc analyses read while compiling, owned by the workspace they belong to and shared structurally between workspaces. + * + * ==Retention== + * + * An analysis lives exactly as long as its workspace's entry in [[bleep.bsp.BuildCache]]. There is no second retention policy: no size budget, no idle + * timeout, no sweep. An earlier version had all three, on the assumption that holding one workspace's analyses was expensive enough to need its own eviction + * schedule. Interning removes that assumption — see below — so the cache is now a plain index whose lifetime is the workspace's. + * + * ==Why interning== + * + * A class histogram of a live daemon at 7.2GB live set found ~4.5GB in `xsbti.api.*`: 31.7M `NameHash` (1.0GB), 31.1M `Id` (498MB), 7.4M `PathComponent[]` + * (428MB), 839K `AnalyzedClass`, 839K `NameHash[]` (267MB). A forced full GC moved the live set by 68MB, so it was retained, not garbage. + * + * `AnalyzedClass` is 1:1 with `NameHash[]` and owns the lazy `Companions` tree containing all of the above, so interning that one type shares everything + * beneath it. Measured against real builds: 2.06x sharing within one dlab workspace, 2.96x within bleep's own, and — the control — a second copy of an + * identical workspace adds exactly zero distinct instances. Divergent branches share little; a freshly forked worktree shares everything. + * + * ==Why the key omits compilationTimestamp== + * + * Zinc 1.12 read it in exactly one place, as a fast-path gate that falls back to a full structural diff when it differs; zinc 2 removed the read entirely. Two + * worktrees that compiled the same code independently get different timestamps, and including it cost 156,691 extra retained instances across six measured + * worktrees against 94 within a single one. `ConsistentAnalysisFormat(reproducible = true)`, now in use, normalises them anyway. + * + * ==Safety of merging== + * + * A false merge needs two classes with the same name, apiHash, extraHash and nameHashes but different APIs — the state in which zinc's own invalidation is + * already broken, since those hashes are what it compares. So this adds no failure mode zinc does not already have. + */ +class AnalysisCache { + + private case class Entry(mtime: Long, analysis: CompileAnalysis, fileBytes: Long, lastUsedMs: AtomicLong) + + private val byWorkspace = new ConcurrentHashMap[model.WorkspaceKey, ConcurrentHashMap[Path, Entry]]() + + /** The interner: an index, not a cache. + * + * Values are weakly referenced, so an interned `AnalyzedClass` lives exactly as long as some loaded analysis still points at it. Nothing here retains + * anything; dropping a workspace drops its analyses, and whatever they alone held becomes collectable. Stale keys are expunged through the queue on each + * intern, so the map does not accumulate tombstones for classes nobody references any more. + */ + private val interned = new ConcurrentHashMap[String, InternRef]() + private val internQueue = new ReferenceQueue[AnalyzedClass]() + private val internHits = new LongAdder() + private val internMisses = new LongAdder() + + private final class InternRef(val key: String, value: AnalyzedClass) extends WeakReference[AnalyzedClass](value, internQueue) + + private def expungeStaleInterned(): Unit = { + var ref = internQueue.poll() + while (ref != null) { + ref match { + case ir: InternRef => interned.remove(ir.key, ir): Unit + case _ => () + } + ref = internQueue.poll() + } + } + + /** The shared instance for this `AnalyzedClass` — `ac` itself if it is the first of its content seen. */ + private def internOne(ac: AnalyzedClass): AnalyzedClass = { + val key = AnalysisCache.internKey(ac) + var result: AnalyzedClass = null + while (result == null) { + val existing = interned.get(key) + val alive = if (existing == null) null else existing.get() + if (alive != null) { + internHits.increment() + result = alive + } else { + val fresh = new InternRef(key, ac) + val won = + if (existing == null) interned.putIfAbsent(key, fresh) == null + else interned.replace(key, existing, fresh) + if (won) { + internMisses.increment() + result = ac + } + // lost the race — loop and take whatever the winner installed + } + } + result + } + + /** Rebuild an analysis so its `AnalyzedClass` values are the shared instances. + * + * Only `apis` is substituted. Stamps and relations are keyed by `VirtualFileRef`s whose ids are already workspace-neutral (`${BASE}/…`), but their spines + * are per-analysis maps that would have to be rebuilt wholesale for a much smaller return; `apis` is where the measured bytes are. + * + * The un-interned original becomes garbage as soon as the caller drops it, which is why this returns a new analysis rather than mutating. + */ + private def internAnalysis(analysis: CompileAnalysis): CompileAnalysis = + analysis match { + case a: sbt.internal.inc.Analysis => + expungeStaleInterned() + val apis = a.apis + val internal = apis.internal.map { case (name, ac) => (name, internOne(ac)) } + val external = apis.external.map { case (name, ac) => (name, internOne(ac)) } + a.copy(a.stamps, sbt.internal.inc.APIs(internal, external), a.relations, a.infos, a.compilations) + // Any other implementation is left alone rather than guessed at: interning is an optimisation, + // and an analysis we cannot rebuild faithfully is one we should hand back untouched. + case other => other + } + + private def bucket(key: model.WorkspaceKey): ConcurrentHashMap[Path, Entry] = + byWorkspace.computeIfAbsent(key, _ => new ConcurrentHashMap[Path, Entry]()) + + /** The cached analysis for `analysisFile`, if held and the file has not changed underneath it. + * + * The mtime check is what makes a `remote-cache pull` (or any other out-of-band rewrite) safe: a changed file invalidates the entry rather than serving + * something that no longer describes what is on disk. + */ + def get(key: model.WorkspaceKey, analysisFile: Path, currentMtime: Long): Option[CompileAnalysis] = + Option(byWorkspace.get(key)).flatMap(b => Option(b.get(analysisFile))) match { + case Some(entry) if entry.mtime == currentMtime => + entry.lastUsedMs.set(System.currentTimeMillis()) + Some(entry.analysis) + case _ => None + } + + /** Intern and store, returning the instance to use. Callers must use the RETURN value, not what they passed in — that is the whole point. */ + def put(key: model.WorkspaceKey, analysisFile: Path, mtime: Long, analysis: CompileAnalysis): CompileAnalysis = { + val shared = internAnalysis(analysis) + val bytes = + try Files.size(analysisFile) + catch { case _: Exception => 0L } + bucket(key).put(analysisFile, Entry(mtime, shared, bytes, new AtomicLong(System.currentTimeMillis()))): Unit + shared + } + + /** Forget one analysis — used when the file on disk is deleted or found corrupt, so the next read starts from what is actually there. */ + def invalidate(key: model.WorkspaceKey, analysisFile: Path): Unit = + Option(byWorkspace.get(key)).foreach(_.remove(analysisFile): Unit) + + /** Drop everything held for one workspace, returning what was freed. + * + * Called when that workspace's build leaves `BuildCache`. Interned classes it shared with other workspaces stay alive through those; ones only it referenced + * become collectable, and their keys leave the interner via the reference queue on the next intern. + */ + def evictWorkspace(key: model.WorkspaceKey): AnalysisCache.Freed = + Option(byWorkspace.remove(key)) match { + case Some(b) => AnalysisCache.Freed(entries = b.size(), fileBytes = b.values().iterator().asScala.map(_.fileBytes).sum) + case None => AnalysisCache.Freed(0, 0L) + } + + /** What is held right now, for telemetry. Purely observational — nothing here drives eviction. */ + def stats: AnalysisCache.Stats = { + val per = byWorkspace + .entrySet() + .iterator() + .asScala + .map { e => + val bytes = e.getValue.values().iterator().asScala.map(_.fileBytes).sum + AnalysisCache.WorkspaceStats(e.getKey, e.getValue.size(), bytes) + } + .toList + .sortBy(-_.fileBytes) + AnalysisCache.Stats(per, internedClasses = interned.size(), internHits = internHits.sum(), internMisses = internMisses.sum()) + } +} + +object AnalysisCache { + + /** The cache bound to the one workspace a given compile may touch. + * + * Every call inside a compile needs both the cache and the key, and passing them separately makes every call site an opportunity to pass the wrong one — + * which would charge one workspace's analyses to another and, worse, serve them across workspaces. Binding them once, where the workspace is known, makes + * that unrepresentable. + */ + case class Ref(cache: AnalysisCache, workspace: model.WorkspaceKey) { + def get(analysisFile: Path, currentMtime: Long): Option[CompileAnalysis] = cache.get(workspace, analysisFile, currentMtime) + def put(analysisFile: Path, mtime: Long, analysis: CompileAnalysis): CompileAnalysis = cache.put(workspace, analysisFile, mtime, analysis) + def invalidate(analysisFile: Path): Unit = cache.invalidate(workspace, analysisFile) + } + + /** A cache for a compilation belonging to no workspace — a standalone single-file compile, or a DAG built outside a BSP session. It gets its own instance, so + * whatever it holds dies with the call instead of accumulating in a bucket nothing owns. + */ + def standalone(buildDir: Path): Ref = + Ref(new AnalysisCache, model.WorkspaceKey(buildDir, model.BuildVariant.Normal)) + + case class Freed(entries: Int, fileBytes: Long) + case class WorkspaceStats(key: model.WorkspaceKey, entries: Int, fileBytes: Long) + case class Stats(perWorkspace: List[WorkspaceStats], internedClasses: Int, internHits: Long, internMisses: Long) { + def entries: Int = perWorkspace.map(_.entries).sum + def fileBytes: Long = perWorkspace.map(_.fileBytes).sum + + /** How many `AnalyzedClass` instances the daemon would be holding without interning, per instance it actually holds. 1.0 means nothing was shared. */ + def sharingFactor: Double = if (internedClasses == 0) 0.0 else (internHits + internMisses).toDouble / internedClasses + } + + /** Content key for an `AnalyzedClass`: everything it carries except `compilationTimestamp`. + * + * `nameHashes` is digested rather than held, and sorted first because zinc does not promise an order. + */ + private[analysis] def internKey(ac: AnalyzedClass): String = { + val md = MessageDigest.getInstance("SHA-256") + def int(i: Int): Unit = md.update(java.nio.ByteBuffer.allocate(4).putInt(i).array()) + md.update(ac.name().getBytes("UTF-8")) + int(ac.apiHash()) + int(ac.extraHash()) + md.update(if (ac.hasMacro()) Array[Byte](1) else Array[Byte](0)) + md.update(ac.provenance().getBytes("UTF-8")) + ac.nameHashes().sortBy(nh => (nh.name(), nh.scope().ordinal())).foreach { nh => + md.update(nh.name().getBytes("UTF-8")) + int(nh.scope().ordinal()) + int(nh.hash()) + } + md.digest().map("%02x".format(_)).mkString + } +} diff --git a/bleep-bsp/src/scala/bleep/analysis/Compiler.scala b/bleep-bsp/src/scala/bleep/analysis/Compiler.scala index 6b6b5982a..ea07bd2c7 100644 --- a/bleep-bsp/src/scala/bleep/analysis/Compiler.scala +++ b/bleep-bsp/src/scala/bleep/analysis/Compiler.scala @@ -374,7 +374,10 @@ object Compiler { listener, cancellation, Map.empty, // No dependency analyses - progressListener + progressListener, + // Standalone compile: no workspace, no analysis persistence, so the cache lives and + // dies with this call rather than lingering on a daemon. + AnalysisCache.standalone(projectConfig.buildDir) ) .unsafeRunSync() diff --git a/bleep-bsp/src/scala/bleep/analysis/ParallelProjectCompiler.scala b/bleep-bsp/src/scala/bleep/analysis/ParallelProjectCompiler.scala index 8aff41e40..768474046 100644 --- a/bleep-bsp/src/scala/bleep/analysis/ParallelProjectCompiler.scala +++ b/bleep-bsp/src/scala/bleep/analysis/ParallelProjectCompiler.scala @@ -261,7 +261,7 @@ object ParallelProjectCompiler { // Compile with appropriate compiler, passing dependency analyses val compiler = ProjectCompiler.forLanguage(config.language) - compiler.compile(config, diagnosticListener, cancellationToken, depAnalyses, ProgressListener.noop) + compiler.compile(config, diagnosticListener, cancellationToken, depAnalyses, ProgressListener.noop, AnalysisCache.standalone(config.buildDir)) } } } yield result diff --git a/bleep-bsp/src/scala/bleep/analysis/ProjectCompiler.scala b/bleep-bsp/src/scala/bleep/analysis/ProjectCompiler.scala index 4a7469321..e5ec61e3c 100644 --- a/bleep-bsp/src/scala/bleep/analysis/ProjectCompiler.scala +++ b/bleep-bsp/src/scala/bleep/analysis/ProjectCompiler.scala @@ -39,6 +39,9 @@ trait ProjectCompiler { * analysis files from dependent projects (keyed by output dir) * @param progressListener * receives compilation progress updates + * @param analyses + * the Zinc analysis cache, already bound to the workspace being compiled. Only the Zinc-backed compilers read it; the Kotlin ones keep their own + * incremental state and ignore it. * @return * compilation result (success or failure with errors) */ @@ -47,7 +50,8 @@ trait ProjectCompiler { diagnosticListener: DiagnosticListener, cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], - progressListener: ProgressListener + progressListener: ProgressListener, + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] /** Check if this compiler can handle the given language mode */ @@ -84,11 +88,12 @@ object ZincProjectCompiler extends ProjectCompiler { diagnosticListener: DiagnosticListener, cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], - progressListener: ProgressListener + progressListener: ProgressListener, + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] = config.language match { case sl: ProjectLanguage.ScalaJava => - ZincBridge.compile(config, sl, diagnosticListener, cancellationToken, dependencyAnalyses, progressListener) + ZincBridge.compile(config, sl, diagnosticListener, cancellationToken, dependencyAnalyses, progressListener, ecjVersion = None, analyses = analyses) case other => IO.raiseError(new IllegalArgumentException(s"ZincProjectCompiler cannot compile $other")) } @@ -111,7 +116,8 @@ object KotlinProjectCompiler extends ProjectCompiler { diagnosticListener: DiagnosticListener, cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], - progressListener: ProgressListener + progressListener: ProgressListener, + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] = // Kotlin compiler doesn't have progress callbacks - ignore progressListener config.language match { @@ -360,7 +366,8 @@ object JavacProjectCompiler extends ProjectCompiler { diagnosticListener: DiagnosticListener, cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], - progressListener: ProgressListener + progressListener: ProgressListener, + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] = config.language match { case javaLang: ProjectLanguage.JavaOnly => @@ -380,7 +387,8 @@ object JavacProjectCompiler extends ProjectCompiler { cancellationToken, dependencyAnalyses, progressListener, - ecjVersion = javaLang.ecjVersion + ecjVersion = javaLang.ecjVersion, + analyses = analyses ) case other => IO.raiseError(new IllegalArgumentException(s"JavacProjectCompiler cannot compile $other")) @@ -402,7 +410,8 @@ object KotlinJsProjectCompiler extends ProjectCompiler { diagnosticListener: DiagnosticListener, cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], - progressListener: ProgressListener + progressListener: ProgressListener, + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] = config.language match { case kt: ProjectLanguage.KotlinJs => @@ -605,7 +614,8 @@ object KotlinNativeProjectCompiler extends ProjectCompiler { diagnosticListener: DiagnosticListener, cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], - progressListener: ProgressListener + progressListener: ProgressListener, + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] = config.language match { case kt: ProjectLanguage.KotlinNative => diff --git a/bleep-bsp/src/scala/bleep/analysis/ZincBridge.scala b/bleep-bsp/src/scala/bleep/analysis/ZincBridge.scala index 98acc5560..5ff8637b4 100644 --- a/bleep-bsp/src/scala/bleep/analysis/ZincBridge.scala +++ b/bleep-bsp/src/scala/bleep/analysis/ZincBridge.scala @@ -3,7 +3,6 @@ package bleep.analysis import bleep.bsp.protocol.KillReason import cats.effect.IO import java.io.{File, IOException} -import java.lang.ref.SoftReference import java.nio.file.{Files, Path, StandardCopyOption, StandardOpenOption} import java.util.Optional import scala.jdk.CollectionConverters.* @@ -92,12 +91,29 @@ object ZincBridge { */ private val ecjJarCache = new java.util.concurrent.ConcurrentHashMap[String, Seq[Path]]() - /** Soft-reference cache for dependency analyses. Entries are evicted under memory pressure. Key: analysis file path. Value: SoftReference to (mtime, - * analysis). The mtime is checked before returning a cached entry — if the file on disk has changed (e.g. after `remote-cache pull`), the stale cache entry - * is discarded and the file is re-read. This avoids re-reading 10-50MB analysis files from disk when the same dependency is loaded by multiple projects - * within the same build, while allowing GC to reclaim them when heap is tight. + /** Threads each analysis read/write may use. + * + * Zinc defaults this to `availableProcessors()`, which assumes one build at a time. This daemon runs up to `parallelism` operations at once, each loading + * several dependency analyses, so the default would multiply out to hundreds of threads competing for the same disk — the oversubscription the machine + * governor exists to prevent. Analysis I/O is not the bottleneck a compile waits on; four is enough to overlap decompression with parsing. + */ + private val AnalysisIoParallelism: Int = 4 + + /** The analysis store, in zinc 2's consistent format. + * + * `reproducible = true` (zinc's own default) writes byte-identical files for identical inputs, normalising the timestamps that would otherwise make two + * worktrees' analyses differ despite describing the same code. That is what makes content-level dedup possible at all — see [[AnalysisCache]]. + * + * `file` and `analysisFile` differ on the write path, which serialises to a temp file and renames; the mappers must still be built from the real + * destination, since they relativise against its build directory. */ - private val analysisCache = new java.util.concurrent.ConcurrentHashMap[Path, SoftReference[(Long, CompileAnalysis)]]() + private def analysisStore(file: Path, analysisFile: Path): xsbti.compile.AnalysisStore = + sbt.internal.inc.consistent.ConsistentFileAnalysisStore.binary( + file.toFile, + analysisMappers(analysisFile), + reproducible = true, + parallelism = AnalysisIoParallelism + ) /** Create ReadWriteMappers for portable zinc analysis. * @@ -217,7 +233,8 @@ object ZincBridge { cancellationToken: CancellationToken, dependencyAnalyses: Map[Path, Path], progressListener: ProgressListener, - ecjVersion: Option[String] = None + ecjVersion: Option[String], + analyses: AnalysisCache.Ref ): IO[ProjectCompileResult] = IO.interruptible { debug(s"[ZincBridge] compile() called for ${config.name}") Files.createDirectories(config.outputDir) @@ -230,7 +247,19 @@ object ZincBridge { if (sources.isEmpty) { ProjectCompileSuccess(config.outputDir, Set.empty, Some(analysisFile)) } else { - compileOnce(config, sources, language, diagnosticListener, cancellationToken, dependencyAnalyses, progressListener, ecjVersion, analysisFile) + // Allocation is measured HERE, around the body of the `IO.interruptible`, because + // `getThreadAllocatedBytes` is per thread and this block is the one place a compile is + // pinned to one. Measured at the BSP layer instead — around the IO rather than inside it — + // the fiber had usually moved between start and end, and attribution succeeded for 1% of + // 34k compiles, all of them the trivial ones that never yielded. + val begin = bleep.bsp.BspMetrics.threadAllocatedBytes() + val startMs = System.currentTimeMillis() + val result = + compileOnce(config, sources, language, diagnosticListener, cancellationToken, dependencyAnalyses, progressListener, ecjVersion, analysisFile, analyses) + val end = bleep.bsp.BspMetrics.threadAllocatedBytes() + if (begin >= 0 && end >= begin) + bleep.bsp.BspMetrics.recordCompileAllocation(config.name, end - begin, System.currentTimeMillis() - startMs) + result } } @@ -460,7 +489,8 @@ object ZincBridge { dependencyAnalyses: Map[Path, Path], progressListener: ProgressListener, ecjVersion: Option[String], - analysisFile: Path + analysisFile: Path, + analyses: AnalysisCache.Ref ): ProjectCompileResult = { // Fast path: check noop manifest BEFORE any Zinc work (loading analysis, // creating compilers, hashing files). This skips ~5s of FarmHash I/O per @@ -526,7 +556,8 @@ object ZincBridge { dependencyAnalyses, progressListener, cancellationToken, - diagnosticListener + diagnosticListener, + analyses ) val compiler = incrementalCompiler @@ -591,7 +622,7 @@ object ZincBridge { if (result.hasModified || !hasPrevAnalysis) { diagnosticListener.onCompilePhase(config.name, CompilePhase.SavingAnalysis) debug(s"[ZincBridge] Saving analysis for ${config.name}") - saveAnalysis(analysisFile, result.analysis, result.setup) + saveAnalysis(analysisFile, result.analysis, result.setup, analyses) checkCaseInsensitiveCollisions(result.analysis, config.name, diagnosticListener) checkAnalysisPortability(result.analysis, analysisFile) } @@ -607,7 +638,7 @@ object ZincBridge { // Corrupt analysis — wipe and signal clean rebuild needed System.err.println(s"[ZincBridge] ${config.name}: corrupt analysis detected (${e.getMessage}), deleting for clean rebuild") Files.deleteIfExists(analysisFile) - analysisCache.remove(analysisFile) + analyses.invalidate(analysisFile) noopManifestCache.remove(analysisFile) if (Files.exists(config.outputDir)) { bleep.internal.FileUtils.deleteDirectory(config.outputDir) @@ -635,7 +666,7 @@ object ZincBridge { // analysisFile so the next compile doesn't observe stale state, and surface a structured // failure with the underlying exception named so the user sees something actionable instead // of a raw Zinc stack trace via BSP. - analysisCache.remove(analysisFile) + analyses.invalidate(analysisFile) noopManifestCache.remove(analysisFile) val rendered = { val sw = new java.io.StringWriter @@ -915,7 +946,8 @@ object ZincBridge { dependencyAnalyses: Map[Path, Path], progressListener: ProgressListener, cancellationToken: CancellationToken, - diagnosticListener: DiagnosticListener + diagnosticListener: DiagnosticListener, + analyses: AnalysisCache.Ref ): Inputs = { val outputDir = config.outputDir // Include output directory in classpath for incremental Java compilation. @@ -926,31 +958,26 @@ object ZincBridge { val scalacOptions = language.scalaOptions.toArray val javacOptions = language.javaOptions.toArray - // Load analyses from dependency projects for proper incremental compilation. - // Uses a global SoftReference cache to avoid re-reading from disk when the same - // dependency is referenced by multiple projects in the same build. + // Load analyses from dependency projects for proper incremental compilation. The cache spares + // us re-reading the same file when several projects in one build depend on it; a miss just + // reads from disk, so eviction is never a correctness question. val loadedAnalyses: Map[Path, CompileAnalysis] = dependencyAnalyses.flatMap { case (classDir, analysisFile) => if (Files.exists(analysisFile)) { val currentMtime = Files.getLastModifiedTime(analysisFile).toMillis - val cached = { - val ref = analysisCache.get(analysisFile) - val entry = if (ref != null) ref.get() else null - // Invalidate if file changed on disk (e.g. after remote-cache pull) - if (entry != null && entry._1 == currentMtime) entry._2 else null - } - if (cached != null) { - Some(classDir -> cached) - } else { - try { - val store = sbt.internal.inc.FileAnalysisStore.binary(analysisFile.toFile, analysisMappers(analysisFile)) - store.get().toScala.map { contents => - val analysis = contents.getAnalysis - analysisCache.put(analysisFile, new SoftReference((currentMtime, analysis))) - classDir -> analysis + analyses.get(analysisFile, currentMtime) match { + case Some(cached) => Some(classDir -> cached) + case None => + try { + val store = analysisStore(analysisFile, analysisFile) + store.get().toScala.map { contents => + // The interned instance, not the freshly deserialized one: `put` shares structure + // with what other workspaces already loaded, and the original becomes garbage here. + val analysis = analyses.put(analysisFile, currentMtime, contents.getAnalysis) + classDir -> analysis + } + } catch { + case _: Exception => None } - } catch { - case _: Exception => None - } } } else None } @@ -1151,7 +1178,7 @@ object ZincBridge { debug(s"[ZincBridge] Looking for analysis at: $analysisFile, exists=${Files.exists(analysisFile)}") if (Files.exists(analysisFile)) { try { - val store = sbt.internal.inc.FileAnalysisStore.binary(analysisFile.toFile, analysisMappers(analysisFile)) + val store = analysisStore(analysisFile, analysisFile) val contents = store.get() if (contents.isPresent) { val analysis = contents.get().getAnalysis.asInstanceOf[sbt.internal.inc.Analysis] @@ -1199,7 +1226,7 @@ object ZincBridge { } } - private def saveAnalysis(analysisFile: Path, analysis: CompileAnalysis, setup: MiniSetup): Unit = { + private def saveAnalysis(analysisFile: Path, analysis: CompileAnalysis, setup: MiniSetup, analyses: AnalysisCache.Ref): Unit = { // Write to a temp file first, then atomic rename. // This prevents corrupted analysis.zip when compilation is cancelled mid-write. val tempFile = analysisFile.resolveSibling(analysisFile.getFileName.toString + ".tmp") @@ -1209,12 +1236,14 @@ object ZincBridge { val sample = cpHashes.take(3).map(fh => s"${fh.file}:${fh.hash}").mkString(", ") debug(s"[ZincBridge] Saving with classpath hashes: $sample") } - val store = sbt.internal.inc.FileAnalysisStore.binary(tempFile.toFile, analysisMappers(analysisFile)) + val store = analysisStore(tempFile, analysisFile) store.set(AnalysisContents.create(analysis, setup)) Files.move(tempFile, analysisFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE) - // Update soft reference cache so dependents see the fresh analysis + // Update the cache so dependents see the fresh analysis. The interned instance is discarded + // here — this analysis is already written to disk and the caller holds its own reference; the + // point is to seed the cache so the next reader shares structure instead of re-deserializing. val mtime = Files.getLastModifiedTime(analysisFile).toMillis - analysisCache.put(analysisFile, new SoftReference((mtime, analysis))): Unit + analyses.put(analysisFile, mtime, analysis): Unit } catch { case e: Exception => try Files.deleteIfExists(tempFile) @@ -1954,6 +1983,22 @@ private[bleep] class PlainVirtualFile private (val path: Path, virtualId: String } else { 0L } + + /** Added to `xsbti.VirtualFile` in zinc 2 (via `HashedVirtualFileRef`), together with [[contentHashStr]]. + * + * Zinc 2's own `MappedVirtualFile` is `Files.size(path)` with no existence guard; this keeps the guard the surrounding code already applies to + * [[contentHash]], because bleep hands zinc virtual files for outputs that may not exist yet. + */ + def sizeBytes(): Long = + if (Files.exists(path)) Files.size(path) else 0L + + /** SHA-256 of the content, matching zinc 2's `MappedVirtualFile.contentHashStr` (`HashUtil.sha256HashStr(input)`) rather than inventing a rendering of + * [[contentHash]] — the two are different hashes and zinc compares these strings against ones it produced itself. + */ + def contentHashStr(): String = + if (Files.exists(path)) sbt.internal.inc.HashUtil.sha256HashStr(input()) + else sbt.internal.inc.HashUtil.sha256HashStr(new java.io.ByteArrayInputStream(Array.emptyByteArray)) + def input(): java.io.InputStream = Files.newInputStream(path) def toPath(): Path = path } diff --git a/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala b/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala index 8aa10a8fc..e4233ff1e 100644 --- a/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala +++ b/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala @@ -41,6 +41,54 @@ object BspMetrics { private val concurrentCompiles = AtomicInteger(0) private val activeConnections = AtomicInteger(0) + /** What is compiling right now, so a heap event can name it instead of merely counting it. + * + * A count answers "how loaded was the server"; only the names answer "loaded with what, from which workspace" — which is the question you have when the heap + * is at its ceiling. Reconstructing this after the fact means replaying every start/end pair in the file and hoping none were dropped. + * + * Keyed by workspace+project, valued by the allocation reading taken at start (see [[ThreadAllocation]]). + */ + private val inFlightCompiles = new java.util.concurrent.ConcurrentHashMap[String, ThreadAllocation]() + + /** A thread's cumulative allocation at a point in time, for attributing bytes to whatever ran in between. + * + * `getThreadAllocatedBytes` is per THREAD, and a cats-effect fiber may finish on a different thread than it started on. So the reading records which thread + * it came from, and the delta is only reported when the same thread closes it out — an unattributable compile reports nothing rather than a number that + * silently means something else. + */ + private case class ThreadAllocation(threadId: Long, bytes: Long, startedMs: Long) + + private val threadBeanForAllocation: Option[com.sun.management.ThreadMXBean] = + ManagementFactory.getThreadMXBean match { + case tb: com.sun.management.ThreadMXBean if tb.isThreadAllocatedMemorySupported => + tb.setThreadAllocatedMemoryEnabled(true) + Some(tb) + case _ => None + } + + private def readThreadAllocation(): ThreadAllocation = { + val t = Thread.currentThread() + ThreadAllocation(t.threadId(), threadAllocatedBytes(), System.currentTimeMillis()) + } + + /** Cumulative bytes allocated by the CALLING thread, or -1 where the JVM does not report it. + * + * Public because the only place a compile is reliably pinned to one thread is inside ZincBridge's `IO.interruptible` block, so that is where the two + * readings have to be taken. See [[recordCompileAllocation]]. + */ + def threadAllocatedBytes(): Long = + threadBeanForAllocation.map(_.getThreadAllocatedBytes(Thread.currentThread().threadId())).getOrElse(-1L) + + /** Bytes a single compile allocated, measured across the blocking section that does the work. + * + * Churn, not retention: a compile that allocates 40GB and keeps none of it is a very different problem from one that keeps 4GB, and the two are + * indistinguishable in a heap-usage graph. This is the number that says which module actually needs the headroom. + */ + def recordCompileAllocation(project: String, allocatedBytes: Long, durationMs: Long): Unit = + writeEvent( + s"""{"type":"compile_allocation","ts":${now()},"project":"${esc(project)}","allocated_mb":${allocatedBytes / (1024 * 1024)},"duration_ms":$durationMs}""" + ) + // OOM tracking private val OomThreshold = 0.95 @volatile private var oomPressureStarted = false @@ -157,11 +205,15 @@ object BspMetrics { def recordCompileStart(project: String, workspace: String): Unit = { val current = concurrentCompiles.incrementAndGet() updateMax(maxConcurrentCompiles, current) + inFlightCompiles.put(inFlightKey(project, workspace), readThreadAllocation()): Unit writeEvent(s"""{"type":"compile_start","ts":${now()},"project":"${esc(project)}","workspace":"${esc(workspace)}","concurrent":$current}""") } def recordCompileEnd(project: String, workspace: String, durationMs: Long, success: Boolean): Unit = { val current = concurrentCompiles.decrementAndGet() + inFlightCompiles.remove(inFlightKey(project, workspace)): Unit + // No allocation figure here on purpose — see recordCompileAllocation, which is emitted from + // inside the compile's own blocking section where the thread is stable. writeEvent( s"""{"type":"compile_end","ts":${now()},"project":"${esc(project)}","workspace":"${esc( workspace @@ -169,6 +221,63 @@ object BspMetrics { ) } + private def inFlightKey(project: String, workspace: String): String = s"$workspace::$project" + + /** The in-flight compiles as a JSON array, newest last. Attached to heap events so they name what was running. */ + private def inFlightJson(): String = { + val now = System.currentTimeMillis() + inFlightCompiles + .entrySet() + .asScala + .toVector + .sortBy(_.getValue.startedMs) + .map(e => s"""{"key":"${esc(e.getKey)}","age_ms":${now - e.getValue.startedMs}}""") + .mkString("[", ",", "]") + } + + /** Machine-governor state: what the daemon-wide resource governor is admitting and what is queued behind it. Emitted periodically by the daemon's reporter, + * which is the only thing holding the governor. + */ + def recordMachine( + usedCpu: Int, + totalCpu: Int, + usedMemoryMb: Long, + totalMemoryMb: Long, + activeCompiles: Int, + running: Int, + waiting: Int + ): Unit = + writeEvent( + s"""{"type":"machine","ts":${now()},"used_cpu":$usedCpu,"total_cpu":$totalCpu,"used_memory_mb":$usedMemoryMb,"total_memory_mb":$totalMemoryMb,"active_compiles":$activeCompiles,"running":$running,"waiting":$waiting}""" + ) + + /** What the Zinc analysis cache is holding after each sweep. The largest single retainer in the server heap, so its size is the first number to look at when + * the live set is climbing. + */ + def recordAnalysisCache(stats: bleep.analysis.AnalysisCache.Stats): Unit = { + // Per workspace, because the total alone was what the old telemetry gave and it left the actual + // question — which build is holding the heap — to be reconstructed from a class histogram. + val perWorkspace = stats.perWorkspace + .map(w => + s"""{"workspace":"${esc(w.key.workspace.toString)}","variant":"${esc(w.key.variant.toString)}","entries":${w.entries},"file_bytes":${w.fileBytes}}""" + ) + .mkString("[", ",", "]") + writeEvent( + s"""{"type":"analysis_cache","ts":${now()},"entries":${stats.entries},"file_bytes":${stats.fileBytes},"workspaces":${stats.perWorkspace.size},"interned_classes":${stats.internedClasses},"intern_hits":${stats.internHits},"intern_misses":${stats.internMisses},"sharing_factor":${String + .format(Locale.US, "%.2f", stats.sharingFactor: java.lang.Double)},"per_workspace":$perWorkspace}""" + ) + } + + /** Which workspaces the daemon is holding resolved builds for. The retained-heap floor tracks this number, so recording it is what makes the floor + * attributable instead of merely visible. + */ + def recordWorkspaceState(cached: List[String], maxCached: Int): Unit = + writeEvent( + s"""{"type":"workspace_state","ts":${now()},"cached_count":${cached.size},"max_cached":$maxCached,"cached":${cached + .map(w => s""""${esc(w)}"""") + .mkString("[", ",", "]")}}""" + ) + def recordBuildStart(workspace: String, projectCount: Int): Unit = writeEvent(s"""{"type":"build_start","ts":${now()},"workspace":"${esc(workspace)}","projects":$projectCount}""") @@ -225,6 +334,21 @@ object BspMetrics { // --------------- JVM sampling --------------- + /** The live set: heap still occupied immediately AFTER the last collection of each heap pool, which is the number that says whether the server is retaining + * or merely churning. + * + * `heap_used_mb` cannot answer that. Sampled at an arbitrary moment it includes whatever garbage has accumulated since the last GC, so a healthy server + * churning hard and a server whose floor is creeping up towards its ceiling produce the same sawtooth. Deriving the floor by taking minima over a window (as + * one has to do without this) needs a long window and still only approximates it. + * + * `-1` when the JVM does not report collection usage for its heap pools, which is a real answer, not a zero to be averaged in. + */ + private def liveSetMb(): Long = { + val pools = ManagementFactory.getMemoryPoolMXBeans.asScala.filter(_.getType == java.lang.management.MemoryType.HEAP) + val usages = pools.flatMap(p => Option(p.getCollectionUsage)) + if (usages.isEmpty) -1L else usages.map(_.getUsed).sum / (1024 * 1024) + } + private def sampleJvm(): Unit = { val memBean = ManagementFactory.getMemoryMXBean val heap = memBean.getHeapMemoryUsage @@ -260,12 +384,13 @@ object BspMetrics { val currentCompiles = concurrentCompiles.get() val loadedClasses = ManagementFactory.getClassLoadingMXBean.getLoadedClassCount + val heapLiveMb = liveSetMb() val cpuProcessStr = String.format(Locale.US, "%.4f", cpuProcess: java.lang.Double) val cpuSystemStr = String.format(Locale.US, "%.4f", cpuSystem: java.lang.Double) writeEvent( - s"""{"type":"jvm","ts":${now()},"heap_used_mb":$heapUsedMb,"heap_committed_mb":$heapCommittedMb,"heap_max_mb":$heapMaxMb,"non_heap_used_mb":$nonHeapUsedMb,"gc":$gcJson,"threads":$threads,"peak_threads":$peakThreads,"daemon_threads":$daemonThreads,"cpu_process":$cpuProcessStr,"cpu_system":$cpuSystemStr,"concurrent_compiles":$currentCompiles,"loaded_classes":$loadedClasses}""" + s"""{"type":"jvm","ts":${now()},"heap_used_mb":$heapUsedMb,"heap_committed_mb":$heapCommittedMb,"heap_max_mb":$heapMaxMb,"non_heap_used_mb":$nonHeapUsedMb,"gc":$gcJson,"threads":$threads,"peak_threads":$peakThreads,"daemon_threads":$daemonThreads,"cpu_process":$cpuProcessStr,"cpu_system":$cpuSystemStr,"concurrent_compiles":$currentCompiles,"loaded_classes":$loadedClasses,"heap_live_mb":$heapLiveMb}""" ) // OOM pressure detection: heap used >= 95% of max @@ -276,11 +401,11 @@ object BspMetrics { if (!oomPressureStarted) { oomPressureStarted = true writeEvent( - s"""{"type":"oom_pressure","ts":${now()},"heap_used_mb":$heapUsedMb,"heap_max_mb":$heapMaxMb,"pct":${String.format( + s"""{"type":"oom_pressure","ts":${now()},"heap_used_mb":$heapUsedMb,"heap_live_mb":$heapLiveMb,"heap_max_mb":$heapMaxMb,"pct":${String.format( Locale.US, "%.1f", (usedPct * 100): java.lang.Double - )},"concurrent_compiles":$currentCompiles,"active_connections":${activeConnections.get()}}""" + )},"concurrent_compiles":$currentCompiles,"active_connections":${activeConnections.get()},"in_flight":${inFlightJson()}}""" ) System.err.println(s"[BspMetrics] WARNING: Heap at ${(usedPct * 100).toInt}% ($heapUsedMb/$heapMaxMb MB) with $currentCompiles concurrent compiles") } diff --git a/bleep-bsp/src/scala/bleep/bsp/BspServerDaemon.scala b/bleep-bsp/src/scala/bleep/bsp/BspServerDaemon.scala index 4b6242bf4..9272efe78 100644 --- a/bleep-bsp/src/scala/bleep/bsp/BspServerDaemon.scala +++ b/bleep-bsp/src/scala/bleep/bsp/BspServerDaemon.scala @@ -204,37 +204,30 @@ object BspServerDaemon { // // The starting budget below is only a starting point; `retuneLoop` tracks what the machine can // actually spare from there. - val numCores = Runtime.getRuntime.availableProcessors() + // Read once, here, rather than per connection: this bounds state that spans every client on + // this daemon, and a per-connection reading would let the newest client's config silently + // redefine it for the others. + val daemonConfig = BleepConfigOps.loadOrDefault(UserPaths.fromAppDirs).orThrow.bspServerConfigOrDefault + + // `parallelism`, not the raw core count: it defaults to one per core, but when it is set it is a + // statement about how much of THIS MACHINE bleep may use, and the governor is what enforces that + // across clients. Reading cores here instead meant a user who asked for 2 got 2 per run and 2 per + // pool — but the governor still admitted up to one-per-core across every connected client. + val maxConcurrentOperations = daemonConfig.effectiveParallelism val serverHeapMb = Runtime.getRuntime.maxMemory() / (1024L * 1024L) val physicalMb = MachineResources.physicalMemoryMb(fallbackMb = serverHeapMb * 2) val forkMemoryBudgetMb = MachineResources.forkMemoryBudgetMb(physicalMb, serverHeapMb) logger.info( - s"Machine: $numCores cores, ${physicalMb}MB RAM, server heap ${serverHeapMb}MB -> initial fork-memory budget ${forkMemoryBudgetMb}MB" + s"Machine: ${Runtime.getRuntime.availableProcessors()} cores, ${physicalMb}MB RAM, server heap ${serverHeapMb}MB -> " + + s"parallelism $maxConcurrentOperations, initial fork-memory budget ${forkMemoryBudgetMb}MB" ) val machine = MachineResources.create( - totalCpu = numCores, + totalCpu = maxConcurrentOperations, totalMemoryMb = forkMemoryBudgetMb, logger = logger, longWaitWarnMs = MachineResources.DefaultLongWaitWarnMs ) - // Background reporter: when work is queued waiting for resources, log the machine load - // periodically so a stalled build has a legible cause. - locally { - import cats.effect.unsafe.implicits.global - val reporter = new Thread("bleep-machine-reporter") { - override def run(): Unit = - try - while (!shutdownRequested.get()) { - Thread.sleep(15000) - if (machine.isContended.unsafeRunSync()) logger.info(machine.snapshot.unsafeRunSync().render) - } - catch { case _: InterruptedException => () } - } - reporter.setDaemon(true) - reporter.start() - } - // Track what the machine can actually spare, for as long as the daemon lives. // // The figure computed above from physical RAM is a starting point and nothing more: the budget @@ -267,9 +260,50 @@ object BspServerDaemon { // generated-sources directory, so serializing per project must span connections. val kspMutexes = new KspMutexes - // Resolved builds, cached for the daemon's lifetime rather than per connection, so a one-shot - // `bleep compile` no longer re-resolves the whole build on every invocation. - val buildCache = new BuildCache + // Resolved builds, cached across connections rather than per connection, so a one-shot + // `bleep compile` no longer re-resolves the whole build on every invocation — but bounded, so a + // daemon that has served a dozen worktrees is not still holding all twelve. + // Zinc analyses, held per workspace and bounded per workspace. Constructed here and handed to + // BuildCache, so that dropping a build also drops the analyses read while compiling it — which + // is where the memory actually is. See BuildCache.dropAll for why the cascade runs one way only. + val analysisCache = new bleep.analysis.AnalysisCache + val buildCache = new BuildCache(daemonConfig.effectiveMaxCachedWorkspaces, analysisCache) + + // Background reporter. Two jobs, one thread: + // + // - log the machine load when work is queued, so a stalled build has a legible cause; + // - record governor and cache state to metrics EVERY cycle, contended or not. + // + // The unconditional half is the point. Both quantities this records — how many compiles the + // governor was admitting, and how many workspaces' builds were resident — are the ones you want + // a history of when a heap event lands, and a history with gaps in the uncontended stretches is + // exactly the history that cannot tell you when the floor started climbing. + locally { + import cats.effect.unsafe.implicits.global + val reporter = new Thread("bleep-machine-reporter") { + override def run(): Unit = + try + while (!shutdownRequested.get()) { + Thread.sleep(15000) + val snapshot = machine.snapshot.unsafeRunSync() + if (snapshot.waiting.nonEmpty) logger.info(snapshot.render) + BspMetrics.recordMachine( + usedCpu = snapshot.usedCpu, + totalCpu = snapshot.totalCpu, + usedMemoryMb = snapshot.usedMemoryMb, + totalMemoryMb = snapshot.totalMemoryMb, + activeCompiles = snapshot.activeCompiles, + running = snapshot.active.size, + waiting = snapshot.waiting.size + ) + BspMetrics.recordWorkspaceState(buildCache.cachedWorkspaces, buildCache.bound) + BspMetrics.recordAnalysisCache(analysisCache.stats) + } + catch { case _: InterruptedException => () } + } + reporter.setDaemon(true) + reporter.start() + } // NOTE: Do NOT redirect stdout — Zinc writes massive amounts of data to // stdout which would bloat the log file to tens of GB. @@ -380,7 +414,8 @@ object BspServerDaemon { logger.withContext("client", connId), machine, kspMutexes, - buildCache + buildCache, + analysisCache ) finally BspMetrics.recordConnectionClose(connId) try clientSocket.close() @@ -397,7 +432,8 @@ object BspServerDaemon { logger.withContext("client", connId), machine, kspMutexes, - buildCache + buildCache, + analysisCache ) finally { BspMetrics.recordConnectionClose(connId) @@ -449,7 +485,8 @@ object BspServerDaemon { logger: Logger, machine: MachineResources, kspMutexes: KspMutexes, - buildCache: BuildCache + buildCache: BuildCache, + analysisCache: bleep.analysis.AnalysisCache ): Unit = try { // Create multi-workspace server using the daemon-level logger @@ -460,7 +497,8 @@ object BspServerDaemon { machine = machine, heapMonitor = HeapMonitor.system, kspMutexes = kspMutexes, - buildCache = buildCache + buildCache = buildCache, + analysisCache = analysisCache ) // Run server message loop diff --git a/bleep-bsp/src/scala/bleep/bsp/BuildCache.scala b/bleep-bsp/src/scala/bleep/bsp/BuildCache.scala index d5d281231..4831ab8a8 100644 --- a/bleep-bsp/src/scala/bleep/bsp/BuildCache.scala +++ b/bleep-bsp/src/scala/bleep/bsp/BuildCache.scala @@ -5,6 +5,8 @@ import ryddig.Logger import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters.* /** Resolved builds, cached for the lifetime of the daemon rather than of a connection. * @@ -19,12 +21,17 @@ import java.util.concurrent.ConcurrentHashMap * Keyed by (workspace, variant) and versioned by [[BuildId]], so "the same build again" is a cache hit and "a different build" is an explicit, logged * adoption. */ -class BuildCache { +/** @param maxWorkspaces + * how many entries to keep before evicting the least recently used idle one. A resolved build is not small — exploded model, resolved classpaths, and the + * Zinc analysis reachable from it — and this cache used to be unbounded, so a daemon accumulated one per workspace it had ever served and never gave any of + * it back. Measured on a daemon serving 11 worktrees: the post-GC floor climbed from 3.5GB to 8.2GB of a 12GB heap over 45 minutes, after which compiles + * OOM'd at a concurrency of three. Bounding the cache bounds that floor. + */ +class BuildCache(maxWorkspaces: Int, analysisCache: bleep.analysis.AnalysisCache) { - private case class Key(workspace: Path, variant: model.BuildVariant) - private case class Entry(buildId: BuildId, started: Started) + private case class Entry(buildId: BuildId, started: Started, lastUsedMs: AtomicLong) - private val entries = new ConcurrentHashMap[Key, Entry]() + private val entries = new ConcurrentHashMap[model.WorkspaceKey, Entry]() /** One monitor per key, so loading a build for one workspace does not block another. Deliberately not `entries.synchronized` or `entries.compute`: the load * can take seconds, and both of those would serialize unrelated workspaces. @@ -34,7 +41,7 @@ class BuildCache { * handler-to-IO refactor exists to remove. Blocking here is safe in the meantime: this runs on the blocking pool, the load underneath it is blocking work * (coursier) either way, and nothing inside the monitor touches `IO` or re-enters this cache. Revisit together with that refactor. */ - private val loadLocks = new ConcurrentHashMap[Key, AnyRef]() + private val loadLocks = new ConcurrentHashMap[model.WorkspaceKey, AnyRef]() /** Look up the build for this workspace+variant, loading it if absent or if the client means a different one. * @@ -51,11 +58,12 @@ class BuildCache { buildId: BuildId, logger: Logger )(load: => Either[BleepException, Started]): Either[BleepException, Started] = { - val key = Key(workspace, variant) + val key = model.WorkspaceKey(workspace, variant) loadLocks.computeIfAbsent(key, _ => new AnyRef).synchronized { Option(entries.get(key)) match { case Some(entry) if entry.buildId == buildId => + entry.lastUsedMs.set(System.currentTimeMillis()) Right(entry.started) case existing => @@ -70,7 +78,8 @@ class BuildCache { .info("Adopting a different build for this workspace") } load.map { started => - entries.put(key, Entry(buildId, started)) + entries.put(key, Entry(buildId, started, new AtomicLong(System.currentTimeMillis()))) + evictDownToBound(keep = key, logger) started } } @@ -79,5 +88,86 @@ class BuildCache { /** Drop the entry for a workspace+variant, so the next `getOrLoad` reloads. Used by `workspace/reload`. */ def evict(workspace: Path, variant: model.BuildVariant): Unit = - entries.remove(Key(workspace, variant)): Unit + dropAll(model.WorkspaceKey(workspace, variant)): Unit + + /** Drop a build AND the Zinc analyses read while compiling it. + * + * The cascade runs in this direction only. If nothing wants the build, nothing wants its analyses either, and the analyses are where the memory actually is + * — a resolved build is hundreds of MB, its analyses are gigabytes; evicting the build alone frees the small half and keeps the large one. + * + * The reverse cascade would be wrong, because the costs invert. Re-resolving a build costs seconds of coursier work; re-reading an analysis costs about one + * disk read. So analyses are shed eagerly on their own schedule (idle timeout, per-workspace budget) without ever disturbing the build they belong to. + */ + private def dropAll(key: model.WorkspaceKey): bleep.analysis.AnalysisCache.Freed = { + entries.remove(key): Unit + analysisCache.evictWorkspace(key) + } + + /** The workspaces currently held, for telemetry. Distinct: one workspace can hold several variants, but the interesting quantity is how many builds' worth of + * state is resident. + */ + def cachedWorkspaces: List[String] = + entries.keySet().iterator().asScala.map(_.workspace.toString).toList.distinct.sorted + + /** The bound in force, so telemetry can record what was being enforced rather than only what resulted. */ + def bound: Int = maxWorkspaces + + /** Evict least-recently-used entries until at most [[maxWorkspaces]] remain. + * + * Two entries are never candidates: the one just loaded, and any whose workspace has operations in flight. The second is not a correctness requirement — + * every in-flight operation captured its own `Started` when it began and runs against that, exactly as it does when a client adopts a different build — but + * evicting a workspace that is mid-build only means reloading it moments later, which is pure waste. + * + * Because of that exclusion the cache CAN exceed its bound: with more busy workspaces than slots, nothing is evictable and the bound yields rather than + * stalling a build. It is a cache size, not an admission limit, and a daemon can still serve any number of workspaces at once. + * + * Synchronized on `entries` rather than on the per-key load locks: this pass touches every key, it holds no I/O, and taking the per-key lock of a workspace + * we are about to drop would invert the lock order that `getOrLoad` establishes. + */ + private def evictDownToBound(keep: model.WorkspaceKey, logger: Logger): Unit = + entries.synchronized { + val present = entries.entrySet().iterator().asScala.map(e => (e.getKey, e.getValue.lastUsedMs.get())).toVector + val doomed = BuildCache.selectEvictions( + present = present, + keep = keep, + bound = maxWorkspaces, + isBusy = key => SharedWorkspaceState.getActiveOperations(key.workspace).nonEmpty + ) + doomed.foreach { case (key, lastUsedMs) => + val freed = dropAll(key) + logger + .withContext("workspace", key.workspace.toString) + .withContext("variant", key.variant.toString) + .withContext("idleSeconds", (System.currentTimeMillis() - lastUsedMs) / 1000) + .withContext("cacheSize", entries.size()) + .withContext("maxWorkspaces", maxWorkspaces) + .withContext("analysesFreed", freed.entries) + .withContext("analysisMbFreed", freed.fileBytes / (1024 * 1024)) + .info("Evicting a cached build to bound retained heap; it will be reloaded on next use") + BspMetrics.recordCacheEvict("buildCache", key.workspace.toString) + } + } +} + +object BuildCache { + + /** Which entries to drop so that at most `bound` remain: least recently used first, never `keep`, never a busy one. + * + * Pure and separate from the cache so the policy can be tested without standing up a resolved build. Returns fewer than needed — possibly none — when too + * many entries are busy, which is deliberate: the bound yields to work in progress rather than stalling it. + */ + private[bsp] def selectEvictions[K]( + present: Vector[(K, Long)], + keep: K, + bound: Int, + isBusy: K => Boolean + ): Vector[(K, Long)] = { + val overBy = present.size - bound + if (overBy <= 0) Vector.empty + else + present + .filter { case (key, _) => key != keep && !isBusy(key) } + .sortBy { case (_, lastUsedMs) => lastUsedMs } + .take(overBy) + } } diff --git a/bleep-bsp/src/scala/bleep/bsp/HeapPressureGate.scala b/bleep-bsp/src/scala/bleep/bsp/HeapPressureGate.scala index fe2a12a25..8eb93cac6 100644 --- a/bleep-bsp/src/scala/bleep/bsp/HeapPressureGate.scala +++ b/bleep-bsp/src/scala/bleep/bsp/HeapPressureGate.scala @@ -40,7 +40,7 @@ object HeapPressureGate { */ def waitForHeapPressure( heapMonitor: HeapMonitor, - activeCompileCount: java.util.concurrent.atomic.AtomicInteger, + activeCompiles: IO[Int], threshold: Double, retryMs: DurationMs, projectName: String, @@ -50,7 +50,10 @@ object HeapPressureGate { for { usage <- IO(heapMonitor.heapUsage()) nowMs <- clock.realTime.map(d => EpochMs(d.toMillis)) - othersCompiling = activeCompileCount.get() > 1 + // Daemon-wide, not per connection. See MachineResources.activeCompiles for why that + // distinction is the difference between this gate working and this gate never firing. + compiling <- activeCompiles + othersCompiling = compiling > 1 result <- if (!othersCompiling) { // We're the only active compilation — proceed immediately diff --git a/bleep-bsp/src/scala/bleep/bsp/InProcessBspServer.scala b/bleep-bsp/src/scala/bleep/bsp/InProcessBspServer.scala index 90b9bda50..59d3718a7 100644 --- a/bleep-bsp/src/scala/bleep/bsp/InProcessBspServer.scala +++ b/bleep-bsp/src/scala/bleep/bsp/InProcessBspServer.scala @@ -34,6 +34,7 @@ object InProcessBspServer { try { val numCores = Runtime.getRuntime.availableProcessors() val machine = bleep.MachineResources.forThisMachine(totalCpu = numCores, logger = logger) + val inProcessAnalysisCache = new bleep.analysis.AnalysisCache // One server per in-process run, so fresh daemon-scoped state is correct here. val server = new MultiWorkspaceBspServer( @@ -43,7 +44,8 @@ object InProcessBspServer { machine = machine, heapMonitor = HeapMonitor.system, kspMutexes = new KspMutexes, - buildCache = new BuildCache + buildCache = new BuildCache(bleep.model.BspServerConfig.default.effectiveMaxCachedWorkspaces, inProcessAnalysisCache), + analysisCache = inProcessAnalysisCache ) server.run() } catch { diff --git a/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala b/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala index 447537686..09478f453 100644 --- a/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala +++ b/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala @@ -51,7 +51,8 @@ class MultiWorkspaceBspServer( machine: MachineResources, heapMonitor: HeapMonitor, kspMutexes: KspMutexes, - buildCache: BuildCache + buildCache: BuildCache, + analysisCache: bleep.analysis.AnalysisCache ) { import MultiWorkspaceBspServer.DebugLogging @@ -59,9 +60,6 @@ class MultiWorkspaceBspServer( private val initialized = AtomicBoolean(false) private val shutdownRequested = AtomicBoolean(false) - /** Track active compile count so heap pressure back-pressure can skip stalling when we're the only compile. */ - private val activeCompileCount = new java.util.concurrent.atomic.AtomicInteger(0) - private val clientCapabilities = AtomicReference[Option[BuildClientCapabilities]](None) /** The active workspace for this connection (set during initialize) */ @@ -1448,13 +1446,14 @@ class MultiWorkspaceBspServer( def onLog(message: String, isError: Boolean): Unit = if (isError) bspError(message) else bspInfo(message) } - val forkMemMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(started.config.bspServerConfigOrDefault.sourcegenMaxMemory)) (sgt, killSignal) => killSignal.tryGet.flatMap { case Some(reason) => IO.pure(TaskDag.TaskResult.Killed(reason)) case None => // Sourcegen forks a JVM — reserve machine resources like any other fork. - machine.reserve(MachineResources.ResourceKind.SourcegenFork, s"sourcegen ${sgt.script.main}", cpu = 1, memoryMb = forkMemMb).use { _ => + // No reservation here: the DAG admitted this task against its declared cost before starting + // it, so reserving again would charge the machine twice for one fork. + IO.unit.flatMap { _ => SourceGenRunner .runOne(started, sgt.script, sgt.forProjects, killSignal, listener) .map { @@ -1492,8 +1491,13 @@ class MultiWorkspaceBspServer( val userPaths = UserPaths.fromAppDirs val freshConfig = BleepConfigOps.loadOrDefault(userPaths).getOrElse(model.BleepConfig.default) val serverConfig = freshConfig.bspServerConfigOrDefault - val maxParallelism = serverConfig.effectiveParallelism - debugLog(s"BSP config: parallelism=$maxParallelism") + // Sizes are resolved here, once, so a task can declare the same heap the fork is started with. + val forkHeaps = TaskDag.ForkHeaps( + sourcegenMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(serverConfig.sourcegenMaxMemory)), + kspMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(serverConfig.kspRunnerMaxMemory)), + linkMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(None)) + ) + debugLog(s"BSP config: parallelism=${serverConfig.effectiveParallelism}") // Include transitive dependencies val allProjects = BleepBuildConverter.transitiveDependencies(projectsToCompile, started) @@ -1673,7 +1677,7 @@ class MultiWorkspaceBspServer( // Run executor with guarantee to cancel consumer fiber on completion/error/cancellation dag <- executor - .execute(initialDag, maxParallelism, eventQueue, killSignal) + .execute(initialDag, machine, forkHeaps, eventQueue, killSignal) .flatTap(_ => eventQueue.offer(None) >> eventConsumerFiber.joinWithNever) .guarantee(eventQueue.offer(None).attempt >> eventConsumerFiber.cancel) @@ -1850,7 +1854,7 @@ class MultiWorkspaceBspServer( ): IO[Unit] = HeapPressureGate.waitForHeapPressure( heapMonitor = heapMonitor, - activeCompileCount = activeCompileCount, + activeCompiles = machine.activeCompiles, threshold = threshold, retryMs = HeapPressureGate.DefaultRetryMs, projectName = projectName, @@ -1903,6 +1907,11 @@ class MultiWorkspaceBspServer( val freshConfig = BleepConfigOps.loadOrDefault(userPaths).getOrElse(model.BleepConfig.default) val serverConfig = freshConfig.bspServerConfigOrDefault val maxParallelism = serverConfig.effectiveParallelism + val forkHeaps = TaskDag.ForkHeaps( + sourcegenMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(serverConfig.sourcegenMaxMemory)), + kspMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(serverConfig.kspRunnerMaxMemory)), + linkMb = MachineResources.forkFootprintMb(MachineResources.forkHeapMb(None)) + ) // Sourcegen plan — scripts for test projects and their transitive deps. val allTestAndDeps = BleepBuildConverter.transitiveDependencies(testProjects, started) @@ -2205,7 +2214,7 @@ class MultiWorkspaceBspServer( // Run executor with guarantee to cancel consumer fiber on completion/error/cancellation dag <- executor - .execute(initialDag, maxParallelism, eventQueue, killSignal) + .execute(initialDag, machine, forkHeaps, eventQueue, killSignal) .flatMap { result => IO { val total = result.tasks.size @@ -2445,25 +2454,27 @@ class MultiWorkspaceBspServer( // runs in the server heap (not a forked process), so it reserves no fork memory; server // heap is staggered separately by waitForHeapPressure. val gatedCompile = - machine.reserve(MachineResources.ResourceKind.Compile, s"compile $projectName", cpu = 1, memoryMb = 0L).use { _ => - IO(activeCompileCount.incrementAndGet()) - .bracket { _ => - waitForHeapPressure(projectName, originId, heapPressureThreshold) >> { - val compileStartTime = System.currentTimeMillis() - IO(BspMetrics.recordCompileStart(projectName, wsStr)) >> - compileProject(started, compileTask.project, originId, token, depAnalyses, apFlags, diagnosticTracker) - .guaranteeCase { - case cats.effect.Outcome.Succeeded(resultIO) => - resultIO.flatMap { result => - val dur = System.currentTimeMillis() - compileStartTime - val ok = result == TaskDag.TaskResult.Success - IO(BspMetrics.recordCompileEnd(projectName, wsStr, dur, ok)) - } - case _ => - IO(BspMetrics.recordCompileEnd(projectName, wsStr, System.currentTimeMillis() - compileStartTime, false)) + // Admitted by the DAG before this ran — see TaskDag.admit. + IO.unit.flatMap { _ => + // The reservation IS the count of compiles in flight — held for exactly this scope, + // across every connection, and readable via `machine.activeCompiles`. The connection- + // local tally that used to be maintained here counted only this client's compiles, + // which is not the quantity anything wants to know. + waitForHeapPressure(projectName, originId, heapPressureThreshold) >> { + val compileStartTime = System.currentTimeMillis() + IO(BspMetrics.recordCompileStart(projectName, wsStr)) >> + compileProject(started, compileTask.project, originId, token, depAnalyses, apFlags, diagnosticTracker) + .guaranteeCase { + case cats.effect.Outcome.Succeeded(resultIO) => + resultIO.flatMap { result => + val dur = System.currentTimeMillis() - compileStartTime + val ok = result == TaskDag.TaskResult.Success + IO(BspMetrics.recordCompileEnd(projectName, wsStr, dur, ok)) } - } - }(_ => IO(activeCompileCount.decrementAndGet(): Unit)) + case _ => + IO(BspMetrics.recordCompileEnd(projectName, wsStr, System.currentTimeMillis() - compileStartTime, false)) + } + } } val waitForKill = taskKillSignal.get.map(reason => TaskDag.TaskResult.Killed(reason)) @@ -2663,7 +2674,16 @@ class MultiWorkspaceBspServer( locksResource .use { _ => - compiler.compile(config, diagnosticListener, cancellation, dependencyAnalyses, progressListener) + compiler.compile( + config, + diagnosticListener, + cancellation, + dependencyAnalyses, + progressListener, + // Bound to THIS build, so a compile can only ever read or charge analyses belonging to + // the workspace it is compiling. + bleep.analysis.AnalysisCache.Ref(analysisCache, started.buildPaths.workspaceKey) + ) } .map { case _ if cancellation.isCancelled => diff --git a/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala b/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala index e6d743b86..d3053824d 100644 --- a/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala +++ b/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala @@ -1,5 +1,6 @@ package bleep.bsp +import bleep.MachineResources import bleep.bsp.protocol.KillReason import bleep.bsp.protocol.{BleepBspProtocol, LinkPlatformName, OutputChannel, ProcessExit, SuiteOutcome, TestStatus} import bleep.bsp.protocol.BleepBspProtocol.BuildMode @@ -84,6 +85,36 @@ object TaskDag { def dependencies: Set[TaskId] } + /** A task's claim on the machine. `cpu` is in cores; `memoryMb` is off-heap memory for a forked process. */ + case class Cost(kind: MachineResources.ResourceKind, cpu: Int, memoryMb: Long) + + /** What running `task` costs, given what this machine gives each kind of fork. + * + * A function rather than a field on the task: cost is a property of (what kind of work this is, how this machine is configured), not of the task's identity. + * Putting it on the case class would also make two otherwise-identical tasks unequal because a config value differed. + * + * Admission is done with this by the scheduler loop, in priority order, instead of inside each task's body. Reserving inside meant the loop dispatched blind + * and the governor queued FIFO — so the DAG's dependents-first ordering stopped mattering the moment anything had to wait. + */ + def costOf(task: Task, forkHeaps: ForkHeaps): Cost = + task match { + // In-server work: a core, and no fork memory. Compile heap is watched separately by HeapPressureGate. + case _: CompileTask => Cost(MachineResources.ResourceKind.Compile, cpu = 1, memoryMb = 0L) + case _: DiscoverTask => Cost(MachineResources.ResourceKind.Compile, cpu = 1, memoryMb = 0L) + case _: ResolveAnnotationProcessorsTask => Cost(MachineResources.ResourceKind.Compile, cpu = 1, memoryMb = 0L) + // Forks: charged the heap they are started with plus the non-heap a JVM also commits. + case _: SourcegenTask => Cost(MachineResources.ResourceKind.SourcegenFork, cpu = 1, memoryMb = forkHeaps.sourcegenMb) + case _: RunSymbolProcessorsTask => Cost(MachineResources.ResourceKind.KspFork, cpu = 1, memoryMb = forkHeaps.kspMb) + // Scala.js, Scala Native, Kotlin/JS and Kotlin/Native each fork a linker (and node, for JS). + // These reserved nothing at all until now — counted as one task while being a whole JVM plus a + // toolchain, which is the accounting hole the governor exists to close. + case _: LinkTask => Cost(MachineResources.ResourceKind.SourcegenFork, cpu = 1, memoryMb = forkHeaps.linkMb) + // A core, but no fork memory HERE: acquiring a JVM may reuse a warm one (free) or spawn one + // (measured RSS), and only the pool knows which. It holds that reservation itself, for a + // lifetime this task does not share — one JVM serves many suites. + case _: TestSuiteTask => Cost(MachineResources.ResourceKind.TestFork, cpu = 1, memoryMb = 0L) + } + /** Compile a project. * * `dependencies` is supplied directly (rather than derived) so the DAG builder can combine project-level compile deps with sourcegen deps (SourcegenTask @@ -127,6 +158,7 @@ object TaskDag { ) extends Task { val id: TaskId = TaskId.ResolveAnnotationProcessors(project) val dependencies: Set[TaskId] = Set.empty + // Coursier resolution and jar scanning, in the server. I/O-bound rather than CPU-bound, but it } /** Run KSP for a project. Resolves the KSP standalone-runner classpath + user processors, then forks a JVM running `KSPJvmMain` which processes the project's @@ -157,6 +189,8 @@ object TaskDag { ) extends Task { val id: TaskId = TaskId.Link(project) val dependencies: Set[TaskId] = Set(TaskId.Compile(project)) + // Scala.js, Scala Native, Kotlin/JS and Kotlin/Native all fork a linker (and node, for JS). Those + // forks reserved nothing until now — counted as one task while actually being a whole JVM plus a } /** Platform for linking non-JVM targets. */ @@ -236,6 +270,9 @@ object TaskDag { ) extends Task { val id: TaskId = TaskId.Test(project, suiteName) val dependencies: Set[TaskId] = Set(TaskId.Discover(project)) + // A core, but no fork memory declared here: acquiring a JVM from the pool may reuse a warm one + // (free) or spawn a new one (measured RSS), and only the pool knows which. It holds that + // reservation itself, from spawn until the process is destroyed — a lifetime this task does not } /** Result of task execution. @@ -576,6 +613,18 @@ object TaskDag { apPlan: AnnotationProcessorPlan, kspPlan: SymbolProcessorPlan ) + + /** What each kind of forked JVM is charged: its heap plus the non-heap a JVM also commits (metaspace, code cache, stacks, GC structures). Resolved from + * config once when the DAG is built, so the number a task declares is the number the fork is actually started with. + */ + case class ForkHeaps(sourcegenMb: Long, kspMb: Long, linkMb: Long) + object ForkHeaps { + + /** What bleep gives a fork that states no `-Xmx` of its own. */ + private val defaultFootprint: Long = MachineResources.forkFootprintMb(MachineResources.DefaultForkHeapMb) + val default: ForkHeaps = ForkHeaps(sourcegenMb = defaultFootprint, kspMb = defaultFootprint, linkMb = defaultFootprint) + } + object BuildContext { val empty: BuildContext = BuildContext( allProjectDeps = Map.empty, @@ -777,8 +826,9 @@ object TaskDag { * * @param dag * The DAG to execute - * @param maxParallelism - * Maximum concurrent tasks + * @param machine + * The machine's resources. Admission happens against this, in priority order, so how much runs at once is what the machine can currently afford rather + * than a number carried alongside it. * @param eventQueue * Queue for emitting events * @param killSignal @@ -788,7 +838,8 @@ object TaskDag { */ def execute( dag: Dag, - maxParallelism: Int, + machine: MachineResources, + forkHeaps: ForkHeaps, eventQueue: Queue[IO, Option[DagEvent]], killSignal: Deferred[IO, KillReason] ): IO[Dag] @@ -813,12 +864,45 @@ object TaskDag { override def execute( initialDag: Dag, - maxParallelism: Int, + machine: MachineResources, + forkHeaps: ForkHeaps, eventQueue: Queue[IO, Option[DagEvent]], killSignal: Deferred[IO, KillReason] ): IO[Dag] = { def now: IO[Long] = IO.realTime.map(_.toMillis) + /** Take as many of `candidates` as the machine can currently afford, in the order given. + * + * `tryReserve` rather than `reserve`: a scheduler with a priority order wants "does this fit now", not "queue me". Anything refused stays ready and gets + * another chance on the next wakeup, which fires whenever a task completes — precisely when resources free. + * + * `idle` is the forward-progress guarantee. With nothing running, nothing will ever complete to wake this loop, so a task that does not fit would hang + * the build rather than merely wait. When the machine is idle the first candidate is admitted by blocking reservation instead, which clamps its request + * to the machine's totals — so a task larger than the whole machine waits for the whole machine and then runs, rather than never running. + */ + def admit(candidates: List[Task], idle: Boolean): IO[List[(Task, IO[Unit])]] = + candidates match { + case Nil => IO.pure(Nil) + case first :: rest => + val firstCost = costOf(first, forkHeaps) + val firstAdmission: IO[Option[(Task, IO[Unit])]] = + if (idle) + machine + .reserveUntilReleased(firstCost.kind, first.id.toString, firstCost.cpu, firstCost.memoryMb) + .map(release => Some((first, release))) + else + machine.tryReserve(firstCost.kind, first.id.toString, firstCost.cpu, firstCost.memoryMb).map(_.map(release => (first, release))) + + firstAdmission.flatMap { headResult => + rest + .traverse { task => + val c = costOf(task, forkHeaps) + machine.tryReserve(c.kind, task.id.toString, c.cpu, c.memoryMb).map(_.map(release => (task, release))) + } + .map(tail => (headResult :: tail).flatten) + } + } + def emit(event: DagEvent): IO[Unit] = eventQueue.offer(Some(event)) /** Check if kill has been requested (non-blocking) */ @@ -1086,17 +1170,22 @@ object TaskDag { // Get ready tasks (not already running), prioritized by dependents count readyTasks = dag.ready.filterNot(t => running.contains(t.id)) depCounts = dag.dependentsCount - availableSlots = maxParallelism - running.size - tasksToStart = readyTasks.toList.sortBy(t => -depCounts.getOrElse(t.id, 0)).take(availableSlots) - // Start tasks. The guarantee fires both runningRef cleanup and a wakeup; the - // wakeup is `tryOffer` so concurrent completions coalesce on the bounded(1) queue. - _ <- tasksToStart.toList.parTraverse_ { task => + // Most-unblocking first. Admission then happens HERE, against the machine, instead of + // inside each task — so the ordering survives contention. Dispatching first and + // reserving inside meant everything queued FIFO in the governor and this sort was + // decoration. + prioritized = readyTasks.toList.sortBy(t => -depCounts.getOrElse(t.id, 0)) + admitted <- admit(prioritized, idle = running.isEmpty) + // Start tasks. The guarantee releases the reservation, cleans up runningRef and wakes + // the loop — and the wakeup is what re-runs admission, so a completion is exactly when + // the next task gets its chance. + _ <- admitted.parTraverse_ { case (task, release) => runningRef.update(_ + task.id) >> supervisor .supervise( executeTask(task, dagRef, taskKillSignals) .guarantee( - runningRef.update(_ - task.id) >> wakeup.tryOffer(()).void + release >> runningRef.update(_ - task.id) >> wakeup.tryOffer(()).void ) ) .void diff --git a/bleep-cli/src/scala/bleep/Main.scala b/bleep-cli/src/scala/bleep/Main.scala index 008e9e7c2..1cb66e737 100644 --- a/bleep-cli/src/scala/bleep/Main.scala +++ b/bleep-cli/src/scala/bleep/Main.scala @@ -724,17 +724,23 @@ object Main { Opts(() => BleepConfigOps.rewritePersisted(logger, userPaths)(updateBspServerConfig(_.copy(compileServerMaxMemory = None))).map(_ => ())) ), Opts.subcommand[BleepCommand]( - "heap-pressure-threshold", - "set heap usage fraction (0.0-1.0) above which new compilations wait for memory (default: 0.80)" + "parallelism", + "set how many operations may run at once — compiles, test forks, sourcegen (default: one per core)" )( - Opts.argument[Double]("threshold").map { threshold => () => + Opts.argument[Int]("n").map { n => () => + if (n < 1) throw new BleepException.Text(s"parallelism must be >= 1, got $n") BleepConfigOps - .rewritePersisted(logger, userPaths)(updateBspServerConfig(_.copy(heapPressureThreshold = Some(threshold)))) - .map(_ => ()) + .rewritePersisted(logger, userPaths)(updateBspServerConfig(_.copy(parallelism = Some(n)))) + .map(_ => logger.info("Takes effect when the server next starts — `bleep config compile-server stop-all` to apply now")) } ), - Opts.subcommand[BleepCommand]("heap-pressure-threshold-clear", "remove heap pressure threshold setting (use default: 0.80)")( - Opts(() => BleepConfigOps.rewritePersisted(logger, userPaths)(updateBspServerConfig(_.copy(heapPressureThreshold = None))).map(_ => ())) + Opts.subcommand[BleepCommand]( + "parallelism-clear", + "remove the parallelism setting (back to default: one per core)" + )( + Opts(() => + BleepConfigOps.rewritePersisted(logger, userPaths)(updateBspServerConfig(_.copy(parallelism = None, parallelismRatio = None))).map(_ => ()) + ) ), Opts.subcommand[BleepCommand]( "read-timeout", diff --git a/bleep-core/src/scala/bleep/MachineResources.scala b/bleep-core/src/scala/bleep/MachineResources.scala index 3d02de078..8c3dc6612 100644 --- a/bleep-core/src/scala/bleep/MachineResources.scala +++ b/bleep-core/src/scala/bleep/MachineResources.scala @@ -148,6 +148,7 @@ final class MachineResources private ( usedCpu = totalCpu - st.freeCpu, totalMemoryMb = st.totalMemoryMb, usedMemoryMb = st.totalMemoryMb - st.freeMemoryMb, + activeCompiles = st.activeCompiles, active = st.active.values.toList.sortBy(_.id).map(r => Entry(r.kind, r.label, r.cpu, r.memoryMb, now - r.sinceMs)), waiting = st.waiting.toList.map(w => Entry(w.kind, w.label, w.cpu, w.memoryMb, now - w.sinceMs)) ) @@ -156,6 +157,14 @@ final class MachineResources private ( /** True when work is queued waiting for resources — the signal worth logging periodically. */ def isContended: IO[Boolean] = state.get.map(_.waiting.nonEmpty) + /** Compiles holding a reservation right now, across every client on this server. + * + * [[bleep.bsp.HeapPressureGate]] needs this and not a per-connection tally: it decides whether to stagger by asking "is anything else compiling", and the + * heap it is protecting is shared by every workspace the daemon serves. Asked per connection, a dozen clients each running a single compile all answer "I'm + * the only one" and every one of them skips the stagger — which is the state the server was in at 17 concurrent compiles. + */ + def activeCompiles: IO[Int] = state.get.map(_.activeCompiles) + /** Retune the fork-memory budget to what the machine can currently afford, and wake anything that now fits. * * The budget is not a property of the hardware; it is a property of the hardware *and everything else running on it*, which changes while a build runs. A @@ -203,7 +212,13 @@ object MachineResources { nextId: Long, active: Map[Long, Reservation], waiting: Vector[Waiter] - ) + ) { + + /** Compiles currently holding a reservation. Derived rather than stored: one counter that must be kept in step with `active` across grant, release and + * cancel is one counter that will eventually disagree with it, and `active` is small. + */ + def activeCompiles: Int = active.values.count(_.kind == ResourceKind.Compile) + } /** Grant every currently-fitting waiter, oldest first (work-conserving: a waiter that doesn't fit is skipped rather than head-of-line-blocking). Pure: * returns the updated state and the waiters that were granted (whose gates the caller then completes). @@ -232,6 +247,11 @@ object MachineResources { * admitting more). Without this, a compile — which reserves a core but zero fork-memory — was refused because free memory was -5GB, so a fleet of idle * pooled test JVMs holding memory could wedge every compile in the build behind an over-commit the compiles do not even contribute to. Observed exactly * once: 18 compiles waiting 16 minutes on `mem 28160/23066MB`. + * + * Compiles are admitted on CPU alone, deliberately. They briefly had a count ceiling of their own, on the reasoning that their scarce resource is the + * server's heap rather than a core — but a fixed fraction is a static partition inside an otherwise work-conserving governor: it holds capacity back for + * test forks that may not exist, so a compile-only run leaves half the machine idle. Heap pressure is answered by [[bleep.bsp.HeapPressureGate]], which + * staggers starts against the live heap instead of guessing from a count. */ private def fits(cpu: Int, memoryMb: Long, freeCpu: Int, freeMemoryMb: Long): Boolean = (cpu == 0 || freeCpu >= cpu) && (memoryMb == 0L || freeMemoryMb >= memoryMb) @@ -243,6 +263,7 @@ object MachineResources { usedCpu: Int, totalMemoryMb: Long, usedMemoryMb: Long, + activeCompiles: Int, active: List[Entry], waiting: List[Entry] ) { @@ -250,7 +271,9 @@ object MachineResources { /** Multi-line human-readable rendering for the server log. */ def render: String = { val sb = new StringBuilder - sb.append(f"machine: cpu $usedCpu%d/$totalCpu%d, mem $usedMemoryMb%d/$totalMemoryMb%dMB, running ${active.size}%d, waiting ${waiting.size}%d") + sb.append( + f"machine: cpu $usedCpu%d/$totalCpu%d, mem $usedMemoryMb%d/$totalMemoryMb%dMB, compiles $activeCompiles%d, running ${active.size}%d, waiting ${waiting.size}%d" + ) def line(prefix: String, e: Entry): Unit = sb.append(f"\n $prefix%-8s ${e.kind}%-14s cpu=${e.cpu}%d mem=${e.memoryMb}%dMB age=${e.ageMs / 1000}%ds ${e.label}") active.foreach(line("running", _)) diff --git a/bleep-core/src/scala/bleep/ResolveProjects.scala b/bleep-core/src/scala/bleep/ResolveProjects.scala index baadf9157..66f7572ab 100644 --- a/bleep-core/src/scala/bleep/ResolveProjects.scala +++ b/bleep-core/src/scala/bleep/ResolveProjects.scala @@ -261,6 +261,20 @@ object ResolveProjects { case other => sys.error(s"unexpected: $other") } + /** Version schemes from this project AND from everything it depends on. + * + * A scheme travels with the library it is a claim about. `inherited` above pulls a dependency's libraries into this project's resolution, so this project + * faces that dependency's conflicts — and must therefore be allowed to inherit the dependency's judgement about them too. Without this, a project that + * resolves cleanly makes every consumer fail on a conflict it did not introduce and cannot see, and the fix is to repeat the same scheme in every + * consumer. + * + * Templates still contribute through `explodedProject`, so a scheme can be stated either way: on the project that pulls the library, or on a template for + * a whole family of projects. + */ + val libraryVersionSchemes: SortedSet[model.LibraryVersionScheme] = + explodedProject.libraryVersionSchemes.values ++ + build.transitiveDependenciesFor(crossName).flatMap { case (_, p) => p.libraryVersionSchemes.values } + val (resolvedDependencies, resolvedRuntimeDependencies) = { val fromPlatform = versionCombo.libraries(isTest = explodedProject.isTestProject.getOrElse(false)) @@ -277,7 +291,7 @@ object ResolveProjects { val normal = resolver.force( deps, versionCombo, - explodedProject.libraryVersionSchemes.values, + libraryVersionSchemes, crossName.value, explodedProject.ignoreEvictionErrors.getOrElse(model.IgnoreEvictionErrors.No) ) @@ -294,7 +308,7 @@ object ResolveProjects { resolver.force( deps, versionCombo, - explodedProject.libraryVersionSchemes.values, + libraryVersionSchemes, crossName.value, explodedProject.ignoreEvictionErrors.getOrElse(model.IgnoreEvictionErrors.No) ) @@ -451,7 +465,7 @@ object ResolveProjects { model.Options.Opt.Flag( s"${constants.ScalaPluginPrefix}:" + resolver - .force(Set(dep), versionCombo, explodedProject.libraryVersionSchemes.values, crossName.value, model.IgnoreEvictionErrors.No) + .force(Set(dep), versionCombo, libraryVersionSchemes, crossName.value, model.IgnoreEvictionErrors.No) .fullDetailedArtifacts .collect { case (_, pub, _, Some(file)) if pub.classifier != Classifier.sources && pub.ext == Extension.jar => file.toPath } .filterNot(resolvedScalaCompiler.toSet) diff --git a/bleep-core/src/scala/bleep/bsp/BspServerOperations.scala b/bleep-core/src/scala/bleep/bsp/BspServerOperations.scala index 36538a9b7..6abdf65c3 100644 --- a/bleep-core/src/scala/bleep/bsp/BspServerOperations.scala +++ b/bleep-core/src/scala/bleep/bsp/BspServerOperations.scala @@ -5,7 +5,7 @@ import ryddig.Logger import java.net.{ConnectException, InetSocketAddress, Socket, StandardProtocolFamily} import java.nio.channels.SocketChannel -import java.nio.file.{Files, Path} +import java.nio.file.{Files, Path, StandardCopyOption} import java.nio.file.attribute.PosixFilePermissions import scala.concurrent.duration.* import scala.jdk.CollectionConverters.* @@ -49,11 +49,11 @@ object BspServerOperations { /** A hint to append to connect/startup failure messages when the server log shows an OutOfMemoryError death. * - * By the time these failures surface, startServer has rotated `output` to `output.prev`, so the crashed generation's log may be under either name — check - * both and say which one carried the evidence. + * By the time these failures surface, startServer has rotated the logs, so the crashed generation may be under `output` or any kept generation — check them + * all and say which one carried the evidence. */ def oomHint(socketDir: Path): String = - List("output", "output.prev").map(socketDir.resolve).find(containsOomMarker) match { + ("output" :: (1 to OutputGenerationsKept).map(n => s"output.$n").toList).map(socketDir.resolve).find(containsOomMarker) match { case None => "" case Some(log) => s"""\nNote: $log contains "$OomMarker" — the server ran out of heap.""" + @@ -84,12 +84,8 @@ object BspServerOperations { } val outputFile = socketDir.resolve("output") - // Preserve previous server's output for debugging, then start fresh - val prevOutputFile = socketDir.resolve("output.prev") - Files.deleteIfExists(prevOutputFile) - if (Files.exists(outputFile)) { - Files.move(outputFile, prevOutputFile): Unit - } + rotateOutput(socketDir) + pruneStaleSocketDirs(socketDir, logger) val command = buildServerCommand(config) @@ -108,6 +104,17 @@ object BspServerOperations { // `ProjectDigest` (via scala.sys.process), and everything else without per-site plumbing. Use `putIfAbsent` so a developer who sets `NO_COLOR=` empty in // their shell or wants color in some specific case still wins. pb.environment().putIfAbsent("NO_COLOR", "1"): Unit + // Whatever bleep-specific variables the SPAWNING client happened to have, the daemon must not + // silently inherit. `pb.environment()` starts as a copy of this process's environment, so a + // `BLEEP_FOO=… bleep compile` becomes permanent state on a server that then serves every other + // client and every fork it starts — the first invocation of the day quietly configures the + // machine. Observed exactly that: one `BLEEP_PARALLELISM=3 bleep compile` pinned the value into + // a shared daemon, and unrelated test forks inherited it minutes later. + // + // Anything the daemon should know is passed deliberately — on the command line, or in the + // config it reads for itself. Non-bleep variables (PATH, HOME, JAVA_HOME, proxy settings) are + // left alone: those describe the machine, which the daemon genuinely shares. + pb.environment().keySet().removeIf(k => k.startsWith("BLEEP_") && k != "BLEEP_BSP_DEBUG") // Redirect stderr (where our logger writes) to the output file. // Discard stdout — libraries (Bloop, Zinc) dump massive // amounts of data to stdout which would bloat the log to tens of GB. @@ -397,15 +404,139 @@ object BspServerOperations { ProcessHandle.of(pid).isPresent } - /** Cleanup socket directory (lock, pid, socket, output files) */ + /** Cleanup socket directory: remove the runtime state, KEEP the log. + * + * `output` used to be deleted here, which meant a crash destroyed its own evidence: the client's crash handler calls `forceStop` before restarting, so by + * the time anyone read the log it was gone. Only the OOM marker survived, having been grepped out beforehand — so a crash that was NOT an OutOfMemoryError + * (a hard VM exit, a killed process) left nothing at all to look at. Reported from the field exactly that way: "crashed twice ... no OutOfMemoryError + * recorded", with the log already replaced by the next boot's. + * + * The log is rotated at the next spawn instead, which is where the generations are bounded. + */ def cleanup(socketDir: Path): IO[Unit] = IO.blocking { - val filesToDelete = Seq("lock", "pid", "socket", "output") + val filesToDelete = Seq("lock", "pid", "socket") filesToDelete.foreach { name => try Files.deleteIfExists(socketDir.resolve(name)) catch { case _: Exception => () } } } + /** How many previous server logs to keep beside the live one. + * + * Two, not one. The client retries a crashed server once before giving up, so a systematic crash produces `crash -> restart -> crash` and TWO rotations in + * seconds. With a single `output.prev` the second rotation overwrites the first crash's log with the second boot's — and the second boot is the short, + * uninformative one. Two generations mean the original crash survives the retry that was supposed to help diagnose it. + * + * Not more than two, because these accumulate per socket directory and every bleep version mints a new one (the version is part of the JVM key). + */ + val OutputGenerationsKept: Int = 2 + + /** Total bytes of retained logs allowed in one socket directory. + * + * Bounds the generations, not the live file: a running server holds `output` open through an OS-level redirect, so nothing outside it can rotate or truncate + * that file mid-boot. What CAN be bounded is history, and that is what accumulates — a day of ordinary use is under a megabyte at the default log level, but + * a daemon run with `BLEEP_BSP_DEBUG` traces every event and grows without limit. + * + * When the kept generations exceed this, the oldest are dropped until they fit. So the guarantee is "a socket directory costs at most this much history", + * which is the number that matters when there is one directory per bleep version per JVM. + */ + val MaxRetainedLogBytes: Long = 32L * 1024 * 1024 + + /** A socket directory whose daemon is gone and which nothing has touched for this long is litter. + * + * They accumulate silently: the JVM key includes the bleep version, so every snapshot install mints a fresh directory and abandons the previous one, each + * holding a log, its generations, and up to 200MB of metrics. Two weeks is well past the point where an old version's logs are of any use, and far enough + * out that a machine used intermittently never loses a directory it was still going to use. + */ + val StaleSocketDirMaxAgeDays: Long = 14 + + /** Shift `output` -> `output.1` -> `output.2`, dropping what falls off the end. + * + * Called at spawn, when nothing holds the file open. + */ + private def rotateOutput(socketDir: Path): Unit = + try { + val live = socketDir.resolve("output") + // Nothing to rotate on the very first boot, and nothing gained by shifting an empty file. + if (Files.exists(live) && Files.size(live) == 0L) Files.deleteIfExists(live): Unit + // Retire the oldest first, then shift each generation down, so no rename overwrites a file we + // still want. Going in reverse is what makes this safe without temporary names. + Files.deleteIfExists(socketDir.resolve(s"output.$OutputGenerationsKept")): Unit + (OutputGenerationsKept - 1 to 1 by -1).foreach { n => + val from = socketDir.resolve(s"output.$n") + if (Files.exists(from)) Files.move(from, socketDir.resolve(s"output.${n + 1}"), StandardCopyOption.REPLACE_EXISTING): Unit + } + if (Files.exists(live)) Files.move(live, socketDir.resolve("output.1"), StandardCopyOption.REPLACE_EXISTING): Unit + // Legacy name from when there was exactly one generation; drop it so the directory does not + // keep a file nothing writes or reads any more. + Files.deleteIfExists(socketDir.resolve("output.prev")): Unit + enforceLogBudget(socketDir) + } catch { case _: Exception => () } + + /** Drop the oldest generations until the retained logs fit [[MaxRetainedLogBytes]]. */ + private def enforceLogBudget(socketDir: Path): Unit = { + val generations = (1 to OutputGenerationsKept).map(n => socketDir.resolve(s"output.$n")).filter(Files.exists(_)) + var total = generations.map(Files.size).sum + // Oldest first — the highest generation number is the furthest back. + generations.reverse.foreach { p => + if (total > MaxRetainedLogBytes) { + total -= Files.size(p) + Files.deleteIfExists(p): Unit + } + } + } + + /** Delete sibling socket directories whose daemon is gone and which nothing has touched recently. + * + * Run at spawn, because that is when a new directory is being added and the previous version's is being orphaned — the moment the tail grows is the right + * moment to trim it. A directory is only removed when its recorded pid is dead (or absent) AND every file in it is older than the cutoff, so a daemon that + * is merely idle, or one being started concurrently, is never touched. + */ + private def pruneStaleSocketDirs(currentSocketDir: Path, logger: Logger): Unit = + try { + val parent = currentSocketDir.getParent + val cutoff = System.currentTimeMillis() - StaleSocketDirMaxAgeDays * 24L * 60L * 60L * 1000L + val stream = Files.list(parent) + val siblings = + try stream.iterator().asScala.filter(Files.isDirectory(_)).filterNot(_.equals(currentSocketDir)).toList + finally stream.close() + + siblings.foreach { dir => + val alive = + try Files.exists(dir.resolve("pid")) && ProcessHandle.of(Files.readString(dir.resolve("pid")).trim.toLong).isPresent + catch { case _: Exception => false } + if (!alive) { + val newest = + try { + val s2 = Files.list(dir) + try s2.iterator().asScala.map(Files.getLastModifiedTime(_).toMillis).maxOption.getOrElse(0L) + finally s2.close() + } catch { case _: Exception => Long.MaxValue } // unreadable: leave it alone + if (newest < cutoff) { + val freed = directorySizeBytes(dir) + deleteRecursively(dir) + logger.info(s"Removed stale compile-server directory $dir (no live server, untouched for ${StaleSocketDirMaxAgeDays}+ days, ${freed / 1024}KB)") + } + } + } + } catch { case _: Exception => () } + + private def directorySizeBytes(dir: Path): Long = + try { + val stream = Files.walk(dir) + try stream.iterator().asScala.filter(Files.isRegularFile(_)).map(Files.size).sum + finally stream.close() + } catch { case _: Exception => 0L } + + private def deleteRecursively(dir: Path): Unit = + try { + val stream = Files.walk(dir) + val paths = + try stream.iterator().asScala.toList + finally stream.close() + paths.reverse.foreach(p => Files.deleteIfExists(p): Unit) + } catch { case _: Exception => () } + /** Ensure a directory exists with proper permissions */ private def ensureDirectoryExists(dir: Path): Unit = if (!Files.exists(dir)) { diff --git a/bleep-core/src/scala/bleep/bsp/SetupBleepBsp.scala b/bleep-core/src/scala/bleep/bsp/SetupBleepBsp.scala index 14bdb9f49..5aa2e0f65 100644 --- a/bleep-core/src/scala/bleep/bsp/SetupBleepBsp.scala +++ b/bleep-core/src/scala/bleep/bsp/SetupBleepBsp.scala @@ -121,6 +121,24 @@ object SetupBleepBsp { private def classpathCacheFile(userPaths: UserPaths): Path = userPaths.cacheDir.resolve(s"bleep-bsp-classpath-${model.BleepVersion.current.value}.txt") + /** What bleep already knows about its own server's dependency graph, applied when resolving it. + * + * zinc 2 brings `compiler-interface` 2.0.4, which evicts the 1.10.7 that `scala3-compiler` depends on. That POM declares `early-semver`, so the eviction is + * an error by default and bleep cannot install its own server. The two are in fact compatible — 2.0.4 is purely additive over 1.10.7: 150 classes, zero + * public or protected members removed — which is the same argument `bleep.yaml` makes for building bleep-bsp in the first place. A version scheme is + * declared per project and does not travel with the published artifact, so the client has to state it too. + * + * A scheme rather than `IgnoreEvictionErrors.Yes`: this is a claim about one library, and every other eviction in the server classpath should still be an + * error, because that classpath is bleep's own and a surprise there is a bug. + */ + private val ServerClasspathVersionSchemes: scala.collection.immutable.SortedSet[model.LibraryVersionScheme] = + scala.collection.immutable.SortedSet( + model.LibraryVersionScheme( + model.LibraryVersionScheme.VersionScheme.Always, + model.Dep.Java("org.scala-sbt", "compiler-interface", "always") + ) + ) + private def resolveServerClasspath(resolver: CoursierResolver, userPaths: UserPaths, logger: Logger): Either[BleepException, Seq[Path]] = { val cacheFile = classpathCacheFile(userPaths) val cachedClasspath = Try { @@ -150,7 +168,7 @@ object SetupBleepBsp { .resolve( Set(dep), versionCombo, - libraryVersionSchemes = scala.collection.immutable.SortedSet.empty[model.LibraryVersionScheme], + libraryVersionSchemes = ServerClasspathVersionSchemes, model.IgnoreEvictionErrors.No ) match { case Left(err) => diff --git a/bleep-core/src/scala/bleep/commands/ReactiveBsp.scala b/bleep-core/src/scala/bleep/commands/ReactiveBsp.scala index 6f51b7546..4139632ab 100644 --- a/bleep-core/src/scala/bleep/commands/ReactiveBsp.scala +++ b/bleep-core/src/scala/bleep/commands/ReactiveBsp.scala @@ -396,13 +396,16 @@ case class ReactiveBsp( case ReactiveBsp.BspAttemptResult.ServerCrashed => // Two crashes in a row is systematic, not transient — fail the command instead // of looping (or worse, reporting a summary from a half-run build). - BspRifle.oomCrashExplanation(config).flatMap { oomSecond => - val msg = oomSecond.orElse(oomFirst) match { - case Some(oom) => s"BSP server crashed twice. $oom" - case None => s"BSP server crashed twice (no OutOfMemoryError recorded; see server log: ${BspRifle.getOutputFile(config)})" + // The run did not complete. Say so in the summary too, so a reader is not left + // reconciling a clean-looking block against the failure printed under it. + display.markServerCrashed >> + BspRifle.oomCrashExplanation(config).flatMap { oomSecond => + val msg = oomSecond.orElse(oomFirst) match { + case Some(oom) => s"BSP server crashed twice. $oom" + case None => s"BSP server crashed twice (no OutOfMemoryError recorded; see server log: ${BspRifle.getOutputFile(config)})" + } + IO.raiseError(new BleepException.Text(msg)) } - IO.raiseError(new BleepException.Text(msg)) - } } } } diff --git a/bleep-core/src/scala/bleep/testing/BuildDisplay.scala b/bleep-core/src/scala/bleep/testing/BuildDisplay.scala index 9ce641a97..e1f5127ff 100644 --- a/bleep-core/src/scala/bleep/testing/BuildDisplay.scala +++ b/bleep-core/src/scala/bleep/testing/BuildDisplay.scala @@ -28,6 +28,9 @@ trait BuildDisplay { /** Reset display state (e.g., before retry after server crash) */ def reset: IO[Unit] + /** Record that the compile server died, so the summary says the run did not complete rather than reading as clean next to a failure message. */ + def markServerCrashed: IO[Unit] + /** Print final summary. Pass a [[FilterContext]] when test filters are active so the summary can show which filters ran and what they pruned. */ def printSummary(filterContext: Option[FilterContext]): IO[Unit] } @@ -87,12 +90,18 @@ case class BuildSummary( durationMs: Long, totalTaskTimeMs: Long, // Sum of all individual task durations (compile + link + test, for parallelism stats) wasCancelled: Boolean, + /** The compile server died mid-run. The compiles that completed really did complete, so the counts stay honest — but the run did not finish, and a summary + * that reads as clean while the command fails is a summary the reader has to reconcile against the error below it. + */ + serverCrashed: Boolean, filterContext: Option[FilterContext] ) { /** Convert this summary to Either — Left for cancelled/failed builds, Right for success. Use this to gate post-build steps (publishing, etc.) */ def toEither: Either[bleep.BleepException, Unit] = - if (wasCancelled || compilesCancelled > 0) + if (serverCrashed) + Left(new bleep.BleepException.Text("Build did not complete: the compile server crashed")) + else if (wasCancelled || compilesCancelled > 0) Left(new bleep.BleepException.Text("Build cancelled by user")) else if (compileFailures.nonEmpty) Left(new bleep.BleepException.Text(s"Build failed: ${compileFailures.size} project(s) failed to compile")) @@ -137,7 +146,8 @@ object BuildSummary { // Anything other than passed/skipped/ignored means failure val totalProblems = summary.testsFailed + summary.testsTimedOut + summary.testsCancelled val hasFailures = - summary.sourcegenFailed > 0 || summary.compileFailures.nonEmpty || summary.linkFailures.nonEmpty || totalProblems > 0 || summary.suitesCancelled > 0 + summary.sourcegenFailed > 0 || summary.compileFailures.nonEmpty || summary.linkFailures.nonEmpty || totalProblems > 0 || summary.suitesCancelled > 0 || + summary.serverCrashed val wasCancelled = summary.wasCancelled || summary.compilesCancelled > 0 val statusColor = if (hasFailures) C.RED else if (wasCancelled) C.YELLOW else C.GREEN val statusIcon = if (hasFailures) "x" else if (wasCancelled) "!" else "✓" @@ -512,6 +522,7 @@ object BuildSummary { durationMs = 0L, totalTaskTimeMs = 0L, wasCancelled = false, + serverCrashed = false, filterContext = None ) } @@ -606,6 +617,8 @@ object BuildDisplay { upToDateProjects: Ref[IO, Set[CrossProjectName]] ) extends BuildDisplay { + override def markServerCrashed: IO[Unit] = state.update(_.copy(serverCrashed = true)) + override def reset: IO[Unit] = state.set(BuildState.empty) >> IO { activePhase.clear() } @@ -917,14 +930,18 @@ object BuildDisplay { // run that did not succeed. Fold them in, so the header and counts tell the same story the // exit code does. Reported for a failed sourcegen showing as `0 failed, 4 skipped`. preCompileFailed = s.sourcegenFailed + s.apResolutionFailed + s.kspResolutionFailed - anyFailure = s.compileFailures.nonEmpty || s.linkFailures.nonEmpty || preCompileFailed > 0 + anyFailure = s.compileFailures.nonEmpty || s.linkFailures.nonEmpty || preCompileFailed > 0 || s.serverCrashed _ <- log("") _ <- log("=" * 60) _ <- log(if (anyFailure) "Build Summary — FAILED" else "Build Summary") _ <- log("=" * 60) failedCount = s.compileFailures.size skippedCount = s.skippedProjects.size - _ <- log(s"Projects: compiled, $failedCount failed, $skippedCount skipped") + _ <- log(s"Projects: ${s.compilesCompleted} compiled, $failedCount failed, $skippedCount skipped") + _ <- + if (s.serverCrashed) + log("Server: crashed mid-run — the counts above are what finished before it died, not the whole build") + else IO.unit _ <- if (s.sourcegenFailed > 0) log(s"Sourcegen: ${s.sourcegenFailed} script(s) failed — see the errors above and the BSP server log") else IO.unit @@ -1051,6 +1068,8 @@ object BuildDisplay { currentTestResults: Ref[IO, List[BuildEvent.TestFinished]] ) extends BuildDisplay { + override def markServerCrashed: IO[Unit] = state.update(_.copy(serverCrashed = true)) + override def reset: IO[Unit] = state.set(BuildState.empty) >> currentTestResults.set(Nil) private def log(msg: String): IO[Unit] = IO.delay(logger.info(msg)) @@ -1178,6 +1197,8 @@ object BuildDisplay { startTime: Long ) extends BuildDisplay { + override def markServerCrashed: IO[Unit] = state.update(_.copy(serverCrashed = true)) + override def reset: IO[Unit] = state.set(BuildState.empty) override def handle(event: BuildEvent): IO[Unit] = diff --git a/bleep-core/src/scala/bleep/testing/BuildState.scala b/bleep-core/src/scala/bleep/testing/BuildState.scala index 9b96bdd60..7332518e9 100644 --- a/bleep-core/src/scala/bleep/testing/BuildState.scala +++ b/bleep-core/src/scala/bleep/testing/BuildState.scala @@ -44,7 +44,11 @@ case class BuildState( compileFailures: List[ProjectCompileFailure], skippedProjects: List[SkippedProject], pendingOutput: Map[SuiteKey, List[String]], - totalTaskTimeMs: Long + totalTaskTimeMs: Long, + /** The compile server died mid-run. Not a compile failure — the compiles that finished really did finish — but the run did not complete, so a summary that + * looked clean while the command failed left the reader to reconcile two blocks that seemed to disagree. + */ + serverCrashed: Boolean ) { /** Project to BuildSummary (lists are reversed since we prepend during accumulation) */ @@ -91,6 +95,7 @@ case class BuildState( durationMs = durationMs, totalTaskTimeMs = totalTaskTimeMs, wasCancelled = wasCancelled, + serverCrashed = serverCrashed, filterContext = None ) } @@ -133,7 +138,8 @@ object BuildState { compileFailures = Nil, skippedProjects = Nil, pendingOutput = Map.empty, - totalTaskTimeMs = 0 + totalTaskTimeMs = 0, + serverCrashed = false ) } diff --git a/bleep-core/src/scala/bleep/testing/ReactiveTestRunner.scala b/bleep-core/src/scala/bleep/testing/ReactiveTestRunner.scala deleted file mode 100644 index f9c7c8b82..000000000 --- a/bleep-core/src/scala/bleep/testing/ReactiveTestRunner.scala +++ /dev/null @@ -1,547 +0,0 @@ -package bleep.testing - -import bleep.MachineResources -import bleep.bsp.protocol.{OutputChannel, SuiteOutcome, TestStatus} -import bleep.bsp.BuildServer -import bleep.{model, Started} -import bleep.model.{CrossProjectName, SuiteName, TestName} -import cats.effect._ -import cats.syntax.all._ -import ch.epfl.scala.bsp4j -import coursier._ - -import java.nio.file.{Files, Path, Paths} -import scala.concurrent.duration._ - -/** Reactive test runner that starts tests as soon as their dependencies compile. - * - * Key features: - * - Runs tests in parallel as soon as they're ready - * - Reuses forked JVMs for performance - * - Streams real-time progress to display - * - Supports cancellation via interrupt - */ -object ReactiveTestRunner { - - /** Options for the test runner */ - case class Options( - jvmOptions: List[String], - testArgs: List[String], - maxParallelJvms: Int, - quietMode: Boolean, - idleTimeout: FiniteDuration - ) - - object Options { - val default: Options = Options( - jvmOptions = Nil, - testArgs = Nil, - maxParallelJvms = Runtime.getRuntime.availableProcessors(), - quietMode = false, - idleTimeout = 2.minutes - ) - } - - /** Result of running tests */ - case class Result( - passed: Int, - failed: Int, - skipped: Int, - ignored: Int, - failedSuites: List[SuiteName], - durationMs: Long - ) { - def success: Boolean = failed == 0 - } - - /** Run tests for the given projects. - * - * @param started - * The bleep Started context - * @param server - * The BSP build server - * @param testProjects - * Projects to test (must be test projects) - * @param options - * Test runner options - * @return - * Test results - */ - def run( - started: Started, - server: BuildServer, - testProjects: Set[CrossProjectName], - options: Options - ): IO[Result] = - - // Standalone (single-process) runner. The governor describes the MACHINE — cores and fork-memory - // budget — exactly as it does in the daemon; `maxParallelJvms` is the user's cap on this one run - // and belongs on the pool's own semaphore, not smuggled in as the machine's core count. Whichever - // is lower ends up binding, which is the intent of both. - JvmPool - .create( - options.maxParallelJvms, - started.jvmCommand, - started.buildPaths.buildDir, - MachineResources.forThisMachine(totalCpu = Runtime.getRuntime.availableProcessors(), logger = started.logger) - ) - .use { pool => - for { - display <- BuildDisplay.create(options.quietMode, started.logger) - - // Build target lookup - buildTargetFor = (p: CrossProjectName) => - new bsp4j.BuildTargetIdentifier( - started.projectPaths(p).targetDir.toUri.toASCIIString - ) - - // Discover all test suites - suites <- IO( - TestDiscovery.discoverAll( - server, - testProjects, - buildTargetFor - ) - ) - - _ <- IO.delay(started.logger.info(s"Discovered ${suites.size} test suites in ${testProjects.size} projects")) - - // Group suites by project for classpath sharing - suitesByProject = suites.groupBy(_.project) - - // Run all suites in parallel, bounded by JVM pool - result <- { - val runSuites = suitesByProject.toList.parTraverse { case (project, projectSuites) => - runProjectSuites( - started = started, - project = project, - suites = projectSuites, - pool = pool, - display = display, - options = options - ) - } - - runSuites.flatMap { _ => - for { - // ReactiveTestRunner is the legacy non-BSP path; it doesn't expose filter flags at this layer, so no FilterContext. - _ <- display.printSummary(None) - summary <- display.summary - } yield Result( - passed = summary.testsPassed, - failed = summary.testsFailed, - skipped = summary.testsSkipped, - ignored = summary.testsIgnored, - failedSuites = summary.failures.map(_.suite).distinct, - durationMs = summary.durationMs - ) - } - } - } yield result - } - - private def runProjectSuites( - started: Started, - project: CrossProjectName, - suites: List[DiscoveredSuite], - pool: JvmPool, - display: BuildDisplay, - options: Options - ): IO[Unit] = { - val classpath = getClasspath(started, project) - - // The ForkedTestRunner class must be on the classpath. It's in the bleep-test-runner project which has minimal dependencies. - val runnerClass = "bleep.testing.runner.ForkedTestRunner" - - val environment = started.build.explodedProjects.get(project).flatMap(_.platform).map(_.jvmEnvironment.toMap).getOrElse(Map.empty) - val projectDir = started.build.explodedProjects.get(project).flatMap(_.folder).map(rp => started.buildPaths.buildDir.resolve(rp.toString)) - - // Each suite acquires its own JVM from the pool, allowing the pool's max-concurrency semaphore to parallelize suites instead of serialising them on a single - // shared JVM. A failure in one suite (e.g. the forked JVM dying) only fails that suite; siblings keep running on their own JVMs. The pool reuses healthy - // JVMs across suites (warm classloader, same classpath hash), drops dead ones in `release`. - suites.parTraverse_ { suite => - pool.acquire(suite.className, classpath, options.jvmOptions, runnerClass, environment, projectDir).use { jvm => - runSuite( - projectName = project, - suite = suite, - jvm = jvm, - display = display, - testArgs = options.testArgs, - idleTimeout = options.idleTimeout, - logger = started.logger - ) - } - } - } - - /** Result of processing test responses */ - private case class SuiteResult( - suiteFinishedSent: Boolean, - passed: Int, - failed: Int, - skipped: Int, - ignored: Int, - errorMessage: Option[String] - ) - - /** Run a single test suite on the given JVM */ - def runSuite( - projectName: CrossProjectName, - suite: DiscoveredSuite, - jvm: TestJvm, - display: BuildDisplay, - testArgs: List[String], - idleTimeout: FiniteDuration, - logger: ryddig.Logger - ): IO[Unit] = { - - def debugLog(msg: String): Unit = logger.debug(s"[RUNNER] $msg") - - val suiteKey = s"$projectName/${suite.className}" - - // Track test counts and whether SuiteFinished was sent - // lastActivityAt is reset on each TestFinished — the idle timeout fires only when - // no test has completed for `idleTimeout` duration. - def processResponses(lastActivityAt: Ref[IO, Long]): IO[SuiteResult] = for { - responses <- jvm.runSuite(suite.className, suite.framework, testArgs).compile.toList - _ <- IO(debugLog(s"Suite: ${suite.className}, framework: ${suite.framework}, responses: ${responses.size}")) - _ <- IO(responses.foreach(r => debugLog(s" Response: $r"))) - suiteFinishedSent <- Ref.of[IO, Boolean](false) - passedCount <- Ref.of[IO, Int](0) - failedCount <- Ref.of[IO, Int](0) - skippedCount <- Ref.of[IO, Int](0) - ignoredCount <- Ref.of[IO, Int](0) - // First Error response wins — captures e.g. "Forked JVM died" as the SuiteError reason without losing subsequent diagnostic Output lines. - firstError <- Ref.of[IO, Option[String]](None) - - // Convert responses to events - _ <- responses.traverse_ { - case TestProtocol.TestResponse.TestStarted(_, test) => - IO.realTime.map(_.toMillis).flatMap(now => display.handle(BuildEvent.TestStarted(projectName, SuiteName(suite.className), TestName(test), now))) - - case TestProtocol.TestResponse.TestFinished(_, test, status, durationMs, message, throwable) => - val testStatus = TestStatus.fromString(status) - val updateCount = testStatus match { - case TestStatus.Passed => passedCount.update(_ + 1) - case TestStatus.Failed | TestStatus.Error | TestStatus.Timeout => failedCount.update(_ + 1) - case TestStatus.Skipped | TestStatus.AssumptionFailed => skippedCount.update(_ + 1) - case TestStatus.Cancelled => failedCount.update(_ + 1) - case TestStatus.Ignored | TestStatus.Pending => ignoredCount.update(_ + 1) - } - updateCount >> IO.realTime - .map(_.toMillis) - .flatMap { now => - // Reset idle timeout on each test completion - lastActivityAt.set(now) >> - display.handle( - BuildEvent.TestFinished(projectName, SuiteName(suite.className), TestName(test), testStatus, durationMs, message, throwable, now) - ) - } - - case TestProtocol.TestResponse.SuiteDone(_, outcome, durationMs) => - IO.realTime - .map(_.toMillis) - .flatMap(now => - display.handle( - BuildEvent.SuiteFinished(projectName, SuiteName(suite.className), outcome, durationMs, now) - ) - ) >> suiteFinishedSent.set(true) - - case TestProtocol.TestResponse.Log(level, message, logSuite) => - val channel = if (level == "error" || level == "warn") OutputChannel.Stderr else OutputChannel.Stdout - val effectiveSuite = logSuite.getOrElse(suite.className) - IO.realTime - .map(_.toMillis) - .flatMap(now => display.handle(BuildEvent.Output(projectName, SuiteName(effectiveSuite), s"[$level] $message", channel, now))) - - case TestProtocol.TestResponse.Error(message, throwable) => - firstError.update(_.orElse(Some(message))) >> - IO.realTime - .map(_.toMillis) - .flatMap(now => - display.handle(BuildEvent.Output(projectName, SuiteName(suite.className), s"[ERROR] $message", OutputChannel.Stderr, now)) >> - throwable.traverse_(t => display.handle(BuildEvent.Output(projectName, SuiteName(suite.className), t, OutputChannel.Stderr, now))) - ) - - case TestProtocol.TestResponse.Ready => - IO.unit // Ignore Ready messages during suite execution - - case TestProtocol.TestResponse.ThreadDump(_) => - IO.unit // Ignore ThreadDump during normal suite execution (used for idle timeout handling) - } - sent <- suiteFinishedSent.get - passed <- passedCount.get - failed <- failedCount.get - skipped <- skippedCount.get - ignored <- ignoredCount.get - err <- firstError.get - } yield SuiteResult(sent, passed, failed, skipped, ignored, err) - - def handleTimeout: IO[Unit] = for { - _ <- IO(debugLog(s"IDLE TIMEOUT TRIGGERED for suite: $suiteKey")) - endTime <- IO.realTime.map(_.toMillis) - _ <- IO(debugLog(s"Getting thread dump for $suiteKey (3s timeout)...")) - // Try to get thread dump with 3 second timeout using racePair - // Can't use .timeoutTo because blocking IO can't be interrupted - threadDumpResult <- IO.racePair(jvm.getThreadDump, IO.sleep(3.seconds)) - threadDump <- threadDumpResult match { - case Left((outcome, timerFiber)) => - // Thread dump completed - cancel timer and get result - timerFiber.cancel >> outcome.embedError.flatMap { dump => - IO(debugLog(s"Thread dump received for $suiteKey: ${dump.map(d => s"${d.threads.size} threads").getOrElse("None")}")).as(dump) - } - case Right((dumpFiber, _)) => - // Timer won - abandon thread dump attempt (don't wait for it) - IO(debugLog(s"Thread dump timed out for $suiteKey, proceeding to kill JVM")) >> - IO.pure(None) - // Note: we intentionally don't cancel dumpFiber here because it's blocked on readLine - // and can't be cancelled anyway. It will terminate when we kill the JVM. - } - threadDumpInfo <- IO(processThreadDump(projectName, SuiteName(suite.className), threadDump)) - _ <- IO(debugLog(s"Killing JVM for $suiteKey...")) - _ <- jvm.kill - _ <- IO(debugLog(s"JVM killed for $suiteKey, sending SuiteTimedOut event")) - _ <- display.handle( - BuildEvent.SuiteTimedOut(projectName, SuiteName(suite.className), idleTimeout.toMillis, threadDumpInfo, endTime) - ) - _ <- IO(debugLog(s"IDLE TIMEOUT COMPLETE for $suiteKey")) - } yield () - - // Idle timeout: polls lastActivityAt every second and fires when no activity for idleTimeout duration - def idleTimeoutFiber(lastActivityAt: Ref[IO, Long]): IO[Unit] = { - val checkInterval = 1.second - def loop: IO[Unit] = for { - now <- IO.realTime.map(_.toMillis) - lastActivity <- lastActivityAt.get - elapsed = now - lastActivity - _ <- - if (elapsed >= idleTimeout.toMillis) { - IO(debugLog(s"Idle timeout fired for $suiteKey (no activity for ${elapsed}ms)")) - } else { - IO.sleep(checkInterval) >> loop - } - } yield () - loop - } - - for { - startTime <- IO.realTime.map(_.toMillis) - lastActivityAt <- Ref.of[IO, Long](startTime) - _ <- display.handle(BuildEvent.SuiteStarted(projectName, SuiteName(suite.className), startTime)) - _ <- IO(debugLog(s"Starting race: processResponses vs idleTimeout(${idleTimeout.toSeconds}s) for $suiteKey")) - // Use racePair instead of race - race waits for cancellation of the loser, - // but blocking IO (readLine) can't be cancelled, so race would hang. - // racePair returns immediately when one side wins, giving us the fiber to cancel later. - raceResult <- IO.racePair( - processResponses(lastActivityAt), - idleTimeoutFiber(lastActivityAt) - ) - _ <- IO(debugLog(s"RacePair completed for $suiteKey")) - endTime <- IO.realTime.map(_.toMillis) - _ <- raceResult match { - case Left((outcome, timerFiber)) => - // processResponses won - cancel the timer (instant) and process result - timerFiber.cancel >> outcome.embedError.flatMap { result => - IO(debugLog(s"Normal completion for $suiteKey: sent=${result.suiteFinishedSent}, error=${result.errorMessage.isDefined}")) >> - ( - if (result.suiteFinishedSent) IO.unit - else - result.errorMessage match { - // The JVM either died mid-suite or sent a malformed protocol response, and `SuiteDone` never arrived. Emit `SuiteError` so the suite shows - // up as a crash, not as `SuiteFinished(0,0,0,0)` (which would render as a spurious "PASSED: 0 passed, 0 failed"). - case Some(msg) => - display.handle( - BuildEvent.SuiteError( - projectName, - SuiteName(suite.className), - msg, - bleep.bsp.protocol.ProcessExit.Unknown, - endTime - startTime, - endTime - ) - ) - case None => - display.handle( - BuildEvent.SuiteFinished( - projectName, - SuiteName(suite.className), - SuiteOutcome.fromCounts(result.passed, result.failed, result.skipped, result.ignored), - endTime - startTime, - endTime - ) - ) - } - ) - } - case Right((responseFiber, _)) => - // Timeout won - kill JVM first (which will unblock readLine), then cancel fiber - IO(debugLog(s"Idle timeout won for $suiteKey, handling...")) >> - handleTimeout >> - responseFiber.cancel // This should now complete quickly since JVM is dead - } - } yield () - } - - /** Process thread dump and create ThreadDumpInfo */ - private def processThreadDump( - projectName: CrossProjectName, - suiteName: SuiteName, - threadDump: Option[TestProtocol.TestResponse.ThreadDump] - ): Option[ThreadDumpInfo] = - threadDump.map { dump => - // Filter for "active" threads (not waiting/timed_waiting on common things) - val activeThreads = dump.threads.filter { t => - val state = t.state - // Consider RUNNABLE threads and BLOCKED threads as active - // Exclude WAITING and TIMED_WAITING on common patterns - state == "RUNNABLE" || state == "BLOCKED" || { - // Include WAITING/TIMED_WAITING only if not on common wait patterns - val firstFrame = t.stackTrace.headOption.getOrElse("") - !firstFrame.contains("Object.wait") && - !firstFrame.contains("LockSupport.park") && - !firstFrame.contains("Thread.sleep") && - !firstFrame.contains("SocketInputStream.read") && - !firstFrame.contains("BufferedInputStream.read") - } - } - - if (activeThreads.size == 1) { - // Single active thread - show inline - val thread = activeThreads.head - val stackPreview = thread.stackTrace.take(15).mkString("\n at ") - val preview = s"Thread: ${thread.name} (${thread.state})\n at $stackPreview" - ThreadDumpInfo( - activeThreadCount = 1, - singleThreadStack = Some(preview), - dumpFile = None - ) - } else { - // Multiple threads - write to temp file - val timestamp = java.time.Instant.now().toString.replace(":", "-") - val safeSuiteName = suiteName.value.replaceAll("[^a-zA-Z0-9.-]", "_") - val fileName = s"thread-dump-$safeSuiteName-$timestamp.txt" - val dumpPath = Paths.get(System.getProperty("java.io.tmpdir"), "bleep-test-dumps", fileName) - - try { - Files.createDirectories(dumpPath.getParent) - val content = new StringBuilder() - content.append(s"Thread dump for suite: ${suiteName.value}\n") - content.append(s"Project: ${projectName.value}\n") - content.append(s"Time: ${java.time.Instant.now()}\n") - content.append(s"Total threads: ${dump.threads.size}\n") - content.append(s"Active threads: ${activeThreads.size}\n") - content.append("\n" + "=" * 80 + "\n\n") - - dump.threads.foreach { thread => - content.append(s"Thread: ${thread.name}\n") - content.append(s"State: ${thread.state}\n") - thread.stackTrace.foreach(line => content.append(s" at $line\n")) - content.append("\n") - } - - Files.write(dumpPath, content.toString.getBytes) - - ThreadDumpInfo( - activeThreadCount = activeThreads.size, - singleThreadStack = None, - dumpFile = Some(dumpPath) - ) - } catch { - case e: Exception => - // Failed to write file, include preview in single thread stack - val preview = activeThreads - .take(3) - .map { t => - s"Thread: ${t.name} (${t.state})\n at ${t.stackTrace.take(5).mkString("\n at ")}" - } - .mkString("\n\n") - ThreadDumpInfo( - activeThreadCount = activeThreads.size, - singleThreadStack = Some(s"(Failed to write dump file: ${e.getMessage})\n$preview"), - dumpFile = None - ) - } - } - } - - /** Get the full classpath for a project including dependencies */ - def getClasspath(started: Started, project: CrossProjectName): List[Path] = { - val logger = started.logger - def debugLog(msg: String): Unit = logger.debug(s"[CLASSPATH] $msg") - - // Get the full classpath for the project including dependencies - val projectPaths = started.projectPaths(project) - val classesDir = projectPaths.classes - - val resolvedProject = started.resolvedProject(project) - val dependencyClasspath = resolvedProject.classpath.map(p => Path.of(p.toString)) - - // Find bleep-test-runner classes for ForkedTestRunner. - // First try from the current build (when running bleep's own tests), - // otherwise always fetch via coursier for consistency - val testRunnerFromBuild = started.build.explodedProjects.keys - .find(p => p.name.value == "bleep-test-runner") - .map(p => started.projectPaths(p).classes) - - debugLog(s"testRunnerFromBuild: $testRunnerFromBuild") - - val testRunnerClasses = testRunnerFromBuild match { - case Some(path) => - debugLog(s"Using build path: $path") - List(path) - case None => - debugLog("Fetching via coursier...") - fetchTestRunnerViaCoursier(logger) - } - - // Verify we actually got the test runner - if (testRunnerClasses.isEmpty) { - throw new RuntimeException( - "Failed to find bleep-test-runner classes. " + - "This is required for running tests with the reactive test runner." - ) - } - - (classesDir :: dependencyClasspath) ++ testRunnerClasses - } - - /** Fetch bleep-test-runner via coursier */ - private def fetchTestRunnerViaCoursier(logger: ryddig.Logger): List[Path] = { - def debugLog(msg: String): Unit = logger.debug(s"[CLASSPATH] $msg") - val version = model.BleepVersion.current.value - debugLog(s"Fetching bleep-test-runner:$version via coursier") - val dep = Dependency( - Module(Organization("build.bleep"), ModuleName("bleep-test-runner")), - version - ) - - val fetch = Fetch() - .addDependencies(dep) - .addRepositories( - Repositories.central, - Repositories.sonatype("snapshots"), - coursier.LocalRepositories.ivy2Local - ) - - try { - val files = fetch.run() - if (files.isEmpty) { - throw new RuntimeException( - s"bleep-test-runner resolution returned no jars for version $version" - ) - } - val paths = files.map(f => Path.of(f.getAbsolutePath)).toList - debugLog(s"Using jars: ${paths.mkString(", ")}") - paths - } catch { - case e: Exception => - throw new RuntimeException( - s"Failed to fetch bleep-test-runner:$version via coursier. " + - "This is needed when running bleep as a native image. " + - s"Error: ${e.getMessage}", - e - ) - } - } - -} diff --git a/bleep-model/src/scala/bleep/BuildPaths.scala b/bleep-model/src/scala/bleep/BuildPaths.scala index bde7c8f5c..cbf4c8331 100644 --- a/bleep-model/src/scala/bleep/BuildPaths.scala +++ b/bleep-model/src/scala/bleep/BuildPaths.scala @@ -37,6 +37,11 @@ case class BuildPaths(cwd: Path, bleepYamlFile: Path, variant: model.BuildVarian lazy val bspBleepJsonFile: Path = buildDir / ".bsp" / "bleep.json" lazy val dotBleepDir: Path = buildDir / ".bleep" + /** What a compile server keys its per-build state by. Derived in one place so that everything caching against a build — the resolved `Started`, the Zinc + * analyses read while compiling it — agrees on what "the same build" means, and dropping one can drop the other. + */ + lazy val workspaceKey: model.WorkspaceKey = model.WorkspaceKey(buildDir, variant) + // === workspace-level paths (project-agnostic) === /** `/.bleep/projects/`. Holds one subdirectory per cross-built project. */ diff --git a/bleep-model/src/scala/bleep/model/BleepConfig.scala b/bleep-model/src/scala/bleep/model/BleepConfig.scala index 48cf1cc40..b8c69ada7 100644 --- a/bleep-model/src/scala/bleep/model/BleepConfig.scala +++ b/bleep-model/src/scala/bleep/model/BleepConfig.scala @@ -29,9 +29,14 @@ object BleepConfig { /** BSP server configuration - applies to compile and test operations */ case class BspServerConfig( - /** Maximum parallelism. If None, uses parallelismRatio or defaults to all cores */ + /** How many operations bleep may run at once on this machine — compiles, test forks, sourcegen. The single concurrency knob; everything else is derived + * from it. Defaults to all cores. + * + * Machine-wide, not per invocation: the compile server is shared, reads this once at startup, and passes it to every fork. A per-client override would + * mean whichever client happened to spawn the daemon silently configured it for everyone else. + */ parallelism: Option[Int], - /** Parallelism as ratio of available cores (e.g., 0.5 = half). Used if parallelism is None */ + /** The same as a fraction of cores, e.g. `0.5` for half. Used when [[parallelism]] is unset. */ parallelismRatio: Option[Double], /** Idle timeout for test suites in minutes — resets each time a test completes */ testIdleTimeoutMinutes: Option[Int], @@ -59,7 +64,16 @@ case class BspServerConfig( * counts only fully-idle time: it resets whenever a client is connected, so a server mid-compile or serving an editor is never reaped. Set to 0 to disable * (stay alive forever). Default: 60 */ - compileServerIdleTimeoutMinutes: Option[Int] + compileServerIdleTimeoutMinutes: Option[Int], + /** How many workspaces' resolved builds [[bleep.bsp.BuildCache]] keeps before evicting the least recently used idle one. + * + * A resolved build is not small — exploded model, resolved classpaths, the Zinc analysis reachable from it — and the cache used to hold every workspace + * the daemon had ever seen for its whole lifetime. Measured on a daemon serving 11 worktrees: the post-GC floor climbed monotonically from 3.5GB to 8.2GB + * of a 12GB heap over 45 minutes, at which point compiles OOM'd at a concurrency of three. Bounding the cache bounds that floor. + * + * Only idle workspaces are evicted, so this is a cache size, not a limit on how many workspaces a daemon can serve. Default: 4. None = default. + */ + maxCachedWorkspaces: Option[Int] ) { def effectiveParallelism: Int = { val cores = Runtime.getRuntime.availableProcessors @@ -81,6 +95,13 @@ case class BspServerConfig( minutes * 60 * 1000 } + /** How many workspaces' builds the daemon caches. Always >= 1: the workspace being compiled has to stay cached while it compiles. */ + def effectiveMaxCachedWorkspaces: Int = { + val n = maxCachedWorkspaces.getOrElse(BspServerConfig.DefaultMaxCachedWorkspaces) + if (n < 1) sys.error(s"maxCachedWorkspaces must be >= 1, got $n") + n + } + /** How long a fully-idle server waits before self-shutdown, in milliseconds. 0 means never. */ def effectiveCompileServerIdleTimeoutMillis: Long = { val minutes = compileServerIdleTimeoutMinutes.getOrElse(BspServerConfig.DefaultCompileServerIdleTimeoutMinutes) @@ -104,6 +125,10 @@ object BspServerConfig { // connected), so this never interrupts a compile or a live editor. val DefaultCompileServerIdleTimeoutMinutes: Int = 60 + // Four builds is enough that an editor plus a couple of CLI worktrees all stay warm, and few enough that the retained floor stays a fraction of the heap + // rather than most of it. Evicted entries are reloaded on next use, so this trades a cold build load for headroom. + val DefaultMaxCachedWorkspaces: Int = 4 + val default: BspServerConfig = BspServerConfig( parallelism = None, parallelismRatio = None, @@ -114,7 +139,8 @@ object BspServerConfig { compileServerMaxMemory = None, heapPressureThreshold = None, bspReadTimeoutMinutes = None, - compileServerIdleTimeoutMinutes = None + compileServerIdleTimeoutMinutes = None, + maxCachedWorkspaces = None ) implicit val decoder: Decoder[BspServerConfig] = deriveDecoder diff --git a/bleep-model/src/scala/bleep/model/WorkspaceKey.scala b/bleep-model/src/scala/bleep/model/WorkspaceKey.scala new file mode 100644 index 000000000..6e53fab9d --- /dev/null +++ b/bleep-model/src/scala/bleep/model/WorkspaceKey.scala @@ -0,0 +1,22 @@ +package bleep.model + +import java.nio.file.Path + +/** Identity of one resolved build: a workspace root and the variant built within it. + * + * This is what a compile server's per-build state is keyed by, and it exists as a named type because two different caches key on it and they have to agree. + * `BuildCache` holds the resolved `Started`; `bleep.analysis.AnalysisCache` holds the Zinc analyses read while compiling it. Those analyses live on disk under + * `.bleep/projects//builds//.zinc/`, i.e. inside the workspace and partitioned by variant — exactly this key — so the association is not a + * convention, it is the layout. + * + * Keeping the two keyed alike is what lets dropping a build also drop its analyses. The reverse does not hold: analyses are cheap to re-read and expensive to + * hold, a resolved build is the other way round, so they are evicted on separate schedules. See `BuildCache` for that asymmetry. + */ +case class WorkspaceKey(workspace: Path, variant: BuildVariant) { + + /** Short rendering for logs and metrics: the workspace's own directory name plus the variant. */ + def short: String = { + val name = Option(workspace.getFileName).map(_.toString).getOrElse(workspace.toString) + s"$name/${variant.toString}" + } +} diff --git a/bleep-tests/src/scala/bleep/model/ResourceKnobsTest.scala b/bleep-tests/src/scala/bleep/model/ResourceKnobsTest.scala new file mode 100644 index 000000000..03f1cec4c --- /dev/null +++ b/bleep-tests/src/scala/bleep/model/ResourceKnobsTest.scala @@ -0,0 +1,55 @@ +package bleep.model + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +/** The concurrency knob and what derives from it. + * + * There is one — `parallelism` — because the previous surface had a separate `maxConcurrentCompiles`, so "how much of this machine may bleep use" had two + * answers that could disagree. It is machine-wide and read by the shared compile server at startup, which is why it lives in config and not in the + * environment: a per-client override would let whichever client spawned the daemon configure it for everyone else. + */ +class ResourceKnobsTest extends AnyFunSuite with Matchers { + + private val cores = Runtime.getRuntime.availableProcessors + + test("parallelism defaults to one per core — cores are the default, not a second limit") { + BspServerConfig.default.effectiveParallelism shouldBe cores + } + + test("an explicit parallelism replaces the core count everywhere it matters") { + // The governor's CPU axis is sized from this, not from availableProcessors. Reading cores there + // instead meant `parallelism = 2` bounded each run and each JVM pool at 2 while the governor + // still admitted one-per-core across every connected client — so a user who asked for two got two + // per client and many in total. + BspServerConfig.default.copy(parallelism = Some(2)).effectiveParallelism shouldBe 2 + BspServerConfig.default.copy(parallelism = Some(64)).effectiveParallelism shouldBe 64 + } + + test("compiles are not capped separately from everything else") { + // A fixed compile ceiling used to sit inside the governor, holding capacity back for test forks + // that may not exist — a static partition in a work-conserving system, so a compile-only run left + // half the machine idle. Heap pressure is answered by HeapPressureGate against the live heap. + BspServerConfig.getClass.getDeclaredMethods.map(_.getName) should not contain "effectiveMaxConcurrentCompiles" + classOf[BspServerConfig].getDeclaredFields.map(_.getName) should not contain "maxConcurrentCompiles" + } + + test("parallelismRatio expresses the same thing as a fraction of cores") { + BspServerConfig.default.copy(parallelismRatio = Some(0.5)).effectiveParallelism shouldBe math.max(1, cores / 2) + } + + test("an explicit parallelism beats the ratio") { + BspServerConfig.default.copy(parallelism = Some(3), parallelismRatio = Some(0.5)).effectiveParallelism shouldBe 3 + } + + test("no resource knob is settable from the environment") { + // Regression guard for a design mistake worth remembering: env overrides were added for these, + // and one `BLEEP_PARALLELISM=3 bleep compile` pinned the value into the shared daemon, which + // then served every other client — and every fork it spawned — at that concurrency. Anything the + // daemon reads has to be machine-wide, because the daemon is. + val fields = classOf[BspServerConfig].getDeclaredFields.map(_.getName).toSet + fields should contain("parallelism") + BspServerConfig.getClass.getDeclaredMethods.map(_.getName) should not contain "envString" + BspServerConfig.getClass.getDeclaredMethods.map(_.getName) should not contain "envInt" + } +} diff --git a/bleep.yaml b/bleep.yaml index d2a451816..17fbfa2f4 100644 --- a/bleep.yaml +++ b/bleep.yaml @@ -290,8 +290,12 @@ projects: - com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-core:2.32.0 - com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-macros:2.32.0 - org.typelevel::cats-effect:3.6.1 - - for3Use213: true - module: org.scala-sbt::zinc:1.12.0 + # zinc 2.x. compiler-interface 2.0.4 evicts the 1.10.7 that scala3-compiler asks for; coursier's + # early-semver rule flags that as suspect, but it is not: diffing the two jars shows 2.0.4 is + # purely additive — 150 classes, zero public or protected members removed, two classes added + # (HashedVirtualFileRef, BasicHashedVirtualFileRef). The Scala 3 bridge compiled against 1.10.7 + # links fine against it. + - org.scala-sbt::zinc:2.0.4 - for3Use213: true module: ch.epfl.scala::libdaemon:0.0.12 dependsOn: @@ -413,6 +417,16 @@ scripts: project: scripts-java templates: template-common: + # zinc 2.x pulls compiler-interface 2.0.4, evicting the 1.10.7 that scala3-compiler asks for. + # Its POM declares early-semver, so the eviction fails the build by default. The compatibility + # argument: diffing the two jars shows 2.0.4 is purely additive over 1.10.7 — 150 classes, zero + # public or protected members removed, two added (HashedVirtualFileRef, BasicHashedVirtualFileRef). + # + # On the TEMPLATE rather than on bleep-bsp, which is the project that actually pulls zinc, because + # THIS build must stay resolvable by the PREVIOUS bleep release: CI bootstraps with it, and it + # predates schemes travelling along `dependsOn`. Once a release carries that, this can move to + # bleep-bsp alone. + libraryVersionSchemes: org.scala-sbt:compiler-interface:always java: options: -proc:none platform: diff --git a/dedup-report.txt b/dedup-report.txt new file mode 100644 index 000000000..2fe7453d0 --- /dev/null +++ b/dedup-report.txt @@ -0,0 +1,6 @@ +report: /Users/oyvind/pr/bleep/worktrees/problems/dedup-report.txt +30 analysis files, 3.3 MB on disk, across 1 root(s) +14 file(s) could not be read +AnalyzedClass total=4716 distinct(no timestamp)=2184 → 2.16x sharing +AnalyzedClass total=4716 distinct(with timestamp)=2184 → 2.16x sharing +cost of keeping timestamp in the key: 0 extra retained instances \ No newline at end of file diff --git a/docs/guides/compile-servers.mdx b/docs/guides/compile-servers.mdx index b666a5b56..14bd46e60 100644 --- a/docs/guides/compile-servers.mdx +++ b/docs/guides/compile-servers.mdx @@ -60,6 +60,33 @@ need to reclaim memory before doing something else heavy. [Reference →](/docs/reference/cli/config/compile-server/) +## Idle self-shutdown + +A shared server that hasn't served a client for a while shuts +itself down, so servers left behind by closed worktrees, JVM +bumps, or upgraded bleep binaries don't pile up and hold memory +indefinitely. The default is one hour. + +```bash +bleep config compile-server idle-timeout 120 # minutes +bleep config compile-server idle-timeout 0 # never self-shut-down +bleep config compile-server idle-timeout-clear # back to default 60 +``` + +The clock measures **fully-idle** time only: it resets whenever a +client connects and stays reset for as long as any client is +connected. So a long compile, or an editor left open over lunch, +keeps the server alive — the timeout only fires once nothing has +been connected for the whole window. When it does fire, the server +releases its lock and removes its socket files, so the next `bleep` +command in that workspace just starts a fresh one. + +This is different from the [idle read timeout](#idle-read-timeout) +below: that drops a single stuck *connection*; this retires the +whole *server* once it has been unused for long enough. + +[Reference →](/docs/reference/cli/config/compile-server/) + ## Memory The compile server holds Zinc analysis, compiler state, and live @@ -89,24 +116,33 @@ bleep versions are deleted automatically at server start. [Reference →](/docs/reference/cli/config/compile-server/) -### Heap-pressure throttle +### How much runs at once + +Compiles, links, tests and sourcegen all draw from one budget: +**`parallelism`** — see +[Memory & Parallelism](/docs/usage/resource-management#how-many-run-at-once). + +There is deliberately no separate cap for compiles. One existed briefly and +was removed: holding capacity back for test forks that might not exist is a +static partition inside an otherwise work-conserving governor, so a +compile-only run left half the machine idle. Heap pressure is answered by +watching the live heap and staggering starts, which responds to what is +actually happening rather than guessing from a count. ```bash -bleep config compile-server heap-pressure-threshold 0.85 -bleep config compile-server heap-pressure-threshold-clear # back to default 0.80 +bleep config compile-server parallelism 4 ``` -When the compile server's heap usage crosses this fraction, new -compile requests **wait** for memory to free up before starting. -Useful when you're running multiple parallel compiles (e.g. -several editors open against the same shared server) and don't -want one runaway to OOM the whole server. +### Cached builds -The default of 0.80 is conservative. If you've already raised -`max-memory` and the throttle is still kicking in too eagerly, -nudge this up to 0.90 or so. +The server keeps resolved builds in memory so repeated invocations don't +re-resolve from scratch, and drops the least recently used *idle* one when +it needs room. The Zinc analyses read while compiling a workspace are held +with it and dropped with it. -[Reference →](/docs/reference/cli/config/compile-server/) +There is no knob: analyses are structurally shared between workspaces, so +holding several costs far less than it looks, and the bound is a number +nobody has a basis to tune. Evictions are logged if you want to see them. ### Idle read timeout @@ -128,8 +164,13 @@ again — so it can never interrupt a running build. ### Test-runner heap -The test runner is a separate forked JVM. It doesn't share -heap with the compile server. Tune it independently: +The test runner is a separate forked JVM. It does **not** share heap with +the compile server, and neither does anything else your build forks — your +code is compiled inside the server, but run inside a fork. A project that +needs more memory for its tests says so in `bleep.yaml` with +`platform.jvmOptions`; this setting is the build-wide default underneath +that. See +[Memory & Parallelism](/docs/usage/resource-management#per-fork-heap). ```bash bleep config test-runner max-memory 2g @@ -157,7 +198,7 @@ compileServerMode: mode: shared # or: new-each-invocation bspServerConfig: compileServerMaxMemory: 4g - heapPressureThreshold: 0.85 + parallelism: 4 testRunnerMaxMemory: 2g bspReadTimeoutMinutes: 30 ``` diff --git a/docs/reference/cli/config/compile-server.mdx b/docs/reference/cli/config/compile-server.mdx index 1a6b9e611..c6adfb8b8 100644 --- a/docs/reference/cli/config/compile-server.mdx +++ b/docs/reference/cli/config/compile-server.mdx @@ -48,25 +48,25 @@ bleep config compile-server max-memory

remove compile server max heap setting (back to default: 1/4 of physical RAM, clamped to 4g..16g)

-## `bleep config compile-server heap-pressure-threshold` +## `bleep config compile-server parallelism` -

set heap usage fraction (0.0-1.0) above which new compilations wait for memory (default: 0.80)

+

set how many operations may run at once — compiles, test forks, sourcegen (default: one per core)

### Synopsis ```bash -bleep config compile-server heap-pressure-threshold +bleep config compile-server parallelism ``` ### Arguments | Argument | Type | |----------|------| -| `threshold` | one | +| `n` | one | -## `bleep config compile-server heap-pressure-threshold-clear` +## `bleep config compile-server parallelism-clear` -

remove heap pressure threshold setting (use default: 0.80)

+

remove the parallelism setting (back to default: one per core)

## `bleep config compile-server read-timeout` @@ -88,3 +88,23 @@ bleep config compile-server read-timeout

remove read timeout setting (use default: 30 minutes)

+## `bleep config compile-server idle-timeout` + +

set minutes the server stays alive with no connected client before shutting itself down, 0 to stay alive forever (default: 60)

+ +### Synopsis + +```bash +bleep config compile-server idle-timeout +``` + +### Arguments + +| Argument | Type | +|----------|------| +| `minutes` | one | + +## `bleep config compile-server idle-timeout-clear` + +

remove idle timeout setting (use default: 60 minutes)

+ diff --git a/docs/reference/cli/remote-cache/index.mdx b/docs/reference/cli/remote-cache/index.mdx index 3adc9d458..02f6d4af9 100644 --- a/docs/reference/cli/remote-cache/index.mdx +++ b/docs/reference/cli/remote-cache/index.mdx @@ -8,7 +8,7 @@ title: bleep remote-cache # `bleep remote-cache` -

push and pull compiled classes to/from a remote S3-compatible cache

+

push and pull compiled classes to/from a build cache (S3-compatible service or local directory)

## Synopsis @@ -18,6 +18,6 @@ bleep remote-cache [args] [flags] ## Subcommands -- [`bleep remote-cache pull`](./pull): pull cached compiled classes from remote cache -- [`bleep remote-cache push`](./push): push compiled classes to remote cache +- [`bleep remote-cache pull`](./pull): pull cached compiled classes from the cache +- [`bleep remote-cache push`](./push): push compiled classes to the cache diff --git a/docs/reference/cli/remote-cache/pull.mdx b/docs/reference/cli/remote-cache/pull.mdx index e725b61af..9a852a417 100644 --- a/docs/reference/cli/remote-cache/pull.mdx +++ b/docs/reference/cli/remote-cache/pull.mdx @@ -8,7 +8,7 @@ title: bleep remote-cache pull # `bleep remote-cache pull` -

pull cached compiled classes from remote cache

+

pull cached compiled classes from the cache

## Synopsis diff --git a/docs/reference/cli/remote-cache/push.mdx b/docs/reference/cli/remote-cache/push.mdx index 2566c2c0d..c7fb64a33 100644 --- a/docs/reference/cli/remote-cache/push.mdx +++ b/docs/reference/cli/remote-cache/push.mdx @@ -8,7 +8,7 @@ title: bleep remote-cache push # `bleep remote-cache push` -

push compiled classes to remote cache

+

push compiled classes to the cache

## Synopsis diff --git a/docs/usage/conflict-resolution.mdx b/docs/usage/conflict-resolution.mdx index 159766ac0..d2cd97aa9 100644 --- a/docs/usage/conflict-resolution.mdx +++ b/docs/usage/conflict-resolution.mdx @@ -81,9 +81,15 @@ binary-compatible", you say so once with a library version scheme, and bleep stops complaining about every eviction that involves that library. -Schemes are declared per project and inherited by every transitive -consumer. Bleep ships exactly the five scheme names sbt uses, with -identical semantics, because the underlying check +Schemes are declared per project and reach it through `extends`, the +same way dependencies and everything else in a template do. They are +**not** inherited along `dependsOn`: a scheme set on a library project +does not cover the projects that depend on it, so if two projects both +resolve the conflicting library, both need the scheme — put it on a +template they share. + +Bleep ships exactly the five scheme names sbt uses, with identical +semantics, because the underlying check (`bleep.nosbt.librarymanagement.EvictionError`) is the sbt code, liberated. diff --git a/docs/usage/resource-management.mdx b/docs/usage/resource-management.mdx index 70ee50b76..b8adfe71a 100644 --- a/docs/usage/resource-management.mdx +++ b/docs/usage/resource-management.mdx @@ -5,18 +5,35 @@ When a test, a source generator, or a compile runs out of memory — you see `Ja ## The short version -A forked JVM (a test runner, a sourcegen script, a KSP processor) has **two** costs, and they are controlled separately: +Two questions, and they belong to different owners: + +| question | who decides | where | +| --- | --- | --- | +| how much memory does *this code* need? | the build | `platform.jvmOptions` in `bleep.yaml` — per project, and it sizes **forks** | +| how much of *this machine* may bleep use? | whoever owns the machine | user config: `parallelism`, `compileServerMaxMemory` | + +That split is deliberate. A repo cannot know whether it is running on a 16 GB CI container or a 64 GB workstation, and a machine cannot know which of your suites is heavy. + +A forked JVM (a test runner, a sourcegen script, a KSP processor) then has **two** costs, and they are controlled separately: 1. **How big each fork is** — its `-Xmx` heap ceiling. If a suite genuinely needs more memory than it is given, it fails with an in-JVM `OutOfMemoryError` naming the suite. **Fix: give that project more heap.** 2. **How many forks run at once** — the parallelism. If too many run together, the machine runs out of real memory and the OS kills processes with `SIGKILL`. **Fix: lower parallelism, or lower per-fork heap so more fit.** bleep also measures what forks actually cost and sizes concurrency against what the machine can currently spare, so most of the time you never touch any of this. You reach for these knobs when a specific suite is heavier than the default, or when bleep is sharing the machine with something it can't see. +:::info Version + +The machine-wide accounting described on this page arrived after `1.0.0-M10`. On M10 and earlier, concurrency is a plain count of permits with **no memory dimension** — so on a small machine you must set `parallelism` yourself, because nothing will do it for you. + +::: + ## Per-fork heap Every forked JVM is given a default heap of **2 GB**. A fork that states no `-Xmx` is *not* unlimited — the JVM would otherwise take a quarter of the machine, and bleep starts one per core, so a default is essential. -Raise it for a project whose tests need more: +### The build sizes forks, and only forks + +`bleep.yaml` has exactly one resource control, and it is per project: ```yaml projects: @@ -26,40 +43,110 @@ projects: jvmOptions: -Xmx4g ``` -The project's `platform.jvmOptions` are appended after the default, and the **last `-Xmx` wins**, so this overrides the 2 GB for just this project. This is the right fix when the failure is an in-JVM `OutOfMemoryError` / `Java heap space` — that suite is telling you, accurately, that 2 GB is not enough for it. +This is the right and only place for a build to say "this code needs more memory", because that is a property of the code, not of the machine it happens to run on. Everything else about resources — how many things run at once, how big the compile server is — belongs to whoever owns the machine, and lives in user config. + +**It applies to forked processes.** `platform.jvmOptions` reaches: + +- **forked test JVMs** for that project, and +- **`bleep run`**, unless `jvmRuntimeOptions` is also set, in which case that wins for `run` only. + +**It does not affect compilation.** Your code is compiled *inside* the long-lived compile server, in its heap — not in a JVM started for your project. So raising `jvmOptions` will not help a compile that runs out of memory, and lowering it will not make compilation lighter. The compile server's own heap is `compileServerMaxMemory`, in user config. + +If that distinction bites, the symptom tells you which side you are on: an `OutOfMemoryError` naming one of *your* test suites is the fork, and `jvmOptions` is the fix; an `OutOfMemoryError` in the build summary with the server dying is the server, and `compileServerMaxMemory` is the fix. -The same applies to the other forked processes, set once for the whole build in your user config (`~/.config/bleep/config.yaml` or equivalent): +### Precedence for a test fork -| Setting | Forked process it sizes | +Three sources are concatenated, and **the last `-Xmx` wins**: + +1. `testRunnerMaxMemory` from user config — the build-wide baseline +2. the project's `platform.jvmOptions` — the per-project exception +3. `--jvm-opt` on the command line — this run only + +So a global baseline can be set once, a heavy project can override it, and a one-off investigation can override both without editing anything. + +### The other forked processes + +Sourcegen scripts and KSP runners are sized in user config, since they are bleep's own tooling rather than your code: + +| Setting | Sizes | | --- | --- | | `testRunnerMaxMemory` | test runner JVMs (all projects) | | `sourcegenMaxMemory` | sourcegen scripts | | `kspRunnerMaxMemory` | KSP (Kotlin symbol processing) runners | -| `compileServerMaxMemory` | the compile server / BSP daemon's own `-Xmx` (this is *not* a fork; it's the long-lived server heap that Zinc compiles run inside) | - -A project-level `platform.jvmOptions` beats the global `testRunnerMaxMemory` for that project, so use the global for a build-wide baseline and the per-project override for the exceptions. +| `compileServerMaxMemory` | the compile server itself — **not** a fork; the long-lived heap your compiles run inside | ## How many run at once -Concurrency is controlled by the compile server's parallelism, in the same user config: +One knob: **`parallelism`** — how many operations bleep may run at once on this machine. It defaults to one per core and covers everything the build scheduler runs: compiles, links, test suites, sourcegen scripts, annotation-processor and KSP resolution. It also sizes the pool of forked test JVMs. -| Setting | Meaning | -| --- | --- | -| `parallelism` | maximum number of concurrent operations (compiles + forks). Defaults to the number of cores. | -| `parallelismRatio` | the same as a fraction of cores, e.g. `0.5` for half. Used when `parallelism` is unset. | +It is a property of the *machine*, not of a run: the compile server is shared between every client and every workspace on it, so this is the total across all of them, not a limit each one gets separately. + +```bash +bleep config compile-server parallelism 2 +bleep config compile-server parallelism-clear # back to one per core +``` + +`parallelismRatio` expresses the same thing as a fraction of cores (`0.5` for half) if you prefer that in a config file shared across machines of different sizes. Lowering parallelism is the direct fix when the machine is being *overwhelmed* — many forks killed by `SIGKILL` at once, rather than one suite failing with a clean `OutOfMemoryError`. Fewer forks run together, so peak memory drops. -## The machine budget (automatic) +### Setting it for a CI job + +There is deliberately **no environment variable** for this, and no per-invocation flag. The compile server is shared: it reads `parallelism` once at startup and passes it to every fork it spawns, so a per-client override would mean whichever client happened to start the daemon silently configured it for everyone else — including runs that had no opinion. The same goes for the server's own heap, which is part of the key that decides *which* daemon you get: two different values would quietly give you two daemons. + +A CI job is one machine, so configure the machine, as a step before you build: + +```yaml +- name: Configure bleep for this runner + run: | + bleep config compile-server parallelism 2 + bleep config compile-server max-memory 4g + +- name: Test + run: bleep test +``` + +That writes the same user config a developer would keep, which is what a fresh runner wants: it has no standing preferences, so state them once and let every later invocation agree. + +## Small machines and CI + +The compile server's heap and the test forks' heaps come out of **one pot**, and a quarter of RAM is held back for everything that isn't bleep. It is worth doing the arithmetic once, because the failure mode is forks being killed at startup rather than anything that names memory. + +For a fork to be admitted, its heap must fit in: + +``` +budget = RAM − (server heap × 1.25) − max(4 GB, RAM/4) +``` + +On a typical 16 GB CI runner, that quarter-of-RAM reserve and the server's own footprint dominate: + +| server heap | reserve | fork budget | what fits | +| --- | --- | --- | --- | +| 8 GB | 4 GB | **2 GB** | one small fork, serialised | +| 4 GB | 4 GB | **7 GB** | one 6 GB fork, or two 3 GB forks | +| 2 GB | 4 GB | **9.5 GB** | three 3 GB forks | + +The trap is asking for a big compile server *and* big forks on a small box: an 8 GB server on 16 GB leaves 2 GB for everything else, so every fork queues behind every other fork and the job takes far longer than the numbers suggest. On a 16 GB runner, a 4 GB server is usually the better trade. + +Two rules of thumb for CI: + +1. **Size the server first.** It is long-lived and shared; the forks divide what's left. +2. **Set `parallelism` explicitly.** A CI runner's core count is not a good guide to its memory, and the default is one fork per core. + +## What decides when something starts + +You never configure this, but it explains why bleep sometimes runs fewer things than the raw numbers suggest — and why it never runs *more*. + +There is one place that decides. The build scheduler holds every task whose dependencies are met, sorts them so the ones unblocking the most other work come first, and asks the machine whether each fits right now. Anything that fits starts; anything that doesn't stays queued and is asked again the moment a running task finishes, which is exactly when resources come back. So the order you'd want is the order you get, even when the machine is full. -You usually don't set this, but it explains why bleep sometimes runs fewer things than the raw numbers suggest. +"Fits" is two questions: -The compile server continuously measures how much of the machine's non-reclaimable memory is held by **other** processes (your IDE, a browser, another build) and sizes its own fork-memory budget as *what's left, minus a safety margin*. So: +- **A slot.** `parallelism` is how many things may run at once, so this is simply whether one is free. +- **Room for a fork.** Only work that starts a *process* asks this: test runners, sourcegen scripts, KSP, and the linkers for Scala.js, Scala Native, Kotlin/JS and Kotlin/Native. Compiles run inside the server and take a slot but no fork memory. -- If you open something memory-hungry mid-build, bleep stops admitting new forks until there's room, rather than pushing the machine into swap. -- If nothing else is running, it uses more of the machine. +The fork-memory budget is not a fixed number. The server continuously measures how much of the machine's non-reclaimable memory is held by **other** processes — your IDE, a browser, another build — and sizes the budget as what's left, minus a safety margin. Open something memory-hungry mid-build and bleep stops admitting new forks until there is room, rather than pushing the machine into swap; close it and bleep uses more. -This adapts on its own and needs no configuration. It also means a fork is charged its **measured** cost, not its `-Xmx` ceiling — a 2 GB-ceiling fork that only uses 600 MB is counted as 600 MB, so many more fit than the ceilings would suggest. +A fork is also charged its **measured** cost rather than its `-Xmx` ceiling: a 2 GB-ceiling fork that only ever uses 600 MB is counted as 600 MB, so many more fit than the ceilings would suggest. ## How the knobs trade off