From cae337d16d51126259d284a765e54d20bf18d174 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Mon, 13 Jul 2026 20:06:36 -0700 Subject: [PATCH] fix(eve): preserve progress across compaction Signed-off-by: Rui Conti --- .changeset/calm-compaction-checkpoints.md | 5 + docs/agent-config.md | 2 +- docs/concepts/default-harness.md | 2 +- .../agent/tools/advance-checkpoint.ts | 1 + .../agent/tools/advance-checkpoint.ts | 1 + .../agent/tools/advance-checkpoint.ts | 1 + .../compaction-regression-shared/package.json | 1 + .../src/advance-checkpoint.ts | 30 +++ .../compaction-regression-shared/src/agent.ts | 37 +++- .../src/constants.ts | 2 + .../compaction-regression-shared/src/evals.ts | 15 +- .../eve/src/harness/compaction-prompt.test.ts | 61 ++++++ packages/eve/src/harness/compaction-prompt.ts | 181 +++++++++++++++++ packages/eve/src/harness/compaction.test.ts | 142 ++++++++------ packages/eve/src/harness/compaction.ts | 183 +++++------------- .../tool-loop-compaction-accounting.test.ts | 2 +- 16 files changed, 462 insertions(+), 204 deletions(-) create mode 100644 .changeset/calm-compaction-checkpoints.md create mode 100644 e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.ts create mode 100644 e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.ts create mode 100644 e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.ts create mode 100644 e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.ts create mode 100644 e2e/fixtures/compaction-regression-shared/src/constants.ts create mode 100644 packages/eve/src/harness/compaction-prompt.test.ts create mode 100644 packages/eve/src/harness/compaction-prompt.ts diff --git a/.changeset/calm-compaction-checkpoints.md b/.changeset/calm-compaction-checkpoints.md new file mode 100644 index 000000000..b3617d387 --- /dev/null +++ b/.changeset/calm-compaction-checkpoints.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Compaction now reserves room for its checkpoint prompt before reaching the configured threshold. The prompt asks the compaction model to distinguish completed work from remaining work, and later compactions receive the previous checkpoint intact instead of truncating it with ordinary transcript text. diff --git a/docs/agent-config.md b/docs/agent-config.md index 604e20473..23b40ab68 100644 --- a/docs/agent-config.md +++ b/docs/agent-config.md @@ -106,7 +106,7 @@ which levels are available and how they map to provider-native settings. Use ## Compaction -Compaction summarizes older turns as you approach the context window. It's on by default, so you only tune when it kicks in. Lower `thresholdPercent` to compact sooner: +Compaction summarizes older turns as you approach the context window. It's on by default, so you only tune when it kicks in. eve adds the estimated fixed checkpoint-prompt envelope to the trigger count, so compaction starts sooner than the conversation-only estimate. Lower `thresholdPercent` to compact sooner: ```ts title="agent/agent.ts" export default defineAgent({ diff --git a/docs/concepts/default-harness.md b/docs/concepts/default-harness.md index 8963e3145..8b76b8633 100644 --- a/docs/concepts/default-harness.md +++ b/docs/concepts/default-harness.md @@ -7,7 +7,7 @@ The default harness is eve's built-in agent loop. It manages model calls, compac ## Compaction -The harness keeps a long session from overflowing the model's context window. Once the conversation crosses a fraction of the window (`thresholdPercent`, `0.9` by default), it summarizes the older turns into a compact form and keeps going. The summary uses the active turn model unless you override it. Tune when and how it kicks in under [`compaction`](../agent-config#compaction) in `agent.ts`: +The harness keeps a long session from overflowing the model's context window. Before comparing the conversation with `thresholdPercent` (`0.9` by default), it adds the estimated fixed envelope of the checkpoint prompt used for compaction. It then summarizes the older turns and keeps going. The prompt asks the compaction model to distinguish completed progress and decisions from remaining work and to retain the constraints, preferences, data, and references needed to continue. When eve compacts again, it passes the previous checkpoint separately and without the transcript's per-message truncation, then replaces it with the updated checkpoint. The summary uses the active turn model unless you override it. Tune when and how it kicks in under [`compaction`](../agent-config#compaction) in `agent.ts`: ```ts title="agent/agent.ts" export default defineAgent({ diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.ts new file mode 100644 index 000000000..e7c2434b0 --- /dev/null +++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.ts @@ -0,0 +1 @@ +export { default } from "@eve/e2e-compaction-regression-shared/tools/advance-checkpoint"; diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.ts new file mode 100644 index 000000000..e7c2434b0 --- /dev/null +++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.ts @@ -0,0 +1 @@ +export { default } from "@eve/e2e-compaction-regression-shared/tools/advance-checkpoint"; diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.ts new file mode 100644 index 000000000..e7c2434b0 --- /dev/null +++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.ts @@ -0,0 +1 @@ +export { default } from "@eve/e2e-compaction-regression-shared/tools/advance-checkpoint"; diff --git a/e2e/fixtures/compaction-regression-shared/package.json b/e2e/fixtures/compaction-regression-shared/package.json index 480176070..7c3e6963c 100644 --- a/e2e/fixtures/compaction-regression-shared/package.json +++ b/e2e/fixtures/compaction-regression-shared/package.json @@ -6,6 +6,7 @@ "exports": { "./agent": "./src/agent.ts", "./evals": "./src/evals.ts", + "./tools/advance-checkpoint": "./src/advance-checkpoint.ts", "./tools/inspect-repository": "./src/inspect-repository.ts", "./tools/perform-source-analysis": "./src/perform-source-analysis.ts" }, diff --git a/e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.ts b/e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.ts new file mode 100644 index 000000000..d61f1c769 --- /dev/null +++ b/e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.ts @@ -0,0 +1,30 @@ +import { defineState } from "eve/context"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { SECOND_CHECKPOINT_MARKER } from "./constants"; + +const invocationCount = defineState("compaction-regression.advance-checkpoint", () => 0); +const modelFamilySchema = z.enum(["gpt-5.6", "opus-4.8", "sonnet-5"]); + +export default defineTool({ + description: + "Test-only second-compaction trigger tool. Records a completed fixture work unit and adds enough evidence for the harness to cross the compaction threshold again.", + inputSchema: z.object({ + modelFamily: modelFamilySchema, + regressionCase: z.enum(["redundant-tool-calls", "stale-todo-work"]), + }), + async execute(input) { + const attempt = invocationCount.get() + 1; + invocationCount.update(() => attempt); + + return { + checkpointMarker: SECOND_CHECKPOINT_MARKER, + completed: true, + modelFamily: input.modelFamily, + regressionCase: input.regressionCase, + attempt, + evidencePadding: "second checkpoint evidence ".repeat(100), + }; + }, +}); diff --git a/e2e/fixtures/compaction-regression-shared/src/agent.ts b/e2e/fixtures/compaction-regression-shared/src/agent.ts index c21e0b946..b951f5020 100644 --- a/e2e/fixtures/compaction-regression-shared/src/agent.ts +++ b/e2e/fixtures/compaction-regression-shared/src/agent.ts @@ -1,6 +1,8 @@ import { defineAgent, type AgentDefinition } from "eve"; import { mockModel, type MockModelRequest } from "eve/evals"; +import { SECOND_CHECKPOINT_MARKER } from "./constants"; + const TEST_CONTEXT_WINDOW_TOKENS = 32_000; const MAX_TOOL_CALLS = 10; @@ -16,6 +18,7 @@ export function createCompactionRegressionAgent(input: { readonly modelFamily: ModelFamily; }): AgentDefinition { let activeRegression: ActiveRegression | undefined; + const checkpointAdvanceCallCounts = new Map(); const toolCallCounts = new Map(); const taskModel = mockModel({ modelId: `compaction-regression-task-model-${input.modelFamily}`, @@ -26,6 +29,7 @@ export function createCompactionRegressionAgent(input: { activeRegression?.regressionCase !== initialRegression.regressionCase ) { activeRegression = initialRegression; + checkpointAdvanceCallCounts.set(initialRegression.regressionCase, 0); toolCallCounts.set(initialRegression.regressionCase, 0); } @@ -36,8 +40,32 @@ export function createCompactionRegressionAgent(input: { const regression = activeRegression; const marker = completionMarker(regression.regressionCase); - if (checkpointProvesCompletion(request.messages, marker)) { - return `Done: ${marker}`; + // These are fixture markers, not compaction protocol fields. `marker` records the + // regression work tool; `SECOND_CHECKPOINT_MARKER` records the test-only tool + // whose output makes the harness cross the compaction threshold a second time. + if (checkpointContains(request.messages, marker)) { + if (checkpointContains(request.messages, SECOND_CHECKPOINT_MARKER)) { + return `Done: ${marker}; ${SECOND_CHECKPOINT_MARKER}`; + } + + const advanceCalls = checkpointAdvanceCallCounts.get(regression.regressionCase) ?? 0; + if (advanceCalls >= MAX_TOOL_CALLS) { + return `Hard stop after ${MAX_TOOL_CALLS} checkpoint advances: ${marker}`; + } + + checkpointAdvanceCallCounts.set(regression.regressionCase, advanceCalls + 1); + return { + toolCalls: [ + { + id: `advance-checkpoint-${advanceCalls + 1}`, + input: { + modelFamily: input.modelFamily, + regressionCase: regression.regressionCase, + }, + name: "advance-checkpoint", + }, + ], + }; } const completedCalls = toolCallCounts.get(regression.regressionCase) ?? 0; @@ -110,10 +138,7 @@ function completionMarker(regressionCase: RegressionCase): string { : "SOURCE_ANALYSIS_COMPLETE"; } -function checkpointProvesCompletion( - messages: MockModelRequest["messages"], - marker: string, -): boolean { +function checkpointContains(messages: MockModelRequest["messages"], marker: string): boolean { return messages.some((message, index) => { if (message.role !== "user" || message.text !== "Summary of our conversation so far:") { return false; diff --git a/e2e/fixtures/compaction-regression-shared/src/constants.ts b/e2e/fixtures/compaction-regression-shared/src/constants.ts new file mode 100644 index 000000000..9773bae5d --- /dev/null +++ b/e2e/fixtures/compaction-regression-shared/src/constants.ts @@ -0,0 +1,2 @@ +/** Fixture evidence emitted by the test-only second-compaction trigger tool. */ +export const SECOND_CHECKPOINT_MARKER = "SECOND_CHECKPOINT_READY"; diff --git a/e2e/fixtures/compaction-regression-shared/src/evals.ts b/e2e/fixtures/compaction-regression-shared/src/evals.ts index 0da27cea7..a2d670472 100644 --- a/e2e/fixtures/compaction-regression-shared/src/evals.ts +++ b/e2e/fixtures/compaction-regression-shared/src/evals.ts @@ -1,6 +1,7 @@ import type { EveEvalContext } from "eve/evals"; import type { ModelFamily } from "./agent"; +import { SECOND_CHECKPOINT_MARKER } from "./constants"; export async function testRedundantToolCalls( t: EveEvalContext, @@ -22,8 +23,13 @@ export async function testRedundantToolCalls( input: { scope: "repository" }, output: { completed: true, completionMarker: "REPOSITORY_INSPECTION_COMPLETE" }, }); - t.event("compaction.completed", { count: (count) => count >= 1 }); + t.calledTool("advance-checkpoint", { + count: 1, + output: { checkpointMarker: SECOND_CHECKPOINT_MARKER, completed: true }, + }); + t.event("compaction.completed", { count: 2 }); t.messageIncludes("REPOSITORY_INSPECTION_COMPLETE"); + t.messageIncludes(SECOND_CHECKPOINT_MARKER); } export async function testStaleTodoWork( @@ -46,6 +52,11 @@ export async function testStaleTodoWork( count: 1, output: { completed: true, workUnit: "source-analysis" }, }); - t.event("compaction.completed", { count: (count) => count >= 1 }); + t.calledTool("advance-checkpoint", { + count: 1, + output: { checkpointMarker: SECOND_CHECKPOINT_MARKER, completed: true }, + }); + t.event("compaction.completed", { count: 2 }); t.messageIncludes("SOURCE_ANALYSIS_COMPLETE"); + t.messageIncludes(SECOND_CHECKPOINT_MARKER); } diff --git a/packages/eve/src/harness/compaction-prompt.test.ts b/packages/eve/src/harness/compaction-prompt.test.ts new file mode 100644 index 000000000..293c47af9 --- /dev/null +++ b/packages/eve/src/harness/compaction-prompt.test.ts @@ -0,0 +1,61 @@ +import type { ModelMessage } from "ai"; +import { describe, expect, it } from "vitest"; + +import { COMPACTION_PROMPT_ENVELOPE, createCompactionPrompt } from "#harness/compaction-prompt.js"; + +describe("createCompactionPrompt", () => { + it("preserves the previous checkpoint without applying transcript truncation", () => { + const markerAfterTextLimit = "CRITICAL_STATE_AFTER_280_CHARACTERS"; + const previousCheckpoint = `${"completed work ".repeat(24)}${markerAfterTextLimit}`; + + const result = createCompactionPrompt({ + messages: [{ content: "New evidence", role: "user" }], + previousCheckpoint, + }); + + expect(result.system).toBe(COMPACTION_PROMPT_ENVELOPE.system); + expect(result.prompt).toContain(`\n${previousCheckpoint}`); + expect(result.prompt).toContain(markerAfterTextLimit); + }); + + it("summarizes structured tool messages without dumping raw JSON", () => { + const messages: ModelMessage[] = [ + { + content: [ + { + input: { query: "debug" }, + toolCallId: "call-1", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + { + content: [ + { + output: { + type: "json", + value: ["alpha", "beta", "gamma", "delta"], + }, + toolCallId: "call-1", + toolName: "search", + type: "tool-result", + }, + ], + role: "tool", + }, + ]; + + const result = createCompactionPrompt({ messages, previousCheckpoint: undefined }); + + expect(result.prompt).toContain("Conversation transcript:"); + expect(result.prompt).toContain("### assistant"); + expect(result.prompt).toContain("Called search with object(query=debug)"); + expect(result.prompt).toContain( + "Tool search returned object(type=json, value=array(4: alpha, beta, gamma, …))", + ); + expect(result.prompt).not.toContain('{"query"'); + expect(result.prompt).not.toContain('{"items"'); + }); +}); diff --git a/packages/eve/src/harness/compaction-prompt.ts b/packages/eve/src/harness/compaction-prompt.ts new file mode 100644 index 000000000..9f3ce0773 --- /dev/null +++ b/packages/eve/src/harness/compaction-prompt.ts @@ -0,0 +1,181 @@ +import type { ModelMessage } from "ai"; + +export const COMPACTION_CHECKPOINT_MARKER = "Summary of our conversation so far:"; + +const COMPACTION_SYSTEM_PROMPT = `You are performing a CONTEXT CHECKPOINT COMPACTION. Create a handoff summary for another LLM that will resume the task. + +Include: + +- Current progress and key decisions made +- Important context, constraints, or user preferences +- What remains to be done, with clear next steps +- Any critical data, examples, or references needed to continue + +Be concise, structured, and focused on helping the next LLM seamlessly continue the work. Write in the same language as the conversation. Do not continue the conversation, answer its questions, or invent facts. Only output the handoff summary.`; + +const COMPACTION_CHECKPOINT_PROMPT = `Update the previous checkpoint with the newer information in the conversation. If there is no previous checkpoint, create one from the conversation. + +Make completed work explicit so the next model does not repeat it. Keep completed work separate from current and remaining work, and do not describe completed work as pending unless later messages show it must be redone. Preserve exact file paths, function names, commands, error messages, identifiers, and measured values when they are needed to continue.`; + +const COMPACTION_TEXT_LIMIT = 280; +const COMPACTION_COLLECTION_LIMIT = 3; + +interface CompactionTranscriptMessage { + readonly content: string; + readonly role: ModelMessage["role"]; +} + +export interface CompactionPrompt { + readonly prompt: string; + readonly system: string; +} + +/** Static prompt text added around checkpoint and conversation content. */ +export const COMPACTION_PROMPT_ENVELOPE = { + prompt: formatCompactionPrompt({ previousCheckpoint: "", transcript: "" }), + system: COMPACTION_SYSTEM_PROMPT, +} satisfies CompactionPrompt; + +/** Builds the compaction model input from framework-owned checkpoint state and older messages. */ +export function createCompactionPrompt(input: { + readonly messages: readonly ModelMessage[]; + readonly previousCheckpoint: string | undefined; +}): CompactionPrompt { + const transcript = input.messages.map((message) => ({ + content: summarizeCompactionMessageContent(message), + role: message.role, + })); + + return { + prompt: formatCompactionPrompt({ + previousCheckpoint: input.previousCheckpoint?.trim() ?? "(none)", + transcript: formatCompactionTranscript(transcript), + }), + system: COMPACTION_SYSTEM_PROMPT, + }; +} + +function formatCompactionPrompt(input: { + readonly previousCheckpoint: string; + readonly transcript: string; +}): string { + return ` +${input.previousCheckpoint} + + + +Conversation transcript: +${input.transcript} + + +${COMPACTION_CHECKPOINT_PROMPT}`; +} + +function formatCompactionTranscript(messages: readonly CompactionTranscriptMessage[]): string { + const sections = messages + .filter((message) => message.content.trim().length > 0) + .map((message) => `### ${message.role}\n${message.content.trim()}`); + + return sections.length === 0 ? "(empty)" : sections.join("\n\n"); +} + +function summarizeCompactionMessageContent(message: ModelMessage): string { + if (typeof message.content === "string") { + return summarizeText(message.content); + } + + return message.content + .map((part) => summarizeCompactionContentPart(part)) + .filter((summary) => summary.length > 0) + .join("\n") + .trim(); +} + +type ModelMessageContentPart = Exclude[number]; + +function summarizeCompactionContentPart(part: ModelMessageContentPart): string { + switch (part.type) { + case "text": + return summarizeText(part.text); + case "reasoning": + return ""; + case "file": + return part.filename + ? `Attached file ${part.filename} (${part.mediaType})` + : `Attached file attachment (${part.mediaType})`; + case "tool-call": + return summarizeToolCallPart(part); + case "tool-result": + return summarizeToolResultPart(part); + default: + return ""; + } +} + +function summarizeToolCallPart(part: { toolName: string; input?: unknown }): string { + const input = part.input !== undefined ? summarizeCompactValue(part.input) : ""; + return input ? `Called ${part.toolName} with ${input}` : `Called ${part.toolName}`; +} + +function summarizeToolResultPart(part: { + toolName: string; + output?: unknown; + isError?: boolean; +}): string { + const output = part.output !== undefined ? summarizeCompactValue(part.output) : ""; + const status = part.isError ? "errored" : "returned"; + return output ? `Tool ${part.toolName} ${status} ${output}` : `Tool ${part.toolName} ${status}`; +} + +function summarizeCompactValue(value: unknown, depth = 0): string { + if (value === null) return "null"; + if (value === undefined) return ""; + if (typeof value === "string") return summarizeText(value); + if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") { + return String(value); + } + + if (Array.isArray(value)) { + if (value.length === 0) { + return "array(0)"; + } + + if (depth >= 2) { + return `array(${value.length})`; + } + + const entries = value + .slice(0, COMPACTION_COLLECTION_LIMIT) + .map((item) => summarizeCompactValue(item, depth + 1)); + const suffix = value.length > COMPACTION_COLLECTION_LIMIT ? ", …" : ""; + return `array(${value.length}: ${entries.join(", ")}${suffix})`; + } + + if (typeof value === "object") { + const entries = Object.entries(value as Record); + if (entries.length === 0) { + return "object(0)"; + } + + if (depth >= 2) { + return `object(${entries.length} keys)`; + } + + const rendered = entries + .slice(0, COMPACTION_COLLECTION_LIMIT) + .map(([key, nested]) => `${key}=${summarizeCompactValue(nested, depth + 1)}`); + const suffix = entries.length > COMPACTION_COLLECTION_LIMIT ? ", …" : ""; + return `object(${rendered.join(", ")}${suffix})`; + } + + return ""; +} + +function summarizeText(value: string): string { + const normalized = value.replace(/\s+/g, " ").trim(); + if (normalized.length <= COMPACTION_TEXT_LIMIT) { + return normalized; + } + + return `${normalized.slice(0, COMPACTION_TEXT_LIMIT).trimEnd()}…`; +} diff --git a/packages/eve/src/harness/compaction.test.ts b/packages/eve/src/harness/compaction.test.ts index e91916cde..abde90235 100644 --- a/packages/eve/src/harness/compaction.test.ts +++ b/packages/eve/src/harness/compaction.test.ts @@ -1,6 +1,7 @@ import type { ModelMessage } from "ai"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { COMPACTION_PROMPT_ENVELOPE } from "#harness/compaction-prompt.js"; import { compactMessages, estimateTokens, @@ -157,13 +158,45 @@ describe("getInputTokenCount", () => { describe("shouldCompact", () => { it("returns false when under threshold", () => { const messages: ModelMessage[] = [{ content: "short", role: "user" }]; - expect(shouldCompact(messages, config)).toBe(false); + expect(shouldCompact(messages, { ...config, threshold: 1_000 })).toBe(false); }); it("returns true when over threshold", () => { const messages: ModelMessage[] = [{ content: "a".repeat(500), role: "user" }]; expect(shouldCompact(messages, config)).toBe(true); }); + + it("uses the fixed prompt envelope in threshold accounting", () => { + const messages: ModelMessage[] = [{ content: "Continue the investigation.", role: "user" }]; + const compaction: CompactionConfig = { + lastKnownInputTokens: 200, + lastKnownPromptMessageCount: messages.length, + recentWindowSize: 2, + threshold: 1_000, + }; + const activeInputTokens = getInputTokenCount(messages, compaction); + const promptEnvelopeTokens = estimateTokens([ + { content: COMPACTION_PROMPT_ENVELOPE.system, role: "system" }, + { content: COMPACTION_PROMPT_ENVELOPE.prompt, role: "user" }, + ] satisfies ModelMessage[]); + + expect( + shouldCompact(messages, { + ...compaction, + threshold: activeInputTokens + promptEnvelopeTokens, + }), + ).toBe(false); + expect( + shouldCompact(messages, { + ...compaction, + threshold: activeInputTokens + promptEnvelopeTokens - 1, + }), + ).toBe(true); + }); + + it("does not compact an empty history based on prompt overhead alone", () => { + expect(shouldCompact([], { ...config, threshold: 0 })).toBe(false); + }); }); describe("resolveCompactionModel", () => { @@ -220,6 +253,60 @@ describe("resolveCompactionModel", () => { }); describe("compactMessages", () => { + it("carries a prior checkpoint into the next compaction without truncating it", async () => { + const { generateText } = await import("ai"); + const markerAfterTextLimit = "CRITICAL_STATE_AFTER_280_CHARACTERS"; + const previousCheckpoint = `${"completed work ".repeat(24)}${markerAfterTextLimit}`; + + vi.mocked(generateText).mockResolvedValue({ + text: "Updated checkpoint", + } as Awaited>); + + const messages: ModelMessage[] = [ + { content: "Summary of our conversation so far:", role: "user" }, + { content: previousCheckpoint, role: "assistant" }, + { content: "new older context", role: "user" }, + { content: "new older response", role: "assistant" }, + { content: "recent question", role: "user" }, + { content: "recent answer", role: "assistant" }, + ]; + + const model = {} as Parameters[1]; + await compactMessages(messages, model, { + recentWindowSize: 2, + threshold: 100_000, + }); + + const call = vi.mocked(generateText).mock.calls[0]?.[0]; + expect(call?.prompt).toContain(previousCheckpoint); + expect(call?.prompt).toContain(markerAfterTextLimit); + }); + + it("replaces a prior checkpoint instead of retaining it in the recent window", async () => { + const { generateText } = await import("ai"); + const previousCheckpoint = "Previous checkpoint"; + + vi.mocked(generateText).mockResolvedValue({ + text: "Updated checkpoint", + } as Awaited>); + + const messages: ModelMessage[] = [ + { content: "Summary of our conversation so far:", role: "user" }, + { content: previousCheckpoint, role: "assistant" }, + { content: "new evidence", role: "user" }, + { content: "latest response", role: "assistant" }, + ]; + + const model = {} as Parameters[1]; + const result = await compactMessages(messages, model, { + recentWindowSize: 10, + threshold: 100_000, + }); + + expect(result.filter((message) => message.content === previousCheckpoint)).toHaveLength(0); + expect(result.filter((message) => message.content === "Updated checkpoint")).toHaveLength(1); + }); + it("summarizes older messages and keeps recent window", async () => { const { generateText } = await import("ai"); @@ -472,59 +559,6 @@ describe("compactMessages", () => { ); }); - it("summarizes structured tool messages without dumping raw JSON", async () => { - const { generateText } = await import("ai"); - - vi.mocked(generateText).mockResolvedValue({ - text: "Summary of prior context", - } as Awaited>); - - const messages: ModelMessage[] = [ - { content: "old message 1", role: "user" }, - { - content: [ - { - input: { query: "debug" }, - toolCallId: "call-1", - toolName: "search", - type: "tool-call", - }, - ], - role: "assistant", - }, - { - content: [ - { - output: { - type: "json", - value: ["alpha", "beta", "gamma", "delta"], - }, - toolCallId: "call-1", - toolName: "search", - type: "tool-result", - }, - ], - role: "tool", - }, - { content: "recent 1", role: "user" }, - { content: "recent 2", role: "assistant" }, - ]; - - const model = {} as Parameters[1]; - await compactMessages(messages, model, config); - - const call = vi.mocked(generateText).mock.calls[0]?.[0]; - expect(call?.system).toContain("conversation summarizer"); - expect(call?.prompt).toContain("Conversation transcript:"); - expect(call?.prompt).toContain("### assistant"); - expect(call?.prompt).toContain("Called search with object(query=debug)"); - expect(call?.prompt).toContain( - "Tool search returned object(type=json, value=array(4: alpha, beta, gamma, …))", - ); - expect(call?.prompt).not.toContain('{"query"'); - expect(call?.prompt).not.toContain('{"items"'); - }); - it("appends synthetic user message when recent window trails with assistant", async () => { const { generateText } = await import("ai"); diff --git a/packages/eve/src/harness/compaction.ts b/packages/eve/src/harness/compaction.ts index 648a4884f..d37044a81 100644 --- a/packages/eve/src/harness/compaction.ts +++ b/packages/eve/src/harness/compaction.ts @@ -1,25 +1,14 @@ import { generateText, type LanguageModel, type ModelMessage, type TelemetryOptions } from "ai"; +import { + COMPACTION_CHECKPOINT_MARKER, + COMPACTION_PROMPT_ENVELOPE, + createCompactionPrompt, +} from "#harness/compaction-prompt.js"; import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js"; import type { CompactionConfig, ToolLoopHarnessConfig } from "#harness/types.js"; -const COMPACTION_SYSTEM_PROMPT = [ - "You are a conversation summarizer.", - "Write a concise but useful summary for continuing the work.", - "Preserve the goal, important instructions, technical decisions, discoveries, open work, and relevant tool results.", - "Use the same language as the conversation.", - "Prefer short labeled sections such as Goal, Instructions, Discoveries, Accomplished, and Next steps when helpful.", - "Do not answer questions or invent facts.", -].join(" "); - const COMPACTION_SUMMARY_RESERVE_TOKENS = 2_048; -const COMPACTION_TEXT_LIMIT = 280; -const COMPACTION_COLLECTION_LIMIT = 3; - -interface CompactionTranscriptMessage { - readonly content: string; - readonly role: ModelMessage["role"]; -} /** * Element type of a non-string `ModelMessage.content` array. @@ -38,6 +27,11 @@ export function estimateTokens(value: unknown): number { return JSON.stringify(value).length / 4; } +const COMPACTION_PROMPT_OVERHEAD_TOKENS = estimateTokens([ + { content: COMPACTION_PROMPT_ENVELOPE.system, role: "system" }, + { content: COMPACTION_PROMPT_ENVELOPE.prompt, role: "user" }, +] satisfies ModelMessage[]); + /** * Best available input-token count: the model-reported count from the last * step, plus a rough character-based estimate of whatever messages have been @@ -64,13 +58,17 @@ export function getInputTokenCount( } /** - * Returns true when the message history exceeds the compaction threshold. + * Returns true when the message history and fixed compaction-prompt envelope + * exceed the compaction threshold. */ export function shouldCompact( messages: readonly ModelMessage[], config: CompactionConfig, ): boolean { - return getInputTokenCount(messages, config) > config.threshold; + return ( + messages.length > 0 && + getInputTokenCount(messages, config) + COMPACTION_PROMPT_OVERHEAD_TOKENS > config.threshold + ); } /** @@ -113,26 +111,24 @@ export async function compactMessages( headers?: Record, abortSignal?: AbortSignal, ): Promise { - let keep = selectRecentWindowSize(messages, config); + const { conversation, previousCheckpoint } = extractPreviousCheckpoint(messages); + let keep = selectRecentWindowSize(conversation, config); while (true) { - const { older, recent } = splitMessagesForCompaction(messages, keep); - if (older.length === 0) { + const { older, recent } = splitMessagesForCompaction(conversation, keep); + if (older.length === 0 && previousCheckpoint === undefined) { return keepNonToolResultMessages(recent); } - const prunedOlder: CompactionTranscriptMessage[] = older.map((message) => ({ - content: summarizeCompactionMessageContent(message), - role: message.role, - })); + const summaryPrompt = createCompactionPrompt({ messages: older, previousCheckpoint }); const result = await generateText({ abortSignal, headers, model, - prompt: formatCompactionPrompt(prunedOlder), + prompt: summaryPrompt.prompt, providerOptions, - system: COMPACTION_SYSTEM_PROMPT, + system: summaryPrompt.system, telemetry: telemetry ? { ...telemetry, functionId: "eve.compaction" } : undefined, temperature: 0, }); @@ -154,7 +150,7 @@ export async function compactMessages( : []; const compacted: ModelMessage[] = [ - { content: "Summary of our conversation so far:", role: "user" }, + { content: COMPACTION_CHECKPOINT_MARKER, role: "user" }, { content: result.text, role: "assistant" }, ...keptTail, ...trailingAssistantGuard, @@ -168,6 +164,26 @@ export async function compactMessages( } } +function extractPreviousCheckpoint(messages: readonly ModelMessage[]): { + readonly conversation: ModelMessage[]; + readonly previousCheckpoint: string | undefined; +} { + const marker = messages[0]; + const checkpoint = messages[1]; + if ( + marker?.role !== "user" || + marker.content !== COMPACTION_CHECKPOINT_MARKER || + checkpoint?.role !== "assistant" + ) { + return { conversation: [...messages], previousCheckpoint: undefined }; + } + + return { + conversation: messages.slice(2), + previousCheckpoint: assistantMessageText(checkpoint), + }; +} + /** * Returns the kept tail for a compacted history: recent messages with tool * activity removed. Tool-result messages are dropped, and assistant messages are @@ -269,114 +285,3 @@ function splitMessagesForCompaction( recent: messages.slice(-keep), }; } - -function formatCompactionPrompt(messages: readonly CompactionTranscriptMessage[]): string { - const sections = messages - .filter((message) => message.content.trim().length > 0) - .map((message) => `### ${message.role}\n${message.content.trim()}`); - - if (sections.length === 0) { - return "Summarize the conversation so far."; - } - - return ["Conversation transcript:", ...sections].join("\n\n"); -} - -function summarizeCompactionMessageContent(message: ModelMessage): string { - if (typeof message.content === "string") { - return summarizeText(message.content); - } - - return message.content - .map((part) => summarizeCompactionContentPart(part)) - .filter((summary) => summary.length > 0) - .join("\n") - .trim(); -} - -function summarizeCompactionContentPart(part: ModelMessageContentPart): string { - switch (part.type) { - case "text": - return summarizeText(part.text); - case "reasoning": - return ""; - case "file": - return part.filename - ? `Attached file ${part.filename} (${part.mediaType})` - : `Attached file attachment (${part.mediaType})`; - case "tool-call": - return summarizeToolCallPart(part); - case "tool-result": - return summarizeToolResultPart(part); - default: - return ""; - } -} - -function summarizeToolCallPart(part: { toolName: string; input?: unknown }): string { - const input = part.input !== undefined ? summarizeCompactValue(part.input) : ""; - return input ? `Called ${part.toolName} with ${input}` : `Called ${part.toolName}`; -} - -function summarizeToolResultPart(part: { - toolName: string; - output?: unknown; - isError?: boolean; -}): string { - const output = part.output !== undefined ? summarizeCompactValue(part.output) : ""; - const status = part.isError ? "errored" : "returned"; - return output ? `Tool ${part.toolName} ${status} ${output}` : `Tool ${part.toolName} ${status}`; -} - -function summarizeCompactValue(value: unknown, depth = 0): string { - if (value === null) return "null"; - if (value === undefined) return ""; - if (typeof value === "string") return summarizeText(value); - if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") { - return String(value); - } - - if (Array.isArray(value)) { - if (value.length === 0) { - return "array(0)"; - } - - if (depth >= 2) { - return `array(${value.length})`; - } - - const entries = value - .slice(0, COMPACTION_COLLECTION_LIMIT) - .map((item) => summarizeCompactValue(item, depth + 1)); - const suffix = value.length > COMPACTION_COLLECTION_LIMIT ? ", …" : ""; - return `array(${value.length}: ${entries.join(", ")}${suffix})`; - } - - if (typeof value === "object") { - const entries = Object.entries(value as Record); - if (entries.length === 0) { - return "object(0)"; - } - - if (depth >= 2) { - return `object(${entries.length} keys)`; - } - - const rendered = entries - .slice(0, COMPACTION_COLLECTION_LIMIT) - .map(([key, nested]) => `${key}=${summarizeCompactValue(nested, depth + 1)}`); - const suffix = entries.length > COMPACTION_COLLECTION_LIMIT ? ", …" : ""; - return `object(${rendered.join(", ")}${suffix})`; - } - - return ""; -} - -function summarizeText(value: string): string { - const normalized = value.replace(/\s+/g, " ").trim(); - if (normalized.length <= COMPACTION_TEXT_LIMIT) { - return normalized; - } - - return `${normalized.slice(0, COMPACTION_TEXT_LIMIT).trimEnd()}…`; -} diff --git a/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts b/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts index ea63d0498..2b92774a2 100644 --- a/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts +++ b/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts @@ -206,7 +206,7 @@ describe("tool-loop structured compaction accounting", () => { createTestSession({ compaction: { recentWindowSize: 10, - threshold: 101, + threshold: 500, }, }), { message: "Compute something" },