From 08a7bd9f99e3bcdbc15d4a1af477437df52c3ef1 Mon Sep 17 00:00:00 2001 From: 1bcMax <195689928+1bcMax@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:03:50 -0500 Subject: [PATCH 1/2] =?UTF-8?q?fix(tokens):=20gpt-5.5=20and=20gpt-5.4=20to?= =?UTF-8?q?=20128000=20=E2=80=94=20now=20actually=20measured?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were 32768. The catalog said 128000 and I twice failed to establish whether that was true. #111 raised them to 100000 citing an upstream maximum. That number came from blockrun_llm/validation.py:233 — a client-side guard — so the probe never reached a provider. #112 reverted them to 32768 and recorded the catalog's 128000 as unrefuted but unverified. Re-probed 2026-07-21 with the SDK guard bypassed in-process: both accept 128000, as did all 19 models advertising a ceiling above 100000, including zai/glm-5.2 at 262144. Zero rejections. The catalog was right the whole time; the measurement was broken. Saying so explicitly because the previous conclusion pointed the other way, and this table is exactly where a wrong belief gets frozen. Root cause fixed upstream in blockrun-llm#27 and blockrun-llm-ts#15. Effect: normal turns are unchanged (CAPPED_MAX_TOKENS 16384). On a max_tokens stop the escalation ceiling goes from min(65536, 32768)=32768 to min(65536, 128000)=65536 — recovery doubles instead of clamping back below where it started. Local suite 636 pass. --- src/agent/optimize.ts | 15 +++++++-------- test/local.mjs | 25 +++++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/agent/optimize.ts b/src/agent/optimize.ts index 3a5e998..546774f 100644 --- a/src/agent/optimize.ts +++ b/src/agent/optimize.ts @@ -53,14 +53,13 @@ const MODEL_MAX_OUTPUT: Record = { 'openai/gpt-5.6-sol': 128_000, 'openai/gpt-5.6-terra': 128_000, 'openai/gpt-5.6-luna': 128_000, - // gpt-5.5 / gpt-5.4 stay at 32768. The catalog claims 128000 for both and - // that claim is UNREFUTED — an earlier attempt to verify it was invalid: - // both SDKs reject max_tokens > 100000 client-side (blockrun_llm - // validation.py:233, blockrun-llm-ts validation.ts:74), so the "rejections" - // collected at 128000 never reached a provider. Their real ceilings are - // unknown. Raising them needs a probe path that bypasses the SDK guard. - 'openai/gpt-5.5': 32_768, - 'openai/gpt-5.4': 32_768, + // 128000 is now measured, not read off the catalog. The earlier probe that + // seemed to refute it was invalid — both SDKs rejected max_tokens > 100000 + // client-side, so nothing reached a provider. Re-probed 2026-07-21 with that + // guard bypassed: both accept 128000, as did all 19 models advertising a + // ceiling above 100000. The catalog was right; the measurement was broken. + 'openai/gpt-5.5': 128_000, + 'openai/gpt-5.4': 128_000, 'openai/gpt-5.4-mini': 128_000, 'openai/gpt-5.4-nano': 32_768, // Probed accepted at 65536 through the live gateway (2026-07-21) — a real diff --git a/test/local.mjs b/test/local.mjs index ae7d354..d3346d0 100644 --- a/test/local.mjs +++ b/test/local.mjs @@ -6710,21 +6710,22 @@ test('anthropic max output matches the documented ceilings', async () => { assert.equal(getMaxOutputTokens('anthropic/claude-sonnet-4.6'), 128_000); }); -test('openai ceilings: one probed, two still unverified', async () => { - // gpt-5-mini was raised to 65536 on a real accepted request through the live - // gateway (2026-07-21). It is a FLOOR, not a proven ceiling — 65536 is also - // ESCALATED_MAX_TOKENS, so nothing here could tell a higher true limit apart. +test('openai ceilings are measured, not read off the catalog', async () => { + // All three are now probed against the live gateway. // - // gpt-5.5 / gpt-5.4 stay at 32768. The catalog claims 128000 and that claim - // is UNREFUTED. An attempt to verify it was invalid: both SDKs reject - // max_tokens > 100000 client-side (blockrun_llm validation.py:233, - // blockrun-llm-ts validation.ts:74), so probes at 128000 never reached a - // provider — the "maximum: 100000" in those errors is an SDK constant, not - // an upstream response. Raising these needs a probe that bypasses the guard. + // gpt-5-mini 65536 accepted (probe was below the SDK guard, so it went out) + // gpt-5.5 128000 accepted, 2026-07-21, guard bypassed + // gpt-5.4 128000 accepted, same run + // + // An earlier pass recorded these as REJECTED at 128000 and concluded the + // catalog was wrong. That was measurement error: both SDKs threw on + // max_tokens > 100000 client-side, so 19 "rejections" never left the + // process. The uniform limit across three different providers was the tell. + // Fixed upstream in blockrun-llm#27 and blockrun-llm-ts#15. const { getMaxOutputTokens } = await import('../dist/agent/optimize.js'); + assert.equal(getMaxOutputTokens('openai/gpt-5.5'), 128_000); + assert.equal(getMaxOutputTokens('openai/gpt-5.4'), 128_000); assert.equal(getMaxOutputTokens('openai/gpt-5-mini'), 65_536); - assert.equal(getMaxOutputTokens('openai/gpt-5.5'), 32_768); - assert.equal(getMaxOutputTokens('openai/gpt-5.4'), 32_768); }); // ─── gateway catalog as a GAP-FILLER, never an override ─────────────────── From 920652658611fdc3b761477c5ce9b29652b769a3 Mon Sep 17 00:00:00 2001 From: 1bcMax <195689928+1bcMax@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:08:22 -0500 Subject: [PATCH 2/2] docs(tokens): record that Franklin does not run the SDK max_tokens guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of this PR turned up a wrong dependency claim I had made out loud: that the SDK fixes needed to merge before Franklin could raise these values, or users' local SDK would reject the larger number. Franklin never runs that guard. Both request paths — src/agent/llm.ts and src/proxy/server.ts — use raw fetch and import only payment helpers from @blockrun/llm, never chatCompletion, so validateMaxTokens is not on either path. These values are independent of whether blockrun-llm#27 and blockrun-llm-ts#15 land, and the merge ordering I recommended was unnecessary. Also drops the hardcoded '19 models' count, which would rot as the catalog changes, and marks the referenced SDK PRs as open rather than implying the root cause is already fixed everywhere. --- src/agent/optimize.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/agent/optimize.ts b/src/agent/optimize.ts index 546774f..1e23fab 100644 --- a/src/agent/optimize.ts +++ b/src/agent/optimize.ts @@ -53,11 +53,16 @@ const MODEL_MAX_OUTPUT: Record = { 'openai/gpt-5.6-sol': 128_000, 'openai/gpt-5.6-terra': 128_000, 'openai/gpt-5.6-luna': 128_000, - // 128000 is now measured, not read off the catalog. The earlier probe that + // 128000 is measured, not read off the catalog. The earlier probe that // seemed to refute it was invalid — both SDKs rejected max_tokens > 100000 // client-side, so nothing reached a provider. Re-probed 2026-07-21 with that - // guard bypassed: both accept 128000, as did all 19 models advertising a + // guard bypassed: both accept 128000, as did every model advertising a // ceiling above 100000. The catalog was right; the measurement was broken. + // + // Franklin does not run that SDK guard on either request path — the agent + // loop and the proxy both use raw fetch and import only payment helpers from + // @blockrun/llm — so these values are independent of whether the SDK-side + // fix lands. (blockrun-llm#27 / blockrun-llm-ts#15, open at time of writing.) 'openai/gpt-5.5': 128_000, 'openai/gpt-5.4': 128_000, 'openai/gpt-5.4-mini': 128_000,