From 752ea4eedc73ab2ef6964837b2a827e4cdbab07a Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Tue, 21 Jul 2026 00:15:13 -0500 Subject: [PATCH] test: assert the error names the SDK's own number, not a model's Follow-up to #27. Two gaps in its tests: - The real-ceiling assertions (128000, 262144) sat inside test_accept_valid_values alongside 1/100/1000, so a failure there would not say which kind of value broke. Split into their own test with the probe evidence attached. - Nothing asserted the message content. The whole reason #27 exists is that the old text, 'max_tokens too large (maximum: 100000)', read like a provider response and was recorded as an upstream model ceiling in a downstream token table. If the message regresses to that shape the bug returns, and no test would have caught it. Now pinned: the message must name the SDK's own limit and disclaim being a model limit. Also references MAX_TOKENS_SANITY_LIMIT instead of hardcoding 2_000_000, so the test follows the constant if it moves. --- tests/unit/test_validation.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index 02933b9..ae68294 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -2,6 +2,7 @@ import pytest from blockrun_llm.validation import ( + MAX_TOKENS_SANITY_LIMIT, validate_private_key, validate_api_url, validate_model, @@ -130,10 +131,6 @@ def test_accept_valid_values(self): validate_max_tokens(100) validate_max_tokens(1000) validate_max_tokens(100000) - # Real ceilings the gateway serves — these were rejected before the - # sanity bound was raised, despite every provider accepting them. - validate_max_tokens(128000) # opus-4.8 / sonnet-5 / gpt-5.6 / glm-5 - validate_max_tokens(262144) # zai/glm-5.2 def test_accept_none(self): """Should accept None.""" @@ -151,17 +148,30 @@ def test_reject_zero(self): def test_reject_implausible(self): """Should reject values no model could mean (typo guard, not a limit).""" + with pytest.raises(ValueError, match="implausibly large") as exc: + validate_max_tokens(MAX_TOKENS_SANITY_LIMIT * 2) + # The message must name the SDK's own number, so a caller can tell it + # apart from a provider ceiling. + assert str(MAX_TOKENS_SANITY_LIMIT) in str(exc.value) + + def test_boundary_is_inclusive(self): + """Pin the exact edge: the limit passes, one above it fails. + + Without this, flipping ``>`` to ``>=`` keeps the suite green while + rejecting a legal value. + """ + validate_max_tokens(MAX_TOKENS_SANITY_LIMIT) with pytest.raises(ValueError, match="implausibly large"): - validate_max_tokens(2_000_000) + validate_max_tokens(MAX_TOKENS_SANITY_LIMIT + 1) def test_does_not_cap_below_real_ceilings(self): """The bound must never be the binding constraint on a real request. Regression: this was 100000, which rejected every ceiling above it client-side — the caller saw a ValueError naming a limit no provider - had set, and the request never reached the network. Probed against the - live gateway 2026-07-21: 19 models advertise more than 100000 and all - 19 accepted their advertised ceiling. + had set, and the request never reached the network. 128000 is the + common ceiling (opus-4.8 / sonnet-5 / gpt-5.6 / glm-5); 262144 is + zai/glm-5.2, the highest any model serves. """ for real_ceiling in (128_000, 262_144): validate_max_tokens(real_ceiling)