Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions tests/unit/test_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import pytest
from blockrun_llm.validation import (
MAX_TOKENS_SANITY_LIMIT,
validate_private_key,
validate_api_url,
validate_model,
Expand Down Expand Up @@ -130,10 +131,6 @@ def test_accept_valid_values(self):
validate_max_tokens(100)
validate_max_tokens(1000)
validate_max_tokens(100000)
# Real ceilings the gateway serves — these were rejected before the
# sanity bound was raised, despite every provider accepting them.
validate_max_tokens(128000) # opus-4.8 / sonnet-5 / gpt-5.6 / glm-5
validate_max_tokens(262144) # zai/glm-5.2

def test_accept_none(self):
"""Should accept None."""
Expand All @@ -151,17 +148,30 @@ def test_reject_zero(self):

def test_reject_implausible(self):
"""Should reject values no model could mean (typo guard, not a limit)."""
with pytest.raises(ValueError, match="implausibly large") as exc:
validate_max_tokens(MAX_TOKENS_SANITY_LIMIT * 2)
# The message must name the SDK's own number, so a caller can tell it
# apart from a provider ceiling.
assert str(MAX_TOKENS_SANITY_LIMIT) in str(exc.value)

def test_boundary_is_inclusive(self):
"""Pin the exact edge: the limit passes, one above it fails.

Without this, flipping ``>`` to ``>=`` keeps the suite green while
rejecting a legal value.
"""
validate_max_tokens(MAX_TOKENS_SANITY_LIMIT)
with pytest.raises(ValueError, match="implausibly large"):
validate_max_tokens(2_000_000)
validate_max_tokens(MAX_TOKENS_SANITY_LIMIT + 1)

def test_does_not_cap_below_real_ceilings(self):
"""The bound must never be the binding constraint on a real request.

Regression: this was 100000, which rejected every ceiling above it
client-side — the caller saw a ValueError naming a limit no provider
had set, and the request never reached the network. Probed against the
live gateway 2026-07-21: 19 models advertise more than 100000 and all
19 accepted their advertised ceiling.
had set, and the request never reached the network. 128000 is the
common ceiling (opus-4.8 / sonnet-5 / gpt-5.6 / glm-5); 262144 is
zai/glm-5.2, the highest any model serves.
"""
for real_ceiling in (128_000, 262_144):
validate_max_tokens(real_ceiling)
Expand Down
Loading