diff --git a/architecture/dlq.md b/architecture/dlq.md index e7fe7f0..0703f8a 100644 --- a/architecture/dlq.md +++ b/architecture/dlq.md @@ -35,3 +35,5 @@ There is no built-in retention/pruning. Operators are responsible for archival `validate_schema()` delegates to `alembic.autogenerate.compare_metadata` against a throwaway `MetaData` populated by `make_outbox_table(...)` — so the canonical `Table` is the single source of truth and the validator never duplicates the schema declaration. It only flags **missing** schema (`add_*` / `modify_*` ops); `remove_*` ops are intentionally ignored so users may attach extras (audit columns, their own indexes). Alembic is an **optional dependency** (`faststream-outbox[validate]`); without it, `validate_schema()` raises `ImportError`, but every other code path works (the import lives at the top of `client.py` inside a try/except, with module-level sentinels `_alembic_compare_metadata` / `_AlembicMigrationContext` set to `None` on failure). Alembic's diff is **blind to three things the producer's `ON CONFLICT` arbiter and the lease invariant depend on**, so `validate_schema()` runs extra `pg_catalog` probes alongside it: the partial-index **WHERE predicates** (alembic ignores `postgresql_where`), the **uniqueness** of `timer_id_uq` (`pg_index.indisunique` — a same-named non-unique index passes the predicate check yet breaks `ON CONFLICT` at publish time), and the **`_lease_ck` CHECK** definition (alembic has no check-constraint comparator). Each surfaces a drifted/non-partial/non-unique index or a missing/altered CHECK that the diff alone would miss. + +The CHECK probe must not hard-code the `
_lease_ck` name: a `MetaData` carrying a SQLAlchemy `ck` `naming_convention` re-templates the explicitly-named `CheckConstraint` (the name becomes the `%(constraint_name)s` token, so the live constraint is e.g. `ck_
_
_lease_ck`). `_validate_check_constraints_sync` therefore reads the expected name **off the `Table` object** (`_resolve_check_constraint_name` identifies the lease constraint by its normalized predicate and returns its convention-resolved `.name`), falling back to the literal `
_lease_ck` only when the table carries no matching constraint. Without this, every deployment using the SQLAlchemy/Alembic-recommended convention falsely fails validation with "missing CHECK constraint". The explicitly-named indexes are **not** affected — the `ix`/`uq` convention keys only re-template auto-named indexes. diff --git a/docs/operations/alembic.md b/docs/operations/alembic.md index 519467e..d8b6787 100644 --- a/docs/operations/alembic.md +++ b/docs/operations/alembic.md @@ -216,9 +216,34 @@ Substitute the columns / `unique` / predicate from the table above for `outbox_pending_idx` and `outbox_lease_idx`. The recipes pass literal names (`'outbox_lease_ck'`, `'outbox_timer_id_uq'`) — -the exact names the package emits. If your `MetaData` carries a SQLAlchemy -`naming_convention`, wrap each name in `op.f('outbox_lease_ck')` so Alembic -treats it as final and does not re-template it. +the exact names the package emits **with no `naming_convention`**. + +If your `MetaData` carries a SQLAlchemy `naming_convention` with a `ck` key, the +lease `CheckConstraint` is re-templated — `outbox_lease_ck` becomes e.g. +`ck_outbox_outbox_lease_ck` (the explicit name fills the `%(constraint_name)s` +token). `validate_schema()` reads the expected CHECK name **off your `Table` +object**, so it already honours your convention — but your migration must create +the constraint under that **same** rendered name, or the probe won't find it. + +Introspect the rendered name from your own table and recreate the constraint +under it (`op.f(...)` passes it through literally): + +```python +from sqlalchemy import CheckConstraint + +# `outbox_table` is what `make_outbox_table(your_metadata)` returned. +ck = next(c for c in outbox_table.constraints if isinstance(c, CheckConstraint)) +print(ck.name) # the exact name to use, e.g. 'ck_outbox_outbox_lease_ck' + +op.create_check_constraint( + op.f(ck.name), + 'outbox', + '(acquired_token IS NULL) = (acquired_at IS NULL)', +) +``` + +The explicitly-named indexes (`outbox_pending_idx` etc.) are **not** re-templated +by the `ix`/`uq` convention keys, so their literal names stay correct as-is. ## DLQ retention via partition drop { #dlq-retention-via-partition-drop } diff --git a/faststream_outbox/client.py b/faststream_outbox/client.py index 286f4e8..a978a9d 100644 --- a/faststream_outbox/client.py +++ b/faststream_outbox/client.py @@ -23,6 +23,7 @@ from sqlalchemy import ( ARRAY, + CheckConstraint, Float, MetaData, String, @@ -543,6 +544,26 @@ def _validate_index_predicates_sync(connection: "Connection", table: "Table") -> } +def _resolve_check_constraint_name(table: "Table", suffix: str, want: str) -> str: + """ + Return the name the lease CHECK constraint actually carries on *table*. + + A ``MetaData`` with a SQLAlchemy ``ck`` ``naming_convention`` re-templates an + explicitly-named ``CheckConstraint`` — the package's ``
_lease_ck`` becomes e.g. + ``ck_
_
_lease_ck`` — so the live DB name is NOT ``f"{table.name}{suffix}"``. + The convention-resolved name is already carried on the constraint object, so identify + our constraint by its (normalized) predicate and read its ``.name``. Falls back to the + literal ``f"{table.name}{suffix}"`` when the table carries no matching constraint (a + reflected/hand-built ``Table``), preserving the original "missing" report. + """ + for constraint in table.constraints: + if isinstance(constraint, CheckConstraint) and constraint.name is not None: + predicate = _normalize_predicate(str(constraint.sqltext)).removeprefix("check ").strip() + if predicate == want: + return str(constraint.name) + return f"{table.name}{suffix}" + + def _validate_check_constraints_sync(connection: "Connection", table: "Table") -> list[str]: """ Compare the live CHECK constraint(s) against what the package expects. @@ -565,7 +586,7 @@ def _validate_check_constraints_sync(connection: "Connection", table: "Table") - live = {row["name"]: row["definition"] for row in rows} errors: list[str] = [] for suffix, want in _EXPECTED_CHECK_CONSTRAINTS.items(): - name = f"{table.name}{suffix}" + name = _resolve_check_constraint_name(table, suffix, want) if name not in live: errors.append(f"missing CHECK constraint {name!r} (expected '{want}')") continue diff --git a/planning/changes/active/2026-06-16.01-actionable-schema-drift-error/design.md b/planning/changes/active/2026-06-16.01-actionable-schema-drift-error/design.md deleted file mode 100644 index 786c124..0000000 --- a/planning/changes/active/2026-06-16.01-actionable-schema-drift-error/design.md +++ /dev/null @@ -1,194 +0,0 @@ ---- -status: draft -date: 2026-06-16 -slug: actionable-schema-drift-error -supersedes: null -superseded_by: null -pr: null -outcome: null ---- - -# Design: Actionable error for Alembic-blind schema drift - -## Summary - -`validate_schema()` detects schema drift that `alembic revision --autogenerate` -**cannot** generate a migration for — the `
_lease_ck` CHECK constraint and -the load-bearing partial-index predicates. Today the operator gets a bare -`RuntimeError` and a dead end: re-running autogenerate produces nothing. This -change appends a one-line pointer to that error — only when an Alembic-blind -drift is present — directing the operator to a new docs section that holds the -exact hand-written migration recipe. No comparator hook; no new exception type. - -## Motivation - -`make_outbox_table` declares three partial indexes (each with a load-bearing -`postgresql_where`) and a `
_lease_ck` CHECK -(`(acquired_token IS NULL) = (acquired_at IS NULL)`). On a **fresh** -`create_table`, autogenerate renders all of it. On an **incremental** migration -onto a pre-existing table, Alembic's `compare_metadata` has no check-constraint -comparator and its index comparator ignores `postgresql_where`, so a missing or -drifted CHECK — and a non-partial / wrong-predicate / non-unique index — ships -silently. `validate_schema()` backstops this with direct `pg_catalog` / -`pg_constraint` probes (`_validate_index_predicates_sync`, -`_validate_check_constraints_sync`), so the drift *is* caught at runtime. - -But the resulting error — e.g. - -> `RuntimeError: Outbox schema mismatch: missing CHECK constraint 'outbox_lease_ck' (expected 'acquired_token is null = acquired_at is null')` - -— is a dead end. The operator runs `alembic revision --autogenerate` expecting a -remediation migration and gets an empty `upgrade()`, because the same blindness -that let the drift through also prevents autogenerate from fixing it. The -detection is correct; the remediation path is missing. - -The third validation pass (the Alembic `compare_metadata` diff for -tables/columns/plain indexes) reports drift that autogenerate *can* fix, so it -needs no special handling — the operator there just re-runs autogenerate. - -## Non-goals - -- An Alembic autogenerate comparator hook ("make autogenerate emit it"): - explicitly declined — we are not registering a `comparators.dispatch_for` - extension or asking users to wire one into `env.py`. -- A structured exception type (`SchemaMismatchError` with a `.remediation` - list): out of scope; the raised type stays `RuntimeError`. -- Embedding the full copy-pasteable DDL in the error text: the error carries a - short pointer; the DDL lives in docs. -- Server-default drift: still not validated (`compare_server_default=False`, - unchanged). - -## Design - -### 1. Error composition in `validate_schema()` (`client.py`) - -`validate_schema()` runs the same three probe groups as today, but tracks -whether the **Alembic-blind** probes contributed any errors. The blind probes -are exactly `_validate_index_predicates_sync` and -`_validate_check_constraints_sync`, both run only on the outbox table -(`self._table`) — the DLQ table declares no partial-index predicates or CHECK -constraints, so it is validated solely by the Alembic diff. The Alembic -`compare_metadata` diff (`_validate_schema_sync` / `_validate_dlq_schema_sync`) -is autogenerate-fixable and does **not** set the blind flag. - -The message is built through a new **pure** helper so the append logic is -unit-testable without a live engine (the 100 % coverage gate forbids an -untested branch, and `test_unit.py` runs with no Postgres): - -```python -_SCHEMA_MISMATCH_PREFIX = "Outbox schema mismatch: " -_AUTOGEN_BLIND_HINT = ( - "These (CHECK constraints and partial-index predicates) are invisible to " - "'alembic revision --autogenerate' — hand-write the migration: " - "https://faststream-outbox.modern-python.org/operations/alembic/" - "#fixing-drift-autogenerate-cant-see" -) - - -def _compose_schema_mismatch_message(errors: list[str], *, has_blind_drift: bool) -> str: - msg = _SCHEMA_MISMATCH_PREFIX + "; ".join(errors) - if has_blind_drift: - msg += "\n\n" + _AUTOGEN_BLIND_HINT - return msg -``` - -`validate_schema()` collects the blind-probe errors separately, ORs their -presence into `has_blind_drift`, and raises -`RuntimeError(_compose_schema_mismatch_message(errors, has_blind_drift=...))`. - -The prefix `"Outbox schema mismatch: " + "; ".join(errors)` and the per-error -strings are unchanged, so every existing `pytest.raises(..., match=...)` -substring assertion keeps passing. The pointer is appended on its own line -(`\n\n`) after the joined errors. - -### 2. Docs section (`docs/operations/alembic.md`) - -New section `## Fixing drift autogenerate can't see { #fixing-drift-autogenerate-cant-see }`, -placed after "Drift detection in CI" and before "DLQ retention via partition -drop". It states the two Alembic-blind classes and why autogenerate misses them -(no check-constraint comparator; index comparator ignores `postgresql_where`), -then gives exact hand-written `op.*` recipes: - -- **Missing or drifted `lease_ck` CHECK** — drop first only if it exists but - drifted, then create: - - ```python - # only if it exists with a wrong predicate: - op.drop_constraint('outbox_lease_ck', 'outbox', type_='check') - op.create_check_constraint( - 'outbox_lease_ck', 'outbox', - '(acquired_token IS NULL) = (acquired_at IS NULL)', - ) - ``` - -- **Non-partial / wrong-predicate / non-unique index** — drop and recreate with - the load-bearing `postgresql_where` (and `unique=True` for the timer-id - index): - - ```python - op.drop_index('outbox_timer_id_uq', table_name='outbox') - op.create_index( - 'outbox_timer_id_uq', 'outbox', ['queue', 'timer_id'], - unique=True, postgresql_where=sa.text('timer_id IS NOT NULL'), - ) - # outbox_pending_idx: postgresql_where=sa.text('acquired_token IS NULL') - # outbox_lease_idx: postgresql_where=sa.text('acquired_token IS NOT NULL') - ``` - -The recipe names the three index suffixes (`_pending_idx`, `_lease_idx`, -`_timer_id_uq`) and their expected predicates, matching -`_EXPECTED_INDEX_PREDICATES` in `client.py`. - -A one-line cross-link is added from `docs/usage/schema-validation.md` to this -anchor. - -### 3. Anchor stability - -The error URL is `https://faststream-outbox.modern-python.org/operations/alembic/#fixing-drift-autogenerate-cant-see` -(site_url from `mkdocs.yml`, directory-URL form matching the existing -`#dlq-retention-via-partition-drop` anchor). The explicit `{ #... }` attr-list -anchor pins the slug so a later heading reword can't silently break the link. - -## Operations - -None. No infra, DNS, or external-account changes. - -## Testing - -- **`test_unit.py`** (no Postgres): - - `_compose_schema_mismatch_message(errors, has_blind_drift=True)` contains the - pointer URL and the `;`-joined prefix. - - `has_blind_drift=False` omits the pointer entirely. - - Prefix and `; ` join format are intact in both cases. -- **`test_integration.py`**: - - Extend an Alembic-blind case (e.g. - `test_validate_schema_fails_when_lease_check_constraint_missing`) to assert - the pointer URL is in the raised message. - - Assert an autogenerate-fixable case (e.g. - `test_validate_schema_fails_when_columns_missing`) does **not** contain the - pointer. -- **Docs**: `just docs-build` (`mkdocs build --strict`) passes — the new anchor - resolves and the cross-link from `schema-validation.md` does not 404. -- **Lint**: `just lint-ci` clean. - -## Risk - -- **Low — link rot.** If the docs site_url or page path changes, the error URL - goes stale. Mitigation: the explicit attr-list anchor + the `--strict` docs - build (which fails on a broken in-repo cross-link) catch the in-repo half; the - hostname is the one piece a strict build can't verify, and it is the published - canonical domain. -- **Low — flag plumbing.** `has_blind_drift` must be ORed from the two blind - probes specifically, not from the full error list, or the pointer would also - fire on pure column drift. Covered by the autogenerate-fixable negative test. -- **Negligible — message contract.** Appending a trailing line preserves the - prefix and per-error substrings, so existing `match=` assertions and any - operator log-greps on the prefix are unaffected. - -## On merge - -Promote into `architecture/`: the schema-validation / drift behavior is -described in `CLAUDE.md`'s "User-owned schema" section (and any -`architecture/` deep-dive that covers `validate_schema`). Add a sentence noting -that an Alembic-blind drift error now carries a remediation pointer to -`docs/operations/alembic.md#fixing-drift-autogenerate-cant-see`. diff --git a/planning/changes/active/2026-06-16.01-actionable-schema-drift-error/plan.md b/planning/changes/active/2026-06-16.01-actionable-schema-drift-error/plan.md deleted file mode 100644 index 1a6ca34..0000000 --- a/planning/changes/active/2026-06-16.01-actionable-schema-drift-error/plan.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -status: draft -date: 2026-06-16 -slug: actionable-schema-drift-error -spec: actionable-schema-drift-error -pr: null ---- - -# actionable-schema-drift-error — implementation plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> superpowers:subagent-driven-development (recommended) or -> superpowers:executing-plans to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `validate_schema()`'s `RuntimeError` point operators to a -hand-written-migration recipe whenever it reports drift that -`alembic revision --autogenerate` cannot remediate (the `lease_ck` CHECK and the -partial-index predicates). - -**Spec:** [`design.md`](./design.md) - -**Branch:** `fix/actionable-schema-drift-error` - -**Commit strategy:** Per-task commits. - -**Tooling notes for the executor:** -- Pure-helper unit tests run with no Postgres: `uv run pytest tests/test_unit.py -k `. -- Integration tests need Postgres via docker compose: `just test tests/test_integration.py -k `. - Pass a single `-k` keyword — `just test -k "a or b"` word-splits and fails. -- Coverage gate is `--cov-fail-under=100` on the full `just test` run; partial - runs fail it, so use `--no-cov` while iterating and rely on the final full run. -- All imports go at module top — never inline (project convention). Type every - new test parameter, including fixtures. - ---- - -### Task 1: Add the pure message-composition helper (TDD) - -**Files:** -- Modify: `faststream_outbox/client.py` -- Test: `tests/test_unit.py` - -Introduce `_compose_schema_mismatch_message` plus the `_SCHEMA_MISMATCH_PREFIX` -and `_AUTOGEN_BLIND_HINT` constants. Pure function, fully unit-testable without -Postgres. - -- [ ] **Step 1: Write the failing tests** - - Add to `tests/test_unit.py`. Extend the existing client import on line 42 - (`from faststream_outbox.client import OutboxClient, _validate_schema_sync`) to - also import the new names: - - ```python - from faststream_outbox.client import ( - OutboxClient, - _AUTOGEN_BLIND_HINT, - _SCHEMA_MISMATCH_PREFIX, - _compose_schema_mismatch_message, - _validate_schema_sync, - ) - ``` - - Then add these three tests (place them near the other client unit tests): - - ```python - def test_compose_schema_mismatch_message_appends_hint_on_blind_drift() -> None: - msg = _compose_schema_mismatch_message( - ["missing CHECK constraint 'outbox_lease_ck' (expected '...')"], - has_blind_drift=True, - ) - assert msg.startswith(_SCHEMA_MISMATCH_PREFIX) - assert _AUTOGEN_BLIND_HINT in msg - assert "#fixing-drift-autogenerate-cant-see" in msg - - - def test_compose_schema_mismatch_message_omits_hint_without_blind_drift() -> None: - msg = _compose_schema_mismatch_message( - ["table 'outbox' missing column 'headers'"], - has_blind_drift=False, - ) - assert msg == _SCHEMA_MISMATCH_PREFIX + "table 'outbox' missing column 'headers'" - assert _AUTOGEN_BLIND_HINT not in msg - - - def test_compose_schema_mismatch_message_joins_multiple_errors() -> None: - msg = _compose_schema_mismatch_message(["a", "b"], has_blind_drift=False) - assert msg == _SCHEMA_MISMATCH_PREFIX + "a; b" - ``` - -- [ ] **Step 2: Run the tests to verify they fail** - - Run: `uv run pytest tests/test_unit.py -k compose_schema_mismatch --no-cov -v` - Expected: collection/import error or FAIL — `_compose_schema_mismatch_message` - (and the two constants) do not exist yet. - -- [ ] **Step 3: Implement the helper in `client.py`** - - Add the following at module scope in `faststream_outbox/client.py`, immediately - after the `_validate_check_constraints_sync` function (after the block ending at - the current line 576), so all schema-validation machinery stays grouped: - - ```python - # The published docs anchor for hand-written migrations that fix drift - # `alembic revision --autogenerate` cannot emit (no check-constraint comparator; - # the index comparator ignores postgresql_where). Appended to the RuntimeError - # only when an Alembic-blind probe actually fired — see validate_schema(). - _SCHEMA_MISMATCH_PREFIX = "Outbox schema mismatch: " - _AUTOGEN_BLIND_HINT = ( - "These (CHECK constraints and partial-index predicates) are invisible to " - "'alembic revision --autogenerate' — hand-write the migration: " - "https://faststream-outbox.modern-python.org/operations/alembic/" - "#fixing-drift-autogenerate-cant-see" - ) - - - def _compose_schema_mismatch_message(errors: list[str], *, has_blind_drift: bool) -> str: - """Build the validate_schema RuntimeError text; append the remediation pointer for Alembic-blind drift.""" - msg = _SCHEMA_MISMATCH_PREFIX + "; ".join(errors) - if has_blind_drift: - msg += "\n\n" + _AUTOGEN_BLIND_HINT - return msg - ``` - -- [ ] **Step 4: Run the tests to verify they pass** - - Run: `uv run pytest tests/test_unit.py -k compose_schema_mismatch --no-cov -v` - Expected: 3 passed. - -- [ ] **Step 5: Commit** - - ```bash - git add faststream_outbox/client.py tests/test_unit.py - git commit -m "feat(client): add schema-mismatch message composer with autogen-blind hint - - Co-Authored-By: Claude Opus 4.8 (1M context) " - ``` - ---- - -### Task 2: Wire `validate_schema()` to the helper and assert the pointer in integration tests - -**Files:** -- Modify: `faststream_outbox/client.py:401-425` (the `validate_schema` method) -- Test: `tests/test_integration.py` - -Track the Alembic-blind probe errors separately and raise through the new helper. -Cover the present/absent pointer behaviour against real Postgres. - -- [ ] **Step 1: Update the integration tests (failing assertions first)** - - In `tests/test_integration.py`, edit - `test_validate_schema_fails_when_lease_check_constraint_missing` (currently - lines 1919-1929) to capture the exception and assert the pointer: - - ```python - async def test_validate_schema_fails_when_lease_check_constraint_missing( - pg_engine: AsyncEngine, - outbox_table: Table, - ) -> None: - """A dropped ``
_lease_ck`` CHECK must be caught — alembic's diff can't see it (audit 2026-06-14).""" - drop_sql = f'ALTER TABLE "{outbox_table.name}" DROP CONSTRAINT "{outbox_table.name}_lease_ck"' - async with pg_engine.begin() as conn: - await conn.exec_driver_sql(drop_sql) - client = OutboxClient(pg_engine, outbox_table) - with pytest.raises(RuntimeError, match="missing CHECK constraint") as excinfo: - await client.validate_schema() - assert "operations/alembic/#fixing-drift-autogenerate-cant-see" in str(excinfo.value) - ``` - - And edit `test_validate_schema_fails_when_columns_missing` (currently lines - 454-461) to assert the pointer is **absent** for autogenerate-fixable drift: - - ```python - async def test_validate_schema_fails_when_columns_missing(pg_engine, outbox_table) -> None: - """Drop a column the package expects and verify validate_schema reports it.""" - drop_sql = f'ALTER TABLE "{outbox_table.name}" DROP COLUMN headers' - async with pg_engine.begin() as conn: - await conn.exec_driver_sql(drop_sql) - client = OutboxClient(pg_engine, outbox_table) - with pytest.raises(RuntimeError, match="missing column 'headers'") as excinfo: - await client.validate_schema() - assert "fixing-drift-autogenerate-cant-see" not in str(excinfo.value) - ``` - -- [ ] **Step 2: Run the two tests to verify the new assertions fail** - - Run: `just test tests/test_integration.py -k test_validate_schema_fails_when_lease_check_constraint_missing` - Expected: FAIL — the current message has no pointer, so the new - `assert ... in str(excinfo.value)` fails. - (The columns-missing test still passes at this point — the pointer is already - absent — but its new negative assertion guards Step 3.) - -- [ ] **Step 3: Rewire `validate_schema` in `client.py`** - - Replace the body of `validate_schema` (lines 410-425) — keep the docstring - above it unchanged. Old: - - ```python - async with self._engine.connect() as conn: - errors = await conn.run_sync(_validate_schema_sync, self._table) - # S2: alembic's autogenerate diff compares index columns + uniqueness but NOT - # the partial-index WHERE predicate, so a wrong postgresql_where slips through - # and later breaks the producer's ON CONFLICT arbiter. Probe the predicates - # directly against the live catalog. - errors.extend(await conn.run_sync(_validate_index_predicates_sync, self._table)) - # Alembic's compare_metadata has no check-constraint comparator, so a missing - # or altered
_lease_ck (the half-set-lease guard) passes the diff above - # silently. Probe pg_constraint directly, mirroring the partial-index probe. - errors.extend(await conn.run_sync(_validate_check_constraints_sync, self._table)) - if self._dlq_table is not None: - errors.extend(await conn.run_sync(_validate_dlq_schema_sync, self._dlq_table)) - if errors: - msg = "Outbox schema mismatch: " + "; ".join(errors) - raise RuntimeError(msg) - ``` - - New: - - ```python - async with self._engine.connect() as conn: - errors = await conn.run_sync(_validate_schema_sync, self._table) - # S2 / lease_ck: these two probes catch drift that `alembic revision - # --autogenerate` cannot remediate — its index comparator ignores - # postgresql_where and it has no check-constraint comparator at all. - # Collect them separately so the raised error can point operators at - # the hand-written-migration recipe (_AUTOGEN_BLIND_HINT) only when - # one of them actually fired. - blind_errors = await conn.run_sync(_validate_index_predicates_sync, self._table) - blind_errors.extend(await conn.run_sync(_validate_check_constraints_sync, self._table)) - errors.extend(blind_errors) - if self._dlq_table is not None: - errors.extend(await conn.run_sync(_validate_dlq_schema_sync, self._dlq_table)) - if errors: - raise RuntimeError( - _compose_schema_mismatch_message(errors, has_blind_drift=bool(blind_errors)), - ) - ``` - -- [ ] **Step 4: Run the affected integration tests to verify they pass** - - Run each separately (single `-k` keyword each): - - `just test tests/test_integration.py -k test_validate_schema_fails_when_lease_check_constraint_missing` - - `just test tests/test_integration.py -k test_validate_schema_fails_when_columns_missing` - - `just test tests/test_integration.py -k test_validate_schema_fails_when_lease_check_constraint_predicate_wrong` - Expected: PASS for each. The third confirms the predicate-drift path also still - raises (it now flows through the helper with `has_blind_drift=True`). - -- [ ] **Step 5: Commit** - - ```bash - git add faststream_outbox/client.py tests/test_integration.py - git commit -m "fix(client): point operators at the migration recipe on autogen-blind drift - - Co-Authored-By: Claude Opus 4.8 (1M context) " - ``` - ---- - -### Task 3: Document the hand-written-migration recipe - -**Files:** -- Modify: `docs/operations/alembic.md` -- Modify: `docs/usage/schema-validation.md` - -Add the anchored section the error points to, and cross-link it from the schema -validation page. - -- [ ] **Step 1: Add the recipe section to `docs/operations/alembic.md`** - - Insert the following **between** the end of the "Drift detection in CI" section - (after the server-defaults paragraph that ends at line 158) and the - `## DLQ retention via partition drop { #dlq-retention-via-partition-drop }` - heading (line 160): - - ````markdown - ## Fixing drift autogenerate can't see { #fixing-drift-autogenerate-cant-see } - - Two kinds of drift that - [`validate_schema()`](../usage/schema-validation.md) reports **cannot** be - remediated by `alembic revision --autogenerate` — the same blindness that - let them drift in also stops autogenerate from emitting a fix: - - - **The `outbox_lease_ck` CHECK constraint.** Alembic's `compare_metadata` - has no check-constraint comparator, so a missing or altered CHECK never - appears in an autogenerated migration. - - **Partial-index predicates.** Alembic's index comparator ignores - `postgresql_where`, so an index that exists but was created non-partial, - with the wrong `WHERE`, or (for `outbox_timer_id_uq`) non-unique is - invisible to the diff. - - When `validate_schema()` raises for one of these, its error ends with a - pointer to this section. Re-running autogenerate produces an empty - `upgrade()` — hand-write the migration instead, then re-run - `validate_schema()` to confirm the drift is cleared. - - ### Restore the lease CHECK - - ```python - # Drop first ONLY if the constraint exists with a wrong predicate; skip the - # drop if it is absent entirely. - op.drop_constraint('outbox_lease_ck', 'outbox', type_='check') - op.create_check_constraint( - 'outbox_lease_ck', - 'outbox', - '(acquired_token IS NULL) = (acquired_at IS NULL)', - ) - ``` - - ### Restore a partial index - - Drop the drifted index and recreate it with its load-bearing predicate. The - three indexes and their expected shape: - - | Index | Columns | Unique | `postgresql_where` | - | --- | --- | --- | --- | - | `outbox_pending_idx` | `queue, next_attempt_at` | no | `acquired_token IS NULL` | - | `outbox_lease_idx` | `queue, acquired_at` | no | `acquired_token IS NOT NULL` | - | `outbox_timer_id_uq` | `queue, timer_id` | yes | `timer_id IS NOT NULL` | - - ```python - op.drop_index('outbox_timer_id_uq', table_name='outbox') - op.create_index( - 'outbox_timer_id_uq', - 'outbox', - ['queue', 'timer_id'], - unique=True, - postgresql_where=sa.text('timer_id IS NOT NULL'), - ) - ``` - - Substitute the columns / `unique` / predicate from the table above for - `outbox_pending_idx` and `outbox_lease_idx`. - ```` - -- [ ] **Step 2: Cross-link from `docs/usage/schema-validation.md`** - - Insert this paragraph immediately after the "Extras are intentionally ignored" - paragraph (after line 39, before the `!!! warning "Server defaults..."` - admonition): - - ```markdown - Some drift cannot be fixed by re-running `alembic revision --autogenerate` — a - missing/altered `outbox_lease_ck` CHECK or a drifted partial-index predicate. - For those, the `RuntimeError` ends with a pointer to - [Alembic migrations § Fixing drift autogenerate can't see](../operations/alembic.md#fixing-drift-autogenerate-cant-see), - which holds the hand-written migration recipe. - ``` - -- [ ] **Step 3: Build the docs strict to verify anchors resolve** - - Run: `just docs-build` - Expected: `mkdocs build --strict` succeeds with no warnings — the new - `#fixing-drift-autogenerate-cant-see` anchor resolves and the cross-link from - `schema-validation.md` does not 404. - -- [ ] **Step 4: Commit** - - ```bash - git add docs/operations/alembic.md docs/usage/schema-validation.md - git commit -m "docs: hand-written migration recipe for autogen-blind drift - - Co-Authored-By: Claude Opus 4.8 (1M context) " - ``` - ---- - -### Task 4: Full verification - -**Files:** none (verification only). - -Confirm lint, the full test suite, and the 100% coverage gate all pass with the -changes in place. - -- [ ] **Step 1: Lint** - - Run: `just lint-ci` - Expected: clean — `eof-fixer`, `ruff format --check`, `ruff check`, `ty check` - all pass. (If `_compose_schema_mismatch_message` or the constants trip an unused - import in `test_unit.py`, fix the import list rather than suppressing.) - -- [ ] **Step 2: Full test suite with coverage gate** - - Run: `just test` - Expected: all tests pass and `--cov-fail-under=100` is satisfied — the helper's - three branches are covered by the Task 1 unit tests; the `has_blind_drift` True - and False paths in `validate_schema` are covered by the Task 2 integration tests. - -- [ ] **Step 3: Commit (only if Step 1/2 required fixups)** - - ```bash - git add -A - git commit -m "chore: lint/coverage fixups for autogen-blind drift hint - - Co-Authored-By: Claude Opus 4.8 (1M context) " - ``` - ---- - -## On merge - -Move this bundle to `planning/changes/archive/` with `status: shipped`, `pr:`, -and `outcome:` filled, and promote the conclusion into the affected -architecture record: note in `CLAUDE.md`'s "User-owned schema" section (and any -`architecture/` deep-dive covering `validate_schema`) that an Alembic-blind -drift error now carries a remediation pointer to -`docs/operations/alembic.md#fixing-drift-autogenerate-cant-see`. diff --git a/planning/releases/0.10.2.md b/planning/releases/0.10.2.md new file mode 100644 index 0000000..0c2f596 --- /dev/null +++ b/planning/releases/0.10.2.md @@ -0,0 +1,68 @@ +# faststream-outbox 0.10.2 — schema validation honors `naming_convention` + +**Patch release.** One bug fix to `validate_schema()`'s CHECK-constraint probe. +No public-API change. The only change to the installed package is which +constraint name the probe looks up; everything else is identical to 0.10.1. + +## Fixes + +- **`validate_schema()` no longer false-fails under a SQLAlchemy `ck` + `naming_convention`.** A `MetaData` carrying a `naming_convention` with a `ck` + key re-templates the package's explicitly-named lease `CheckConstraint` — the + given name becomes the `%(constraint_name)s` token, so the live constraint is + named e.g. `ck_
_
_lease_ck`, not `
_lease_ck`. The probe + hard-coded the literal `
_lease_ck`, never found the re-templated name, + and raised a spurious + + > `Outbox schema mismatch: missing CHECK constraint '
_lease_ck' …` + + on a perfectly valid schema. `_validate_check_constraints_sync` now reads the + expected name **off the `Table` object** (identifying the lease constraint by + its normalized predicate and using its convention-resolved `.name`), so the + expectation always matches what SQLAlchemy / Alembic emit from your metadata — + convention or not. The explicitly-named indexes were never affected: the + `ix` / `uq` convention keys only re-template *auto-named* indexes. + +## Compatibility + +`validate_schema()` is **opt-in** (you call it from a health check / CI gate; it +is never run by `broker.start()`). If you don't call it, nothing changes. + +If you do, and you use **no** `naming_convention`, behavior is identical to +0.10.1 — the probe still resolves to the literal `
_lease_ck`. If you use a +`ck` convention, validation that previously raised spuriously now passes, +provided the constraint in your DB carries the convention-rendered name (which it +does when the schema was created via `MetaData.create_all` or a +convention-aware Alembic migration). Hand-written migrations must create the +constraint under that same rendered name — see the updated +[`naming_convention` guidance](https://faststream-outbox.modern-python.org/operations/alembic/#fixing-drift-autogenerate-cant-see) +in the Alembic docs. + +No other behavior change — producers, subscribers, the lease / terminal-write +paths, timers, the index/uniqueness probes, and the `dlq_table=None` path are +all identical to 0.10.1. + +## Docs + +- `docs/operations/alembic.md` — replaced the misleading + "wrap each name in `op.f('outbox_lease_ck')`" caveat with an + introspect-the-rendered-name recipe (the probe now expects the + convention-resolved name, not the literal). +- `architecture/dlq.md` — recorded the convention-awareness invariant in the + `validate_schema()` mechanics section. + +## Touched surface + +- `faststream_outbox/client.py` — new `_resolve_check_constraint_name` + + `_validate_check_constraints_sync` reads the expected CHECK name off the + `Table`. **Only package code change.** +- `docs/operations/alembic.md`, `architecture/dlq.md` — caveat fix + invariant. +- `tests/test_unit.py`, `tests/test_integration.py` — regression coverage + (convention-resolved name honored, missing-under-convention, literal-name + fallback, and an end-to-end Postgres pass under a `ck` convention). + +## See also + +- Follow-up to the 0.10.1 schema-validation work + ([#99](https://github.com/modern-python/faststream-outbox/pull/99) change + bundle: [`planning/changes/archive/2026-06-16.01-actionable-schema-drift-error/`](../changes/archive/2026-06-16.01-actionable-schema-drift-error/design.md)). diff --git a/tests/test_integration.py b/tests/test_integration.py index 8e0485c..d4d8e84 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1931,6 +1931,30 @@ async def test_validate_schema_fails_when_lease_check_constraint_missing( assert "operations/alembic/#fixing-drift-autogenerate-cant-see" in str(excinfo.value) +async def test_validate_schema_passes_under_ck_naming_convention( + pg_engine: AsyncEngine, +) -> None: + """ + A MetaData with a ``ck`` naming convention renames the lease CHECK to ``ck___lease_ck``. + + The probe must look it up under that resolved name (carried on the constraint object), not the + literal ``_lease_ck`` — otherwise a perfectly valid schema falsely raises "missing CHECK + constraint" for every deployment using the SQLAlchemy/Alembic-recommended convention. + """ + convention = {"ck": "ck_%(table_name)s_%(constraint_name)s"} + metadata = MetaData(naming_convention=convention) + table_name = f"test_outbox_{uuid.uuid4().hex[:12]}" + table = make_outbox_table(metadata, table_name=table_name) + async with pg_engine.begin() as conn: + await conn.run_sync(metadata.create_all) + try: + client = OutboxClient(pg_engine, table) + await client.validate_schema() # must NOT raise + finally: + async with pg_engine.begin() as conn: + await conn.run_sync(metadata.drop_all) + + async def test_validate_schema_fails_when_lease_check_constraint_predicate_wrong( pg_engine: AsyncEngine, outbox_table: Table, diff --git a/tests/test_unit.py b/tests/test_unit.py index 5eb3be0..ff2b702 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -44,6 +44,7 @@ _SCHEMA_MISMATCH_PREFIX, OutboxClient, _compose_schema_mismatch_message, + _validate_check_constraints_sync, _validate_schema_sync, ) from faststream_outbox.configs import OutboxBrokerConfig @@ -1385,6 +1386,52 @@ def test_compose_schema_mismatch_message_joins_multiple_errors() -> None: assert msg == _SCHEMA_MISMATCH_PREFIX + "a; b" +# SQLAlchemy re-templates an explicitly-named CheckConstraint through a MetaData's ``ck`` +# naming convention (the explicit name becomes the ``%(constraint_name)s`` token), so the live +# constraint name is NOT ``
_lease_ck`` — it is ``ck_
_
_lease_ck``. The probe +# must look it up under the convention-resolved name carried on the constraint object, not a +# literal suffix, or it falsely reports a correct schema as "missing CHECK constraint". +_CK_CONVENTION = {"ck": "ck_%(table_name)s_%(constraint_name)s"} +_RESOLVED_LEASE_CK_NAME = "ck_outbox_faststream_outbox_faststream_lease_ck" +_LEASE_CK_PREDICATE = "acquired_token is null = acquired_at is null" + + +def _mock_check_constraint_connection(rows: list[dict[str, str]]) -> MagicMock: + """Build a connection whose ``execute(...).mappings().all()`` yields *rows* (name/definition dicts).""" + connection = MagicMock() + connection.execute.return_value.mappings.return_value.all.return_value = rows + return connection + + +def test_validate_check_constraints_honors_naming_convention() -> None: + metadata = MetaData(naming_convention=_CK_CONVENTION) + table = make_outbox_table(metadata, table_name="outbox_faststream") + connection = _mock_check_constraint_connection( + [{"name": _RESOLVED_LEASE_CK_NAME, "definition": "CHECK (((acquired_token IS NULL) = (acquired_at IS NULL)))"}], + ) + assert _validate_check_constraints_sync(connection, table) == [] + + +def test_validate_check_constraints_missing_reports_convention_resolved_name() -> None: + metadata = MetaData(naming_convention=_CK_CONVENTION) + table = make_outbox_table(metadata, table_name="outbox_faststream") + connection = _mock_check_constraint_connection([]) # constraint absent from the live DB + errors = _validate_check_constraints_sync(connection, table) + assert errors == [f"missing CHECK constraint {_RESOLVED_LEASE_CK_NAME!r} (expected '{_LEASE_CK_PREDICATE}')"] + + +def test_validate_check_constraints_falls_back_to_literal_name_without_constraint_object() -> None: + """ + Fall back to the literal ``
_lease_ck`` when the Table carries no lease CheckConstraint. + + A hand-built/reflected ``Table`` has no convention to resolve, so the "missing" report still fires. + """ + table = Table("bare_outbox", MetaData()) # no CheckConstraint attached + connection = _mock_check_constraint_connection([]) + errors = _validate_check_constraints_sync(connection, table) + assert errors == [f"missing CHECK constraint 'bare_outbox_lease_ck' (expected '{_LEASE_CK_PREDICATE}')"] + + async def test_broker_ping_done_subscriber_task_is_false() -> None: metadata = MetaData()