feat: spec-sync pipeline (Problem 3)

.github/workflows/pr-gates.yml

@@ -0,0 +1,52 @@
+name: PR Gates
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  surface-lock:
+    # Escape hatch for intentional breaking changes (major releases): label the PR
+    # `breaking-change-approved` to skip the released-surface lock.
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'breaking-change-approved') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0     # tags for the release-tag baseline
+      - name: Install Rye
+        run: |
+          curl -sSf https://rye.astral.sh/get | bash
+          echo "$HOME/.rye/shims" >> "$GITHUB_PATH"
+        env:
+          RYE_VERSION: '0.44.0'
+          RYE_INSTALL_OPTION: '--yes'
+      - name: Install dependencies
+        run: rye sync --all-features
+      - name: Surface lock (no breaking change to released API)
+        run: ./scripts/spec-sync/surface-lock.sh
+
+  contract-tests:
+    # Only spec-sync PRs are gated on the live staging API. Gating *every* PR would let a
+    # staging outage or transient 5xx block all merges repo-wide. Keep this job OUT of the
+    # required-checks list for ordinary PRs (it is skipped there); require it only where it runs.
+    if: startsWith(github.head_ref, 'spec-sync/')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install Rye
+        run: |
+          curl -sSf https://rye.astral.sh/get | bash
+          echo "$HOME/.rye/shims" >> "$GITHUB_PATH"
+        env:
+          RYE_VERSION: '0.44.0'
+          RYE_INSTALL_OPTION: '--yes'
+      - name: Install dependencies
+        run: rye sync --all-features
+      - name: Contract tests vs staging
+        env:
+          LANDINGAI_ADE_STAGING_APIKEY: ${{ secrets.LANDINGAI_ADE_STAGING_APIKEY }}
+        # -n 0 forces serial execution: these hit the live API, and the repo's default
+        # `-n auto` (xdist) would run them in parallel, inviting flakiness/rate-limits.
+        run: rye run pytest tests/contract -m contract -n 0 -v

.github/workflows/spec-sync.yml

@@ -0,0 +1,136 @@
+name: Spec Sync
+on:
+  schedule:
+    - cron: '0 * * * *'   # hourly; cron covers correctness, dispatch cuts latency
+  workflow_dispatch: {}
+
+# Serialize runs so a cron tick and a manual dispatch can't race on the sync branch.
+concurrency:
+  group: spec-sync
+  cancel-in-progress: false
+
+permissions:
+  contents: read            # all writes go through SPEC_SYNC_TOKEN, never GITHUB_TOKEN
+
+jobs:
+  spec-sync:
+    # Guard so the scheduled/dispatch job only runs on the canonical repo — forks lack the
+    # secrets and would just produce noisy failing runs.
+    if: github.repository == 'landing-ai/ade-python'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      V1_SPEC_URL: https://api.va.staging.landing.ai/v1/ade/openapi.json  # staging drives the loop
+      SYNC_BRANCH: spec-sync/v1   # fixed branch: reruns update one PR in place, never a pile of them
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # SPEC_SYNC_TOKEN is a fine-grained PAT (Contents: RW, Pull requests: RW) scoped to this
+          # repo. It must NOT be the default GITHUB_TOKEN: pushes/PRs authored by GITHUB_TOKEN do
+          # not trigger the gate workflows (anti-recursion).
+          token: ${{ secrets.SPEC_SYNC_TOKEN }}
+          fetch-depth: 0     # tags needed for surface-lock baseline
+
+      - name: Install Rye
+        run: |
+          curl -sSf https://rye.astral.sh/get | bash
+          echo "$HOME/.rye/shims" >> "$GITHUB_PATH"
+        env:
+          RYE_VERSION: '0.44.0'
+          RYE_INSTALL_OPTION: '--yes'
+
+      - name: Install dependencies
+        run: rye sync --all-features
+
+      - name: Detect drift
+        id: drift
+        run: |
+          set +e
+          ./scripts/spec-sync/check-drift.sh "$V1_SPEC_URL" specs/v1-ade.json
+          echo "code=$?" >> "$GITHUB_OUTPUT"
+
+      - name: No drift
+        if: steps.drift.outputs.code == '0'
+        run: echo "specs in sync; nothing to do."
+
+      - name: Fail on fetch error
+        if: steps.drift.outputs.code != '0' && steps.drift.outputs.code != '10'
+        run: |
+          echo "spec fetch/normalize failed (exit ${{ steps.drift.outputs.code }})"
+          exit 1
+
+      # If a sync PR is already open, do nothing: it's awaiting human review, and re-running
+      # would duplicate gate runs and Claude API spend every hour until it merges.
+      - name: Check for an open sync PR
+        id: existing
+        if: steps.drift.outputs.code == '10'
+        env:
+          GH_TOKEN: ${{ secrets.SPEC_SYNC_TOKEN }}
+        run: |
+          if gh pr list --state open --head "$SYNC_BRANCH" --json number --jq '.[0].number' | grep -q .; then
+            echo "open=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "open=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Sync PR already open — skip
+        if: steps.drift.outputs.code == '10' && steps.existing.outputs.open == 'true'
+        run: echo "A spec-sync PR is already open; skipping until it is merged or closed."
+
+      # ---- Phase 1: mechanical (deterministic) ----
+      - name: Mechanical commit + push
+        if: steps.drift.outputs.code == '10' && steps.existing.outputs.open != 'true'
+        run: |
+          git config user.name "spec-sync[bot]"
+          git config user.email "[email protected]"
+          git checkout -B "$SYNC_BRANCH"
+          ./scripts/spec-sync/gen-models.sh specs/v1-ade.json specs/_generated/v1_models.py
+          git add specs/v1-ade.json specs/_generated/v1_models.py
+          git commit -m "chore(spec-sync): update V1 spec snapshot + regenerated reference models"
+          git push --force origin "$SYNC_BRANCH"
+
+      # Open the PR now — before the AI step — so a run that dies in phase 2 leaves a visible PR
+      # (with just the mechanical commit) instead of an orphan branch.
+      - name: Open sync PR
+        if: steps.drift.outputs.code == '10' && steps.existing.outputs.open != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.SPEC_SYNC_TOKEN }}
+        run: |
+          gh pr create --base main --head "$SYNC_BRANCH" \
+            --title "spec-sync: track V1 spec drift" \
+            --body $'Automated spec-sync PR.\n\n- **Commit 1 (mechanical):** normalized spec snapshot + regenerated reference models.\n- **Commit 2 (AI):** resources/methods/tests/docs wired from the spec diff (added after this PR opened).\n\nGates (surface-lock, contract tests, lint/test/typecheck) must pass. **Human review required before merge.**'
+
+      # ---- Phase 2: AI wiring — edits ONLY. The agent holds no git/push capability, so a
+      # prompt-injected spec description cannot route around review to push to a branch. ----
+      - name: AI wiring (edits only)
+        if: steps.drift.outputs.code == '10' && steps.existing.outputs.open != 'true'
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          github_token: ${{ secrets.SPEC_SYNC_TOKEN }}
+          prompt: |
+            The previous commit updated specs/v1-ade.json and regenerated reference models in
+            specs/_generated/v1_models.py. Edit the SDK to match the spec diff: add or adjust
+            resource classes, method signatures, param TypedDicts, response models, tests, api.md,
+            and README examples, mirroring the conventions in
+            src/landingai_ade/resources/parse_jobs.py. Then run `./scripts/format`.
+            Rules: PURELY ADDITIVE — never modify or remove an existing public signature (the
+            surface-lock CI job will fail the PR). Do NOT run git or push; a later workflow step
+            commits and pushes your edits.
+          claude_args: |
+            --max-turns 40
+            --allowedTools "Edit,Write,Read,Bash(rye *),Bash(./scripts/*)"
+
+      # Deterministic: commit whatever the AI edited and push to the sync branch. This guarantees
+      # the AI changes actually reach the PR (the action's automation mode does not push), and it
+      # is the only step with push capability.
+      - name: Commit + push AI wiring
+        if: steps.drift.outputs.code == '10' && steps.existing.outputs.open != 'true'
+        run: |
+          git add -A
+          if git diff --cached --quiet; then
+            echo "AI step produced no changes."
+          else
+            git commit -m "feat(spec-sync): wire SDK to spec diff (AI)"
+            git push origin HEAD:"$SYNC_BRANCH"
+          fi

CONTRIBUTING.md

@@ -126,3 +126,44 @@ You can release to package managers by using [the `Publish PyPI` GitHub action](
 
 If you need to manually release a package, you can run the `bin/publish-pypi` script with a `PYPI_TOKEN` set on
 the environment.
+
+## Spec-sync pipeline
+
+The SDK tracks the live ADE OpenAPI spec automatically via `.github/workflows/spec-sync.yml`
+(hourly cron + manual `workflow_dispatch`). It is driven by the **staging** spec; releases gate on
+the **production** spec ("staging in, production out").
+
+On each run it fetches and normalizes the live spec (`scripts/spec-sync/fetch-normalize.sh`) and
+diffs it against the committed snapshot `specs/v1-ade.json` (`scripts/spec-sync/check-drift.sh`).
+On drift it opens one PR with two attributed commits:
+
+1. **Mechanical** — updated `specs/v1-ade.json` snapshot plus regenerated *reference* models in
+   `specs/_generated/v1_models.py` (`scripts/spec-sync/gen-models.sh`, `datamodel-code-generator`).
+   These reference models are an input for the AI step and for review — they are **not** shipped and
+   do **not** replace `src/landingai_ade/types/*`.
+2. **AI** — `anthropics/claude-code-action` (automation mode) wires the resources, methods, param
+   types, tests, and docs from the spec diff, following existing conventions.
+
+Every spec-sync PR (and any PR to `main`) must pass `.github/workflows/pr-gates.yml`:
+
+- **surface-lock** (`scripts/spec-sync/surface-lock.sh`, `griffe`) — baseline is the **last release
+  tag**, so any change to *released* public surface fails mechanically. Merged-but-unreleased surface
+  stays mutable.
+- **contract-tests** — `tests/contract` (marker `contract`) run against staging when
+  `LANDINGAI_ADE_STAGING_APIKEY` is set; skipped otherwise.
+
+Spec-sync PRs are AI-drafted and **require human review** before merge.
+
+**Secrets required:** `SPEC_SYNC_TOKEN` (a fine-grained PAT scoped to this repo with
+`Contents: Read and write` and `Pull requests: Read and write`), `ANTHROPIC_API_KEY`, and
+`LANDINGAI_ADE_STAGING_APIKEY`. `SPEC_SYNC_TOKEN` must **not** be the default `GITHUB_TOKEN`:
+pushes and PRs authored by `GITHUB_TOKEN` do not trigger the gate workflows (GitHub
+anti-recursion), so the gates would never run on the sync PR. A GitHub App installation token
+(org-owned) is the cleaner long-term choice and can replace the PAT without other workflow
+changes.
+
+**V2:** once the aide gateway serves its curated spec unauthenticated, add `specs/v2-aide.json` and
+its URL as a second drift-check in the workflow; the two-phase machinery is unchanged.
+
+The same pipeline shape ports to `ade-typescript` with `openapi-typescript` (mechanical) and
+`api-extractor` (surface-lock), tracked separately.

pyproject.toml

@@ -59,6 +59,10 @@ dev-dependencies = [
     "importlib-metadata>=6.7.0",
     "rich>=13.7.1",
     "pytest-xdist>=3.6.1",
+    # Exact pins: codegen output is version-sensitive, so the committed reference models
+    # under specs/_generated/ must be generated by the same version CI installs.
+    "datamodel-code-generator==0.45.0",
+    "griffe==1.14.0",
 ]
 
 [tool.rye.scripts]
@@ -130,10 +134,16 @@ replacement = '[\1](https://github.com/landing-ai/ade-python/tree/main/\g<2>)'
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-addopts = "--tb=short -n auto"
+# Contract tests (marker `contract`) hit the live staging API — opt-in only, so a plain
+# `rye run pytest` (or the pydantic-v1 nox session) never silently calls staging even when
+# LANDINGAI_ADE_STAGING_APIKEY is exported. CI's explicit `-m contract` overrides this.
+addopts = "--tb=short -n auto -m 'not contract'"
 xfail_strict = true
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
+markers = [
+  "contract: hits the live staging API; requires LANDINGAI_ADE_STAGING_APIKEY",
+]
 filterwarnings = [
   "error"
 ]
@@ -150,6 +160,7 @@ exclude = [
     ".venv",
     ".nox",
     ".git",
+    "specs",
 ]
 
 reportImplicitOverride = true
@@ -168,7 +179,7 @@ show_error_codes = true
 #
 # We also exclude our `tests` as mypy doesn't always infer
 # types correctly and Pyright will still catch any type errors.
-exclude = ['src/landingai_ade/_files.py', '_dev/.*.py', 'tests/.*']
+exclude = ['src/landingai_ade/_files.py', '_dev/.*.py', 'tests/.*', 'specs/.*']
 
 strict_equality = true
 implicit_reexport = true
@@ -214,6 +225,9 @@ ignore_missing_imports = true
 line-length = 120
 output-format = "grouped"
 target-version = "py38"
+# specs/_generated holds datamodel-code-generator reference models (not shipped);
+# they are not held to the SDK's lint standards.
+extend-exclude = ["specs"]
 
 [tool.ruff.format]
 docstring-code-format = true