From 9fa5c2b021ddba3e8be8b05664c2e926adf80dbe Mon Sep 17 00:00:00 2001 From: jp-cruz Date: Wed, 17 Jun 2026 22:04:40 -0500 Subject: [PATCH 1/3] chore: validate + harden the rig (self-CI, fixture fix, rule tuning) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validation pass over the multi-language rig. Adds end-to-end self-testing and fixes three real issues that validation surfaced. Self-test CI: - .github/workflows/ci.yml: on push/PR, invokes all 7 reusable workflows (lint/test/sast/audit/supply-chain/secrets/sbom) against the rig itself via LOCAL refs, so a PR validates its own workflow versions. The reusable workflows had never been exercised end-to-end before this. Bug fixes found by validation: - fixtures/http.py: mock_http_client passed the respx MockRouter as an httpx transport, which is not a valid transport (AttributeError: handle_async_request) on httpx 0.28 / respx 0.23 — the fixture was broken for any consumer. Now returns a plain AsyncClient intercepted by the active respx.mock(). Also drops an unused import (F401). - tests/: add the rig's first real test suite for the shipped fixtures (100% coverage of the fixtures module), which doubles as test.yml's self-test target. Rule tuning: - risky-exec: exclude full-line #-comments via pattern-not-regex so documented "curl … | bash" usage notes don't false-positive (the generic_comment_style option does not apply to pattern-regex). Add --exclude semgrep so the ruleset file doesn't self-match its own patterns; add --exclude .claude so worktree copies aren't double-scanned. - pyproject: per-file-ignore S101 (assert) for tests/ and examples/. Validated locally: ruff/mypy/bandit clean, pytest 6/6 (100% cov), risky-exec regression 10/10 on fixtures with 0 comment false positives, rig self-scan clean, all YAML parses, audit.sh shellcheck-clean. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 53 ++++++++++++++++++++++ .github/workflows/supply-chain.yml | 2 +- pyproject.toml | 5 +++ scripts/audit.ps1 | 5 ++- scripts/audit.sh | 6 ++- semgrep/legionforge-risky-exec.yml | 13 ++++++ src/legionforge_dev_rig/fixtures/http.py | 10 +++-- tests/conftest.py | 5 +++ tests/test_fixtures_http.py | 56 ++++++++++++++++++++++++ 9 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/conftest.py create mode 100644 tests/test_fixtures_http.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ec1f509 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +# Self-test CI for the dev-rig. +# +# The rig's other workflows are all reusable (workflow_call) and were never +# exercised end-to-end. This workflow invokes each of them against the rig +# itself — using LOCAL refs (./.github/workflows/*.yml) so a PR validates its +# own versions of the workflows, not the ones already on main. It both proves +# the reusable workflows work and guards them against silent regressions. + +name: CI (self-test) + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + lint: + uses: ./.github/workflows/lint.yml + with: + source-dirs: "src/legionforge_dev_rig" + extra-mypy-deps: "types-PyYAML httpx respx pytest" + + test: + uses: ./.github/workflows/test.yml + with: + coverage-source: "legionforge_dev_rig" + coverage-threshold: 80 + + sast: + uses: ./.github/workflows/sast.yml + with: + source-dirs: "src/legionforge_dev_rig" + semgrep-configs: "p/python" + permissions: + security-events: write + + audit: + uses: ./.github/workflows/audit.yml + + supply-chain: + uses: ./.github/workflows/supply-chain.yml + with: + # Pull the risky-exec ruleset from THIS branch so self-CI tests the PR's + # rules, not whatever is on main. + rig-ref: ${{ github.head_ref || github.ref_name }} + secrets: inherit + + secrets: + uses: ./.github/workflows/secrets.yml + + sbom: + uses: ./.github/workflows/sbom.yml diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index 07722db..59dc378 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -74,7 +74,7 @@ jobs: run: | semgrep --config .dev-rig/semgrep/legionforge-risky-exec.yml \ --error --metrics=off \ - --exclude .dev-rig . + --exclude .dev-rig --exclude .claude --exclude node_modules --exclude semgrep . # ── Optional Socket.dev behavioral npm scan ────────────────────────────────── # Self-skips when no SOCKET_SECURITY_API_KEY secret is provided, so callers diff --git a/pyproject.toml b/pyproject.toml index 4fa14ef..5ce00f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,11 @@ ignore = [ "S607", # partial executable path — acceptable for system binaries ] +[tool.ruff.lint.per-file-ignores] +# Tests and examples legitimately use `assert` (S101) for verification. +"tests/**" = ["S101"] +"examples/**" = ["S101"] + [tool.bandit] exclude_dirs = [".venv", "venv", ".git", "tests"] skips = [ diff --git a/scripts/audit.ps1 b/scripts/audit.ps1 index 1b80923..4de4dae 100644 --- a/scripts/audit.ps1 +++ b/scripts/audit.ps1 @@ -146,7 +146,7 @@ if (Get-Command osv-scanner -ErrorAction SilentlyContinue) { # Runs only when the repo contains shell scripts. $shellFiles = Get-ChildItem -Path $ProjectPath -Recurse -File -Include *.sh, *.bash -ErrorAction SilentlyContinue | - Where-Object { $_.FullName -notmatch '[\\/](\.git|\.venv|node_modules)[\\/]' } + Where-Object { $_.FullName -notmatch '[\\/](\.git|\.venv|node_modules|\.claude)[\\/]' } if ($shellFiles) { if (Get-Command shellcheck -ErrorAction SilentlyContinue) { Invoke-Tool "shellcheck" { @@ -193,7 +193,8 @@ if (Test-Path $riskyRules) { -v "${ProjectPath}:/src" ` -v "${rigSemgrep}:/rules:ro" ` semgrep/semgrep ` - semgrep --config /rules/legionforge-risky-exec.yml /src --error + semgrep --config /rules/legionforge-risky-exec.yml /src --error ` + --exclude .claude --exclude .git --exclude node_modules --exclude semgrep } } diff --git a/scripts/audit.sh b/scripts/audit.sh index b062f63..b53eff1 100644 --- a/scripts/audit.sh +++ b/scripts/audit.sh @@ -113,7 +113,8 @@ fi shell_files=() while IFS= read -r f; do shell_files+=("$f"); done < <( find "$PROJECT" -type f \( -name '*.sh' -o -name '*.bash' \) \ - -not -path '*/.git/*' -not -path '*/.venv/*' -not -path '*/node_modules/*' 2>/dev/null + -not -path '*/.git/*' -not -path '*/.venv/*' -not -path '*/node_modules/*' \ + -not -path '*/.claude/*' 2>/dev/null ) if [[ ${#shell_files[@]} -gt 0 ]]; then if command -v shellcheck > /dev/null 2>&1; then @@ -156,7 +157,8 @@ if [[ -f "$RISKY_RULES" ]]; then -v "${WIN_PROJECT}:/src" \ -v "${WIN_RIG_SEMGREP}:/rules:ro" \ semgrep/semgrep \ - semgrep --config /rules/legionforge-risky-exec.yml /src --error + semgrep --config /rules/legionforge-risky-exec.yml /src --error \ + --exclude .claude --exclude .git --exclude node_modules --exclude semgrep fi # ── Summary ─────────────────────────────────────────────────────────────────── diff --git a/semgrep/legionforge-risky-exec.yml b/semgrep/legionforge-risky-exec.yml index 42b5abd..6b59686 100644 --- a/semgrep/legionforge-risky-exec.yml +++ b/semgrep/legionforge-risky-exec.yml @@ -7,6 +7,12 @@ # Scope: shell (.sh/.bash), PowerShell (.ps1/.psm1), and CI YAML, matched in # Semgrep "generic" mode so a single ruleset spans every file type. # +# Comment handling: rules use pattern-regex (raw text), which bypasses generic +# mode's comment tokenizer — so each rule ANDs a `pattern-not-regex` excluding +# full-line #-comments. This keeps documented usage examples (e.g. a +# "# curl ... | bash" install note) from tripping the rules; only executable +# lines are flagged. (# is the line-comment char in shell, PowerShell, and YAML.) +# # Run: # semgrep --config semgrep/legionforge-risky-exec.yml --error # Wired into scripts/audit.sh and .github/workflows/supply-chain.yml. @@ -25,6 +31,7 @@ rules: include: ["*.sh", "*.bash", "*.yml", "*.yaml", "*.zsh"] patterns: - pattern-regex: '(curl|wget)\b[^\n|]*\|[^\n]*\b(sh|bash|zsh|dash)\b' + - pattern-not-regex: '(?m)^\s*#.*' # ── PowerShell download-cradle ─────────────────────────────────────────────── - id: lf-powershell-download-cradle @@ -42,6 +49,7 @@ rules: - pattern-regex: '(?i)(New-Object\s+Net\.WebClient)[^\n]*\.DownloadString' - pattern-regex: '(?i)DownloadString\([^\n]*\)[^\n]*\|\s*(iex|Invoke-Expression)' - pattern-regex: '(?i)(iwr|Invoke-WebRequest|curl)\b[^\n|]*\|\s*(iex|Invoke-Expression)' + - pattern-not-regex: '(?m)^\s*#.*' # ── Decode-and-execute (obfuscation) ───────────────────────────────────────── - id: lf-decode-and-exec @@ -58,6 +66,7 @@ rules: - pattern-regex: '\bbase64\s+(-d|--decode)\b[^\n]*\|[^\n]*\b(sh|bash|python|perl|node)\b' - pattern-regex: '(?i)\[Convert\]::FromBase64String\([^\n]*\)[^\n]*(iex|Invoke-Expression)' - pattern-regex: '(?i)-(enc|encodedcommand)\b' + - pattern-not-regex: '(?m)^\s*#.*' # ── eval over fetched / command-substituted content ────────────────────────── - id: lf-eval-remote @@ -71,6 +80,7 @@ rules: include: ["*.sh", "*.bash", "*.zsh", "*.yml", "*.yaml"] patterns: - pattern-regex: '\beval\b[^\n]*\$\((curl|wget)\b' + - pattern-not-regex: '(?m)^\s*#.*' # ── TLS verification disabled on download ──────────────────────────────────── - id: lf-tls-verification-disabled @@ -88,6 +98,7 @@ rules: - pattern-regex: '\bcurl\b[^\n]*\s(-k|--insecure)\b' - pattern-regex: '\bwget\b[^\n]*--no-check-certificate\b' - pattern-regex: '(?i)ServerCertificateValidationCallback\s*=\s*\{?\s*\$?true' + - pattern-not-regex: '(?m)^\s*#.*' # ── Install straight from a VCS URL / git ref ──────────────────────────────── - id: lf-install-from-vcs-url @@ -105,6 +116,7 @@ rules: - pattern-regex: '\bpip(3)?\s+install\b[^\n]*\bgit\+' - pattern-regex: '\bnpm\s+(install|i)\b[^\n]*\b(git\+|https?://|github:)' - pattern-regex: '\bcargo\s+install\b[^\n]*--git\b' + - pattern-not-regex: '(?m)^\s*#.*' # ── npm install scripts re-enabled inline ──────────────────────────────────── - id: lf-npm-scripts-reenabled @@ -118,3 +130,4 @@ rules: include: ["*.sh", "*.bash", "*.yml", "*.yaml"] patterns: - pattern-regex: '\bnpm\b[^\n]*(--foreground-scripts|--ignore-scripts[=\s]+false)' + - pattern-not-regex: '(?m)^\s*#.*' diff --git a/src/legionforge_dev_rig/fixtures/http.py b/src/legionforge_dev_rig/fixtures/http.py index 1e2c988..a5cef48 100644 --- a/src/legionforge_dev_rig/fixtures/http.py +++ b/src/legionforge_dev_rig/fixtures/http.py @@ -1,5 +1,5 @@ """Shared httpx / respx fixtures for async HTTP provider testing.""" -from collections.abc import AsyncGenerator, Generator +from collections.abc import Generator from typing import Any import httpx @@ -26,9 +26,13 @@ def test_something(respx_mock_base_url): @pytest.fixture def mock_http_client(respx_mock_base_url: respx.MockRouter) -> httpx.AsyncClient: """ - An httpx.AsyncClient wired into the respx mock router. + An httpx.AsyncClient whose requests are intercepted by the respx mock. Pass this directly to provider constructors that accept a client parameter. + The respx_mock_base_url fixture activates respx.mock(), which patches httpx's + transport globally — so a plain AsyncClient is routed through the mock. (Do + NOT pass the router as `transport=`; a MockRouter is not an httpx transport.) + Usage: async def test_health(mock_http_client): respx_mock_base_url.get("http://127.0.0.1:11434/api/tags").mock( @@ -37,7 +41,7 @@ async def test_health(mock_http_client): provider = OllamaProvider(client=mock_http_client) assert await provider.health_check() is True """ - return httpx.AsyncClient(transport=respx_mock_base_url) + return httpx.AsyncClient() def json_response(data: dict[str, Any], status: int = 200) -> httpx.Response: diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b227387 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +# Re-export the shared fixtures so pytest discovers them in the rig's own +# self-test suite (mirrors the consuming-project pattern in examples/conftest.py). +from legionforge_dev_rig.fixtures import mock_http_client, respx_mock_base_url + +__all__ = ["mock_http_client", "respx_mock_base_url"] diff --git a/tests/test_fixtures_http.py b/tests/test_fixtures_http.py new file mode 100644 index 0000000..94239a8 --- /dev/null +++ b/tests/test_fixtures_http.py @@ -0,0 +1,56 @@ +"""Tests for the shared httpx/respx fixtures shipped by the dev-rig. + +These also serve as the rig's own self-test target so test.yml is validated +end-to-end against the rig itself (see .github/workflows/ci.yml). +""" +import httpx +import pytest + +from legionforge_dev_rig.fixtures import http as fx + + +def test_json_response_defaults() -> None: + resp = fx.json_response({"ok": True}) + assert resp.status_code == 200 + assert resp.json() == {"ok": True} + + +def test_json_response_custom_status() -> None: + resp = fx.json_response({"created": 1}, status=201) + assert resp.status_code == 201 + assert resp.json() == {"created": 1} + + +def test_error_response_with_detail() -> None: + resp = fx.error_response(404, "missing") + assert resp.status_code == 404 + assert resp.json() == {"detail": "missing"} + + +def test_error_response_without_detail() -> None: + resp = fx.error_response(500) + assert resp.status_code == 500 + assert resp.json() == {} + + +def test_respx_fixture_stubs_route(respx_mock_base_url) -> None: + respx_mock_base_url.get("http://svc.local/ping").mock( + return_value=httpx.Response(200, json={"pong": True}) + ) + # respx.mock() is active via the fixture; a plain client is intercepted. + with httpx.Client() as client: + r = client.get("http://svc.local/ping") + assert r.json() == {"pong": True} + + +@pytest.mark.asyncio +async def test_mock_http_client_is_async_and_wired( + mock_http_client, respx_mock_base_url +) -> None: + respx_mock_base_url.get("http://svc.local/health").mock( + return_value=httpx.Response(200, json={"status": "ok"}) + ) + async with mock_http_client as client: + r = await client.get("http://svc.local/health") + assert r.status_code == 200 + assert r.json() == {"status": "ok"} From df8674e2e14737403f432a7321d3c3bfb1ff7d64 Mon Sep 17 00:00:00 2001 From: jp-cruz Date: Wed, 17 Jun 2026 22:05:28 -0500 Subject: [PATCH 2/3] ci: run self-test on all PRs, not just those targeting main Stacked PRs (and this one) target a feature branch, so the branches:[main] filter would skip self-CI entirely. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec1f509..83251cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,7 @@ name: CI (self-test) on: push: branches: [main] - pull_request: - branches: [main] + pull_request: # all PRs, regardless of base — so stacked branches self-test too jobs: lint: From 6b4fec464b23fa869aede788fe43b29fde9a591b Mon Sep 17 00:00:00 2001 From: jp-cruz Date: Wed, 17 Jun 2026 22:07:41 -0500 Subject: [PATCH 3/3] fix(ci): add [dev] extra so lint.yml/test.yml install lint tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lint.yml runs 'pip install -e .[dev]' but the rig defined its tools only under the [analysis] extra — so ruff/bandit/mypy were never installed and self-CI lint failed with 'No module named ruff'. The rig now provides the [dev] extra its own reusable workflows expect. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5ce00f3..003cd56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,18 @@ analysis = [ "pre-commit>=3.7", "types-PyYAML", ] +# The `dev` extra is the contract the reusable lint.yml / test.yml workflows +# install (`pip install -e .[dev]`). Consuming projects provide their own; the +# rig defines one so it can self-test those workflows (see ci.yml). +dev = [ + "ruff>=0.4", + "bandit[toml]>=1.7", + "mypy>=1.10", + "pytest>=8", + "pytest-asyncio>=0.23", + "pytest-cov>=5", + "types-PyYAML", +] [tool.pytest.ini_options] asyncio_mode = "auto"