diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml new file mode 100644 index 0000000..6dc9c2f --- /dev/null +++ b/.github/workflows/eval.yml @@ -0,0 +1,149 @@ +name: Quality eval (advisory) + +# Runs the RAG quality eval harness end-to-end against the local stack +# (Postgres + Redis + MinIO via compose) plus hosted Pinecone and an LLM key. +# +# Advisory: this job never blocks a merge. It is manual + scheduled so that +# regressions in retrieval/answer quality surface as a visible trend without +# gating PRs on external services and LLM spend. The fast, always-on guardrail +# is the eval unit-test suite in pr.yml (`tests/test_eval_metrics.py`, +# `tests/test_eval_judge.py`). +# +# Requires repo secrets PINECONE_API_KEY and OPENAI_API_KEY. When they are +# absent (e.g. on a fork) the job no-ops instead of failing. + +on: + workflow_dispatch: + inputs: + dataset: + description: "Dataset under evals/datasets/ to run" + required: false + default: "smoke" + schedule: + # Weekly, Monday 06:00 UTC — a low-traffic drift check. + - cron: "0 6 * * 1" + +permissions: + contents: read + +concurrency: + group: eval-${{ github.ref }} + cancel-in-progress: false + +jobs: + eval: + name: RAG quality eval + runs-on: ubuntu-24.04 + timeout-minutes: 25 + # Advisory: surface results, never fail the workflow run. + continue-on-error: true + env: + DATASET: ${{ github.event.inputs.dataset || 'smoke' }} + # Local infra creds (non-secret; only ever bound to throwaway containers). + DB_NAME: agenticrag + DB_ADMIN_USER: admin + DB_ADMIN_PASSWORD: evalpw + DB_USER: reader + DB_PASSWORD: evalpw + DB_HOST: localhost + DB_PORT: "5432" + REDIS_HOST: localhost + REDIS_PORT: "6379" + MINIO_ENDPOINT: localhost:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_SECURE: "false" + OTEL_ENABLED: "false" + DOCUMENT_INGEST_MODE: inline + steps: + - name: Check for required secrets + id: guard + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} + run: | + if [ -z "$OPENAI_API_KEY" ] || [ -z "$PINECONE_API_KEY" ]; then + echo "run=false" >> "$GITHUB_OUTPUT" + echo "### Quality eval skipped" >> "$GITHUB_STEP_SUMMARY" + echo "Missing PINECONE_API_KEY and/or OPENAI_API_KEY repo secrets." >> "$GITHUB_STEP_SUMMARY" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.guard.outputs.run == 'true' + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + if: steps.guard.outputs.run == 'true' + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Write .env for the stack + if: steps.guard.outputs.run == 'true' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} + run: | + { + echo "OPENAI_API_KEY=$OPENAI_API_KEY" + echo "PINECONE_API_KEY=$PINECONE_API_KEY" + echo "DB_NAME=$DB_NAME" + echo "DB_ADMIN_USER=$DB_ADMIN_USER" + echo "DB_ADMIN_PASSWORD=$DB_ADMIN_PASSWORD" + echo "DB_USER=$DB_USER" + echo "DB_PASSWORD=$DB_PASSWORD" + echo "DB_HOST=$DB_HOST" + echo "DB_PORT=$DB_PORT" + echo "REDIS_HOST=$REDIS_HOST" + echo "REDIS_PORT=$REDIS_PORT" + echo "MINIO_ENDPOINT=$MINIO_ENDPOINT" + echo "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" + echo "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" + echo "MINIO_SECURE=$MINIO_SECURE" + } > .env + + - name: Start infra (Postgres, Redis, MinIO) + if: steps.guard.outputs.run == 'true' + run: | + docker compose up -d postgres redis minio minio-setup + # Wait for Postgres + MinIO to accept connections. + for i in $(seq 1 30); do + if docker compose exec -T postgres pg_isready -U "$DB_ADMIN_USER" -d "$DB_NAME" >/dev/null 2>&1; then + break + fi + sleep 2 + done + + - name: Install deps + migrate + if: steps.guard.outputs.run == 'true' + run: | + uv sync --frozen + uv run alembic upgrade head + + - name: Run quality eval + if: steps.guard.outputs.run == 'true' + run: | + uv run python evals/run_eval.py --dataset "$DATASET" | tee eval.out + + - name: Append summary + if: steps.guard.outputs.run == 'true' + run: | + echo "### Quality eval — \`$DATASET\`" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + sed -n '/Eval run/,$p' eval.out >> "$GITHUB_STEP_SUMMARY" || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + + - name: Upload reports + if: steps.guard.outputs.run == 'true' + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: eval-reports + path: evals/reports/ + if-no-files-found: warn + + - name: Tear down infra + if: always() && steps.guard.outputs.run == 'true' + run: docker compose down -v diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 244af24..a3f98f7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -45,6 +45,8 @@ jobs: - 'tasks/**' - 'observability/**' - 'alembic/**' + - 'evals/**' + - 'tests/**' - 'pyproject.toml' - 'uv.lock' - 'Dockerfile' @@ -62,7 +64,6 @@ jobs: test-api: name: API tests needs: changes - if: needs.changes.outputs.api == 'true' runs-on: ubuntu-24.04-arm timeout-minutes: 10 steps: @@ -132,7 +133,6 @@ jobs: helm-lint: name: Helm chart lint needs: changes - if: needs.changes.outputs.helm == 'true' runs-on: ubuntu-24.04-arm timeout-minutes: 5 steps: diff --git a/.gitignore b/.gitignore index 945692f..4a160a9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,7 @@ test-screenshots design_audit_screenshots/ # Eval reports (generated) -evals/reports/helm/values-secrets.yaml +evals/reports/ + +# Helm secrets helm/values-secrets.yaml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f335ae6 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: eval eval-retrieval eval-unit + +# Full quality eval: retrieval metrics + answer generation + LLM judge. +# Requires the local stack (Postgres, Redis, MinIO) plus Pinecone + an LLM key. +eval: + uv run python evals/run_eval.py --dataset $(or $(DATASET),smoke) + +# Fast retrieval-only eval: no answer generation, no judge (no LLM judge cost). +eval-retrieval: + uv run python evals/run_eval.py --dataset $(or $(DATASET),smoke) --retrieval-only + +# Offline unit tests for the eval metrics + judge parsing (no stack, no network). +eval-unit: + uv run pytest tests/test_eval_metrics.py tests/test_eval_judge.py -q diff --git a/evals/README.md b/evals/README.md index 1d3ef37..d4af67e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,12 +1,26 @@ # RAG Evaluation Harness -Evaluate retrieval quality and answer generation for the RunaxAI pipeline. +Make answer quality measurable so we can ship RAG/agent changes with +confidence. This harness scores the core document-chat + RAG flow — both +**retrieval relevance** and **answer quality** — against a fixed +question/document set. ## Quick start +One command runs the full eval against the local stack: + +```bash +make eval # full eval on the `smoke` dataset +make eval DATASET=mydataset # full eval on another dataset +make eval-retrieval # retrieval metrics only (no answer gen, no judge cost) +make eval-unit # offline unit tests — no stack, no network +``` + +Or call the runner directly: + ```bash # Retrieval metrics only (no LLM judge, fast) -uv run python evals/run_eval.py --dataset smoke --skip-judge +uv run python evals/run_eval.py --dataset smoke --retrieval-only # Full evaluation with LLM-as-judge scoring uv run python evals/run_eval.py --dataset smoke @@ -17,19 +31,20 @@ uv run python evals/run_eval.py --dataset smoke --judge-model gpt-4o ## What it does -1. Creates an ephemeral project in the database +1. Creates an ephemeral user + project in the database 2. Uploads and ingests documents from the dataset into Pinecone 3. Runs each query through the retrieval pipeline 4. Computes retrieval metrics: Recall@k, MRR, NDCG@k, substring recall 5. (Optional) Generates answers via `project_chat_stream` and scores them with an LLM judge 6. Writes JSON + markdown reports to `evals/reports/` -7. Cleans up the ephemeral project (Pinecone namespace, DB rows, MinIO objects) +7. Cleans up everything it created (Pinecone namespace, DB rows, MinIO objects) ## Prerequisites -- Running PostgreSQL, Redis, Pinecone, and MinIO (the standard docker-compose stack) -- Environment variables configured (`.env`) -- For judge evaluation: an LLM API key +- The local stack running: `docker compose up -d postgres redis minio minio-setup` +- DB schema applied: `uv run alembic upgrade head` +- Environment variables configured (`.env`), including `PINECONE_API_KEY` +- For judge evaluation: an LLM API key (`OPENAI_API_KEY`) ## Metrics @@ -37,22 +52,71 @@ uv run python evals/run_eval.py --dataset smoke --judge-model gpt-4o - **Recall@k** — fraction of expected documents found in top-k results - **MRR** — reciprocal rank of the first relevant result - **NDCG@k** — normalized discounted cumulative gain -- **Substring Recall** — fraction of expected substrings found in retrieved chunks +- **Substring Recall** — fraction of expected fact substrings found in retrieved chunks -### Answer Quality (LLM Judge) -- **Faithfulness** — claims grounded in retrieved context (1-5) -- **Completeness** — covers all expected mentions (1-5) -- **Hallucination** — free of unsupported or incorrect claims (1-5) -- **Format Adherence** — matches expected output format (1-5) +> Filename-based metrics (recall@k, MRR, NDCG) compare against +> `expected_doc_filenames`. The harness maps each retrieved chunk's +> `document_id` back to its original dataset filename, so these are meaningful +> even though the vector store records `source` as an internal storage name. -## Adding datasets +### Answer Quality (LLM Judge, 1–5) +- **Faithfulness** — claims grounded in retrieved context +- **Completeness** — covers all expected `must_mention` items +- **Hallucination** — free of `must_not_mention` items and unsupported claims +- **Format Adherence** — matches the expected output format -Create a new directory under `evals/datasets//` with: +> ⚠️ The `hallucination` dimension is negatively framed and the judge +> occasionally inverts its scale (scoring 1 while its reason says "no +> hallucinations present"). Read per-query `reason` fields in the JSON report +> rather than trusting the `hallucination` mean alone. See +> [`baselines/smoke.md`](baselines/smoke.md). + +## Adding cases + +**New query against an existing dataset** — append one JSON object per line to +`evals/datasets//queries.jsonl`: + +```json +{"id": "q011", "query": "...", "expected_doc_filenames": ["xr7_datasheet.md"], "expected_chunk_substrings": ["..."], "expected_answer_traits": {"must_mention": ["..."], "must_not_mention": [], "format": "prose"}} +``` + +- `expected_doc_filenames` — drives recall@k / MRR / NDCG +- `expected_chunk_substrings` — drives substring_recall (case-insensitive) +- `expected_answer_traits.must_mention` / `must_not_mention` — feed the judge +- `expected_answer_traits.format` — e.g. `prose`, `list`, `json` + +**New dataset** — create `evals/datasets//` with: - `documents/` — files to ingest (md, txt, pdf, csv, docx) -- `queries.jsonl` — one JSON object per line (see `smoke/queries.jsonl` for schema) +- `queries.jsonl` — as above + +Then run `make eval DATASET=`. + +## Reading results + +Each run writes to `evals/reports/` (gitignored): +- `.json` — structured data for programmatic analysis (all per-query + metrics, answers, and judge reasons) +- `.md` — human-readable summary with an aggregate table and per-query + drill-down + +The terminal prints the aggregate means/min/max at the end of the run. + +## Baselines + +Committed reference scores live in [`evals/baselines/`](baselines/). Compare a +run's aggregates against the matching baseline before/after changing retrieval, +chunking, prompts, or models. Current baseline: [`baselines/smoke.md`](baselines/smoke.md). -## Reports +## CI -Reports are written to `evals/reports/` (gitignored). Each run produces: -- `.json` — structured data for programmatic analysis -- `.md` — human-readable summary with per-query drill-down +- **Always-on (PR gate):** the offline unit tests + (`tests/test_eval_metrics.py`, `tests/test_eval_judge.py`) run as part of + `uv run pytest` in `.github/workflows/pr.yml`. They need no stack or network + and guard the metric + judge-parsing logic. +- **Advisory (manual + weekly):** `.github/workflows/eval.yml` runs the full + end-to-end eval against an ephemeral stack + hosted Pinecone + an LLM key, + uploads the reports as a build artifact, and writes the aggregate table to + the job summary. It is `continue-on-error` (never blocks a merge) and + requires the repo secrets `PINECONE_API_KEY` and `OPENAI_API_KEY`; without + them it no-ops. Trigger it from the Actions tab ("Run workflow") or wait for + the weekly schedule. diff --git a/evals/baselines/smoke.md b/evals/baselines/smoke.md new file mode 100644 index 0000000..c371790 --- /dev/null +++ b/evals/baselines/smoke.md @@ -0,0 +1,59 @@ +# Baseline — `smoke` dataset + +Reference scores for the core document-chat + RAG flow on the `smoke` dataset +(4 documents, 10 queries). Re-run `make eval` and compare against this table +before/after changing retrieval, chunking, prompts, or models. + +## Provenance + +| Field | Value | +|-------|-------| +| Date | 2026-06-19 | +| Dataset | `smoke` (4 docs, 10 queries) | +| Generation model | `gpt-5.4` (whatever `project_chat_stream` routes to) | +| Judge model | `gpt-4o-mini` | +| Retrieval | hybrid dense+sparse, rerank on, `top_k=20`, HyDE off | +| Command | `uv run python evals/run_eval.py --dataset smoke` | + +## Scores + +### Retrieval (0–1, higher is better) + +| Metric | Mean | Min | Max | +|--------|------|-----|-----| +| recall@5 | 1.000 | 1.000 | 1.000 | +| recall@10 | 1.000 | 1.000 | 1.000 | +| recall@20 | 1.000 | 1.000 | 1.000 | +| ndcg@5 | 0.605 | 0.431 | 1.000 | +| mrr | 0.475 | 0.250 | 1.000 | +| substring_recall | 1.000 | 1.000 | 1.000 | + +The expected document is always retrieved within the top 5 (recall@5 = 1.0) and +every expected fact substring appears in the retrieved chunks +(substring_recall = 1.0). NDCG/MRR are lower because the *correct* document is +often not ranked first — there is headroom in result ordering, not coverage. + +### Answer quality — LLM judge (1–5, higher is better) + +| Dimension | Mean | Min | Max | +|-----------|------|-----|-----| +| faithfulness | 5.00 | 5 | 5 | +| completeness | 5.00 | 5 | 5 | +| hallucination | 4.60 | 1 | 5 | +| format_adherence | 4.30 | 4 | 5 | + +Answers are fully grounded and complete across the set. `format_adherence` +dips because some answers add markdown (bold/headers) where plain prose was +expected. + +## Known caveats + +- **`hallucination` dimension polarity is fragile.** On one query (q009) the + judge returned `score: 1` while its own reason said "there are no + hallucinations present" — i.e. it inverted the negatively-framed scale. This + drags the mean down to 4.60 despite no real hallucination. Treat the + `hallucination` mean as noisy until the rubric is reworded (candidate + follow-up: rename to `groundedness` with an explicit "5 = fully grounded" + anchor). Per-query reasons in the JSON report are the source of truth. +- Judge scores are model-dependent. Pin `--judge-model` when comparing runs. +- Numbers are from a 10-query smoke set; treat as directional, not precise. diff --git a/evals/run_eval.py b/evals/run_eval.py index 497c2d8..f8698dd 100644 --- a/evals/run_eval.py +++ b/evals/run_eval.py @@ -52,46 +52,68 @@ def get_document_paths(dataset: str) -> list[Path]: return sorted(docs_dir.glob("*")) -def _create_eval_project(run_id: str) -> tuple[str, any]: - """Create an ephemeral project for this eval run via SQLAlchemy. +def _create_eval_project(run_id: str) -> tuple[str, str, any]: + """Create an ephemeral user + project for this eval run via SQLAlchemy. - Returns (project_id, engine) for cleanup. + The ``project`` table has a non-nullable FK to ``user.id`` (a UUID), so we + spin up a throwaway user for the run and tear it down in cleanup. Table + names are singular (``user``, ``project``) to match the live schema. + + Returns (project_id, user_id, engine) for cleanup. """ from sqlalchemy import create_engine, text from database.core import DATABASE_URL engine = create_engine(DATABASE_URL.replace("+asyncpg", "+psycopg2")) project_id = f"eval-{run_id}" + user_id = str(uuid.uuid4()) with engine.begin() as conn: + # ``user`` is a reserved word in Postgres, so it must be quoted. + conn.execute( + text( + 'INSERT INTO "user" ' + "(id, email, is_active, is_superuser, is_verified, created_at, updated_at) " + "VALUES (:id, :email, true, false, true, now(), now())" + ), + {"id": user_id, "email": f"eval-{run_id}@eval.local"}, + ) conn.execute( text( - "INSERT INTO projects (id, name, description, status, user_id) " - "VALUES (:id, :name, :desc, :status, :user_id)" + "INSERT INTO project " + "(id, user_id, name, description, status, created_at, updated_at) " + "VALUES (:id, :user_id, :name, :desc, :status, now(), now())" ), { "id": project_id, + "user_id": user_id, "name": f"eval-{run_id}", "desc": "Ephemeral eval project", "status": "active", - "user_id": "eval-system", }, ) - return project_id, engine + return project_id, user_id, engine def _ingest_documents( project_id: str, doc_paths: list[Path], config: dict, -) -> int: - """Ingest eval documents and return total chunk count.""" +) -> tuple[int, dict[str, str]]: + """Ingest eval documents. + + Returns (total_chunk_count, doc_id_to_filename). The map lets the retrieval + eval translate a retrieved chunk's ``document_id`` back to its original + dataset filename, which is what the expected-filename metrics compare + against (the vector store records ``source`` as the storage object name). + """ from pipeline.ingestion import ingest_document from pipeline.storage import ensure_bucket, BUCKET_NAME from clients import minio_client ensure_bucket() total_chunks = 0 + doc_id_to_filename: dict[str, str] = {} for doc_path in doc_paths: doc_id = str(uuid.uuid4()) @@ -108,13 +130,14 @@ def _ingest_documents( chunk_size=config.get("ingestion", {}).get("chunk_size", 2000), chunk_overlap=config.get("ingestion", {}).get("chunk_overlap", 300), ) + doc_id_to_filename[doc_id] = doc_path.name total_chunks += result["chunk_count"] logger.info( f"ingested {doc_path.name}: {result['chunk_count']} chunks " f"({result['chunk_strategy']})" ) - return total_chunks + return total_chunks, doc_id_to_filename def _wait_for_pinecone_consistency(project_id: str, expected_chunks: int, config: dict): @@ -148,6 +171,7 @@ def _run_retrieval_eval( queries: list[dict], chunk_count: int, config: dict, + doc_id_to_filename: dict[str, str], ) -> list[dict]: """Run retrieval for each query and compute metrics.""" from pipeline.retriever import retrieve @@ -168,7 +192,13 @@ def _run_retrieval_eval( logger.error(f"retrieval failed for {q['id']}: {e}") retrieved = [] - filenames = [r.get("source", "") for r in retrieved] + # Translate each chunk's document_id to its original dataset filename + # so expected-filename metrics (recall@k, mrr, ndcg) are meaningful. + # Fall back to the raw `source` when a chunk predates the ingest map. + filenames = [ + doc_id_to_filename.get(r.get("document_id", ""), r.get("source", "")) + for r in retrieved + ] texts = [r.get("text", "") for r in retrieved] expected_files = set(q.get("expected_doc_filenames", [])) expected_subs = q.get("expected_chunk_substrings", []) @@ -349,7 +379,7 @@ def _generate_markdown_report( return "\n".join(lines) -def _cleanup(project_id: str, engine, doc_paths: list[Path]): +def _cleanup(project_id: str, user_id: str | None, engine, doc_paths: list[Path]): """Delete Pinecone namespace, DB rows, and MinIO objects.""" errors = [] @@ -361,18 +391,24 @@ def _cleanup(project_id: str, engine, doc_paths: list[Path]): except Exception as e: errors.append(f"Pinecone cleanup: {e}") - # 2. Delete DB rows + # 2. Delete DB rows (singular table names; FK cascade also covers these, + # but we delete explicitly so a partial failure is easy to read). try: from sqlalchemy import text with engine.begin() as conn: conn.execute( - text("DELETE FROM documents WHERE project_id = :pid"), + text("DELETE FROM document WHERE project_id = :pid"), {"pid": project_id}, ) conn.execute( - text("DELETE FROM projects WHERE id = :pid"), + text("DELETE FROM project WHERE id = :pid"), {"pid": project_id}, ) + if user_id: + conn.execute( + text('DELETE FROM "user" WHERE id = :uid'), + {"uid": user_id}, + ) logger.info(f"deleted DB rows for {project_id}") except Exception as e: errors.append(f"DB cleanup: {e}") @@ -414,22 +450,25 @@ def main(): logger.info(f"loaded {len(queries)} queries, {len(doc_paths)} documents from '{args.dataset}'") project_id = None + user_id = None engine = None try: - # 1. Create ephemeral project - project_id, engine = _create_eval_project(run_id) + # 1. Create ephemeral user + project + project_id, user_id, engine = _create_eval_project(run_id) logger.info(f"created eval project: {project_id}") # 2. Ingest documents - total_chunks = _ingest_documents(project_id, doc_paths, config) + total_chunks, doc_id_to_filename = _ingest_documents(project_id, doc_paths, config) logger.info(f"ingested {total_chunks} total chunks") # 3. Wait for consistency _wait_for_pinecone_consistency(project_id, total_chunks, config) # 4. Run retrieval eval - retrieval_results = _run_retrieval_eval(project_id, queries, total_chunks, config) + retrieval_results = _run_retrieval_eval( + project_id, queries, total_chunks, config, doc_id_to_filename + ) # Attach project context for judge phase for ret in retrieval_results: @@ -493,7 +532,7 @@ def main(): finally: if project_id and engine: - _cleanup(project_id, engine, doc_paths) + _cleanup(project_id, user_id, engine, doc_paths) if __name__ == "__main__": diff --git a/tests/test_eval_judge.py b/tests/test_eval_judge.py new file mode 100644 index 0000000..8af62fc --- /dev/null +++ b/tests/test_eval_judge.py @@ -0,0 +1,154 @@ +"""Unit tests for evals/judge.py — judge response parsing and fallback. + +These are pure/offline: the LLM client is faked so no network or API key is +required. They guard the parsing logic that turns a judge's raw text into +structured scores, which is the part most likely to silently break. +""" + +from evals.judge import _parse_judge_response, judge_answer, EXPECTED_DIMENSIONS + + +def _valid_payload() -> dict: + return { + "faithfulness": {"score": 5, "reason": "grounded"}, + "completeness": {"score": 4, "reason": "minor gap"}, + "hallucination": {"score": 5, "reason": "no unsupported claims"}, + "format_adherence": {"score": 5, "reason": "prose as expected"}, + } + + +# --- _parse_judge_response --- + +def test_parse_raw_json(): + import json + parsed = _parse_judge_response(json.dumps(_valid_payload())) + assert parsed is not None + assert EXPECTED_DIMENSIONS <= set(parsed) + assert parsed["faithfulness"]["score"] == 5 + + +def test_parse_json_in_code_fence(): + import json + text = "Here is my evaluation:\n```json\n" + json.dumps(_valid_payload()) + "\n```\n" + parsed = _parse_judge_response(text) + assert parsed is not None + assert parsed["completeness"]["score"] == 4 + + +def test_parse_json_in_bare_fence(): + import json + text = "```\n" + json.dumps(_valid_payload()) + "\n```" + parsed = _parse_judge_response(text) + assert parsed is not None + assert EXPECTED_DIMENSIONS <= set(parsed) + + +def test_parse_json_with_surrounding_prose(): + import json + text = "Sure! " + json.dumps(_valid_payload()) + " Hope that helps." + parsed = _parse_judge_response(text) + assert parsed is not None + assert parsed["hallucination"]["score"] == 5 + + +def test_parse_missing_dimension_returns_none(): + import json + payload = _valid_payload() + del payload["format_adherence"] + assert _parse_judge_response(json.dumps(payload)) is None + + +def test_parse_garbage_returns_none(): + assert _parse_judge_response("not json at all") is None + + +def test_parse_empty_returns_none(): + assert _parse_judge_response("") is None + + +# --- judge_answer with a fake client --- + +class _FakeMessage: + def __init__(self, content): + self.content = content + + +class _FakeChoice: + def __init__(self, content): + self.message = _FakeMessage(content) + + +class _FakeResponse: + def __init__(self, content): + self.choices = [_FakeChoice(content)] + + +class _FakeCompletions: + def __init__(self, content=None, raise_exc=False): + self._content = content + self._raise = raise_exc + self.calls = 0 + + def create(self, **kwargs): + self.calls += 1 + if self._raise: + raise RuntimeError("simulated API failure") + return _FakeResponse(self._content) + + +class _FakeChat: + def __init__(self, completions): + self.completions = completions + + +class _FakeLLMClient: + def __init__(self, content=None, raise_exc=False): + self.chat = _FakeChat(_FakeCompletions(content, raise_exc)) + + +def test_judge_answer_parses_valid_response(): + import json + client = _FakeLLMClient(content=json.dumps(_valid_payload())) + scores = judge_answer( + query="q", + answer="a", + retrieved_chunks=["chunk"], + expected_traits={"must_mention": ["x"]}, + llm_client=client, + max_retries=0, + ) + assert scores["faithfulness"]["score"] == 5 + assert EXPECTED_DIMENSIONS <= set(scores) + + +def test_judge_answer_falls_back_to_zeros_on_failure(): + client = _FakeLLMClient(raise_exc=True) + scores = judge_answer( + query="q", + answer="a", + retrieved_chunks=[], + expected_traits={}, + llm_client=client, + max_retries=1, + ) + assert EXPECTED_DIMENSIONS <= set(scores) + for dim in EXPECTED_DIMENSIONS: + assert scores[dim]["score"] == 0 + assert scores[dim]["reason"] == "judge_failed" + # 1 initial attempt + 1 retry + assert client.chat.completions.calls == 2 + + +def test_judge_answer_retries_on_malformed_then_zeros(): + client = _FakeLLMClient(content="totally not json") + scores = judge_answer( + query="q", + answer="a", + retrieved_chunks=[], + expected_traits={}, + llm_client=client, + max_retries=2, + ) + for dim in EXPECTED_DIMENSIONS: + assert scores[dim]["score"] == 0 + assert client.chat.completions.calls == 3