From b4a2644b638e1d1ca75367525be05003f55e42c5 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 17 Jul 2026 00:00:47 +0100 Subject: [PATCH] test: move pytest spec tests into data-driven regression suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pytest unit tests were pure parse-and-assert against real .log files with the registered pytest backend — exactly what the regression suite expresses. Convert the three logs into .log/.json fixtures under tests/regressions/samples/pytest/; the full-JSON comparison subsumes every prior assertion (issue counts, error types, messages, stack-trace lines, severities) and locks in exact messages too. Remove test_pytest_spec.py and the now-empty tests/test_specs/data/. The remaining unit tests cover only what has no JSON representation (auto-detection, hints, derived-property logic, synthetic-spec verdicts); update CLAUDE.md's Testing note to say so. --- CLAUDE.md | 2 +- .../regressions/samples/pytest/multiple.json | 76 ++++++ .../samples}/pytest/multiple.log | 0 .../pytest/multiple_collection_error.json | 20 ++ .../pytest/multiple_collection_error.log | 0 .../samples/pytest/successful.json | 5 + .../samples}/pytest/successful.log | 0 tests/test_specs/test_pytest_spec.py | 239 ------------------ 8 files changed, 102 insertions(+), 240 deletions(-) create mode 100644 tests/regressions/samples/pytest/multiple.json rename tests/{test_specs/data => regressions/samples}/pytest/multiple.log (100%) create mode 100644 tests/regressions/samples/pytest/multiple_collection_error.json rename tests/{test_specs/data => regressions/samples}/pytest/multiple_collection_error.log (100%) create mode 100644 tests/regressions/samples/pytest/successful.json rename tests/{test_specs/data => regressions/samples}/pytest/successful.log (100%) delete mode 100644 tests/test_specs/test_pytest_spec.py diff --git a/CLAUDE.md b/CLAUDE.md index dd92cf1..97d2a01 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,7 @@ The library uses a **specification-driven regex parsing** pattern with three lay ## Testing -- **Unit tests** in `tests/test_specs/` — test individual spec parsers against sample data in `tests/test_specs/data/` +- **Unit tests** in `tests/test_specs/` — cover only behaviour a regression fixture can't express: auto-detection, missing-flag hints (excluded from serialized output), derived-property logic (`function_name`/`error_location` precedence), and verdict combinations driven by synthetic specs. Anything that's "parse a real log with a registered backend and check the structured output" belongs in the regression suite, not here. - **Regression tests** in `tests/regressions/` — data-driven: drop a `.log` and matching `.json` into `tests/regressions/samples//` and the test auto-discovers them. The `.json` is the expected CLI JSON output minus the `output` field. Marked with `@pytest.mark.regression`, excluded from default pytest runs. ## Adding a New Verifier Backend diff --git a/tests/regressions/samples/pytest/multiple.json b/tests/regressions/samples/pytest/multiple.json new file mode 100644 index 0000000..05cfc3e --- /dev/null +++ b/tests/regressions/samples/pytest/multiple.json @@ -0,0 +1,76 @@ +{ + "successful": false, + "issues": [ + { + "error_type": "AssertionError", + "message": "assert 2048 == 208", + "stack_trace": [ + { + "trace_index": 0, + "path": "tests/test_ai_models.py", + "name": null, + "line_idx": 26 + } + ], + "severity": "error", + "error_location": null + }, + { + "error_type": "AssertionError", + "message": "assert 0 != 0", + "stack_trace": [ + { + "trace_index": 0, + "path": "tests/test_esbmc.py", + "name": null, + "line_idx": 78 + } + ], + "severity": "error", + "error_location": null + }, + { + "error_type": "AssertionError", + "message": "assert '[Counterexample]' not in 'esbmc --int...ound (k = 5)'", + "stack_trace": [ + { + "trace_index": 0, + "path": "tests/test_esbmc.py", + "name": null, + "line_idx": 85 + } + ], + "severity": "error", + "error_location": null + }, + { + "error_type": "AssertionError", + "message": "assert 'Compilation Error' != 'Compilation Error'", + "stack_trace": [ + { + "trace_index": 0, + "path": "tests/test_esbmc.py", + "name": null, + "line_idx": 135 + } + ], + "severity": "error", + "error_location": null + }, + { + "error_type": "AssertionError", + "message": "Different singleton classes should not share the same instance", + "stack_trace": [ + { + "trace_index": 0, + "path": "tests/test_singleton.py", + "name": null, + "line_idx": 45 + } + ], + "severity": "error", + "error_location": null + } + ], + "duration": 0.0 +} diff --git a/tests/test_specs/data/pytest/multiple.log b/tests/regressions/samples/pytest/multiple.log similarity index 100% rename from tests/test_specs/data/pytest/multiple.log rename to tests/regressions/samples/pytest/multiple.log diff --git a/tests/regressions/samples/pytest/multiple_collection_error.json b/tests/regressions/samples/pytest/multiple_collection_error.json new file mode 100644 index 0000000..159263a --- /dev/null +++ b/tests/regressions/samples/pytest/multiple_collection_error.json @@ -0,0 +1,20 @@ +{ + "successful": false, + "issues": [ + { + "error_type": "SyntaxError", + "message": "'(' was never closed", + "stack_trace": [ + { + "trace_index": 0, + "path": "../../.local/share/hatch/env/virtual/esbmc-ai/9ZN5XdZG/hatch-test.py3.12/lib/python3.12/site-packages/_pytest/python.py", + "name": "importtestmodule", + "line_idx": 497 + } + ], + "severity": "error", + "error_location": null + } + ], + "duration": 0.0 +} diff --git a/tests/test_specs/data/pytest/multiple_collection_error.log b/tests/regressions/samples/pytest/multiple_collection_error.log similarity index 100% rename from tests/test_specs/data/pytest/multiple_collection_error.log rename to tests/regressions/samples/pytest/multiple_collection_error.log diff --git a/tests/regressions/samples/pytest/successful.json b/tests/regressions/samples/pytest/successful.json new file mode 100644 index 0000000..4ac13a1 --- /dev/null +++ b/tests/regressions/samples/pytest/successful.json @@ -0,0 +1,5 @@ +{ + "successful": true, + "issues": [], + "duration": 0.0 +} diff --git a/tests/test_specs/data/pytest/successful.log b/tests/regressions/samples/pytest/successful.log similarity index 100% rename from tests/test_specs/data/pytest/successful.log rename to tests/regressions/samples/pytest/successful.log diff --git a/tests/test_specs/test_pytest_spec.py b/tests/test_specs/test_pytest_spec.py deleted file mode 100644 index e6180d0..0000000 --- a/tests/test_specs/test_pytest_spec.py +++ /dev/null @@ -1,239 +0,0 @@ -# Author: Yiannis Charalambous - -"""Comprehensive test suite for IssueSpecOutputParser with pytest spec. - -This module tests the IssueSpecOutputParser implementation with pytest output including: -- Output parsing with pytest_spec -- Issue extraction from collection errors -- Stack trace parsing from pytest output -- Error type and message extraction -- Successful vs failed verification detection -""" - -from pathlib import Path - -import pytest - -from formal_lib import IssueSpecOutputParser -from formal_lib import VerifierOutput -from formal_lib.specs.pytest import pytest_spec - -DATA_DIR = Path(__file__).parent / "data" / "pytest" - - -# ============================================================================= -# Fixtures -# ============================================================================= - - -@pytest.fixture(scope="module") -def multiple_errors_output() -> VerifierOutput: - """Load the multiple.log sample with 2 collection errors.""" - with open(DATA_DIR / "multiple.log") as file: - parser = IssueSpecOutputParser(pytest_spec) - return parser.parse_output( - output=file.read(), - duration=2.79, - ) - - -@pytest.fixture(scope="module") -def multiple_errors_raw_output() -> str: - """Load raw multiple.log output as string.""" - with open(DATA_DIR / "multiple.log") as file: - return file.read() - - -@pytest.fixture(scope="module") -def successful_output() -> VerifierOutput: - """Load the successful.log sample.""" - with open(DATA_DIR / "successful.log") as file: - parser = IssueSpecOutputParser(pytest_spec) - return parser.parse_output( - output=file.read(), - duration=2.03, - ) - - -@pytest.fixture(scope="module") -def successful_raw_output() -> str: - """Load raw successful.log output as string.""" - with open(DATA_DIR / "successful.log") as file: - return file.read() - - -@pytest.fixture(scope="module") -def collection_error_output() -> VerifierOutput: - """Load the multiple_collection_error.log sample with 1 collection error.""" - with open(DATA_DIR / "multiple_collection_error.log") as file: - parser = IssueSpecOutputParser(pytest_spec) - return parser.parse_output( - output=file.read(), - duration=2.03, - ) - - -@pytest.fixture(scope="module") -def collection_error_raw_output() -> str: - """Load raw multiple_collection_error.log output as string.""" - with open(DATA_DIR / "multiple_collection_error.log") as file: - return file.read() - - -# ============================================================================= -# Tests for Successful Output -# ============================================================================= - - -def test_successful_output_has_no_issues( - successful_output: VerifierOutput, -) -> None: - """Test that successful.log has 0 issues.""" - assert len(successful_output.issues) == 0 - - -def test_successful_output_is_successful( - successful_output: VerifierOutput, -) -> None: - """Test that successful.log verification is successful.""" - assert successful_output.successful is True - - -# ============================================================================= -# Tests for Multiple Errors Output -# ============================================================================= - - -def test_multiple_errors_output_has_5_issues( - multiple_errors_output: VerifierOutput, -) -> None: - """Test that multiple.log has 5 issues.""" - assert len(multiple_errors_output.issues) == 5 - - -def test_multiple_errors_output_is_not_successful( - multiple_errors_output: VerifierOutput, -) -> None: - """Test that multiple.log verification is not successful.""" - assert multiple_errors_output.successful is False - - -def test_multiple_errors_output_stack_traces( - multiple_errors_output: VerifierOutput, -) -> None: - """Test that each issue has the correct stack trace location. - - Expected stack traces (1 trace point each due to low verbosity): - - tests/test_ai_models.py:27 - - tests/test_esbmc.py:79 - - tests/test_esbmc.py:86 - - tests/test_esbmc.py:136 - - tests/test_singleton.py:46 - """ - expected_traces = [ - (Path("tests/test_ai_models.py"), 26), # line 27, 0-based = 26 - (Path("tests/test_esbmc.py"), 78), # line 79, 0-based = 78 - (Path("tests/test_esbmc.py"), 85), # line 86, 0-based = 85 - (Path("tests/test_esbmc.py"), 135), # line 136, 0-based = 135 - (Path("tests/test_singleton.py"), 45), # line 46, 0-based = 45 - ] - - assert len(multiple_errors_output.issues) == len(expected_traces) - - for issue, (expected_path, expected_line_idx) in zip( - multiple_errors_output.issues, expected_traces, strict=True - ): - # Each issue should have exactly 1 stack trace entry (low verbosity) - assert len(issue.stack_trace) == 1 - trace = issue.stack_trace[0] - assert trace.path == expected_path - assert trace.line_idx == expected_line_idx - - -def test_multiple_errors_output_error_types( - multiple_errors_output: VerifierOutput, -) -> None: - """Test that all issues have AssertionError as error type.""" - for issue in multiple_errors_output.issues: - assert issue.error_type == "AssertionError" - - -def test_multiple_errors_output_severity( - multiple_errors_output: VerifierOutput, -) -> None: - """Test that all issues have 'error' severity.""" - for issue in multiple_errors_output.issues: - assert issue.severity == "error" - - -def test_multiple_errors_output_messages_not_empty( - multiple_errors_output: VerifierOutput, -) -> None: - """Test that all issues have non-empty messages.""" - for issue in multiple_errors_output.issues: - assert issue.message, "Issue message should not be empty" - - -# ============================================================================= -# Tests for Collection Error Output -# ============================================================================= - - -def test_collection_error_output_has_1_issue( - collection_error_output: VerifierOutput, -) -> None: - """Test that multiple_collection_error.log has 1 issue.""" - assert len(collection_error_output.issues) == 1 - - -def test_collection_error_output_is_not_successful( - collection_error_output: VerifierOutput, -) -> None: - """Test that collection error verification is not successful.""" - assert collection_error_output.successful is False - - -def test_collection_error_output_error_type( - collection_error_output: VerifierOutput, -) -> None: - """Test that the collection error has SyntaxError as error type.""" - assert len(collection_error_output.issues) == 1 - assert collection_error_output.issues[0].error_type == "SyntaxError" - - -def test_collection_error_output_message( - collection_error_output: VerifierOutput, -) -> None: - """Test that the collection error has the correct message.""" - assert len(collection_error_output.issues) == 1 - assert "'(' was never closed" in collection_error_output.issues[0].message - - -def test_collection_error_output_stack_trace( - collection_error_output: VerifierOutput, -) -> None: - """Test that the collection error has a stack trace. - - Note: For collection errors, the stack trace contains pytest internal paths - (the import chain), not the actual error location. The actual error location - (tests/test_ai_models.py:14) is embedded in the error message lines with - format 'E File "path" line X'. - """ - assert len(collection_error_output.issues) == 1 - issue = collection_error_output.issues[0] - - # Should have at least one stack trace entry (pytest internals) - assert len(issue.stack_trace) >= 1 - - # The first trace point should be from pytest's import mechanism - first_trace = issue.stack_trace[0] - assert first_trace.name == "importtestmodule" - assert "_pytest" in str(first_trace.path) - - -def test_collection_error_output_severity( - collection_error_output: VerifierOutput, -) -> None: - """Test that the collection error has 'error' severity.""" - assert len(collection_error_output.issues) == 1 - assert collection_error_output.issues[0].severity == "error"