From 46f707b240398c8db962976d2e3ffa953f2497c6 Mon Sep 17 00:00:00 2001 From: AbiramiR-27 Date: Thu, 9 Jul 2026 00:04:29 +0530 Subject: [PATCH 1/3] feat: add HTML/Markdown reporting and failure diagnostics to verifier (#87) --- src/ovllm.py | 6 +- src/verifier.py | 356 ++++++++++++++++++++++++++++++++++++++++- tests/test_verifier.py | 49 +++++- 3 files changed, 403 insertions(+), 8 deletions(-) diff --git a/src/ovllm.py b/src/ovllm.py index bb39d3f..b69f0d0 100644 --- a/src/ovllm.py +++ b/src/ovllm.py @@ -18,7 +18,7 @@ def _verify(args) -> int: print(f"[FAIL] verifier_error - {exc}") print("\nVERDICT: RED") return 1 - return 0 if print_report(results) else 1 + return 0 if print_report(results, format_type=args.format, output_path=args.output, ref=args.model_ref) else 1 def _prepare_publish(args) -> int: @@ -56,6 +56,10 @@ def main(argv=None) -> int: verify.add_argument("--allow-unsigned", action="store_true", help="treat a missing Sigstore bundle as SKIP instead of FAIL") verify.add_argument("--skip-replay", action="store_true") + verify.add_argument("--format", choices=["text", "markdown", "html"], default="text", + help="the format of the verification report") + verify.add_argument("--output", default=None, + help="path to write the formatted verification report") verify.set_defaults(func=_verify) prep = sub.add_parser("prepare-publish", help="build a publishable model directory") diff --git a/src/verifier.py b/src/verifier.py index 7f8fb5d..e176410 100644 --- a/src/verifier.py +++ b/src/verifier.py @@ -1,6 +1,7 @@ import importlib import json import os +import platform import subprocess import sys from dataclasses import dataclass @@ -318,13 +319,356 @@ def verify_model_reference( return results -def print_report(results: List[CheckResult]) -> bool: +def get_system_diagnostics() -> Dict[str, str]: + diag = { + "OS": f"{platform.system()} {platform.release()} ({platform.machine()})", + "Python Version": platform.python_version(), + } + try: + import torch + diag["PyTorch Version"] = torch.__version__ + if torch.cuda.is_available(): + diag["Accelerator"] = f"CUDA ({torch.cuda.get_device_name(0)})" + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + diag["Accelerator"] = f"XPU ({torch.xpu.get_device_name(0)})" + else: + diag["Accelerator"] = "CPU" + except ImportError: + diag["PyTorch Version"] = "Not Installed" + diag["Accelerator"] = "N/A" + return diag + + +def generate_markdown_report(results: List[CheckResult], ok: bool, ref: str, diag: Dict[str, str]) -> str: + verdict_str = "🟢 **GREEN** (Passed)" if ok else "🔴 **RED** (Failed)" + md = [] + md.append("# OpenVerifiableLLM Verification Report\n") + md.append(f"**Verdict:** {verdict_str}\n") + md.append(f"- **Model Reference:** `{ref}`") + md.append(f"- **Timestamp:** `{time_ref()}`\n") + + md.append("## Check Results\n") + md.append("| Status | Check Name | Expected | Actual | Details |") + md.append("| :--- | :--- | :--- | :--- | :--- |") + for r in results: + status_icon = "✅ PASS" if r.status == PASS else ("❌ FAIL" if r.status == FAIL else "⚠️ SKIP") + expected_val = f"`{r.expected}`" if r.expected else "-" + actual_val = f"`{r.actual}`" if r.actual else "-" + detail_val = r.detail.replace("\n", " ") if r.detail else "" + md.append(f"| {status_icon} | **{r.name}** | {expected_val} | {actual_val} | {detail_val} |") + + md.append("\n## System Diagnostics\n") + for k, v in diag.items(): + md.append(f"- **{k}:** `{v}`") + + return "\n".join(md) + "\n" + + +def generate_html_report(results: List[CheckResult], ok: bool, ref: str, diag: Dict[str, str]) -> str: + verdict_class = "verdict-green" if ok else "verdict-red" + verdict_text = "VERDICT: GREEN" if ok else "VERDICT: RED" + + rows = [] + for r in results: + status_class = f"status-{r.status.lower()}" + expected_val = f"{r.expected}" if r.expected else "-" + actual_val = f"{r.actual}" if r.actual else "-" + detail_val = r.detail.replace("\n", "
") if r.detail else "" + rows.append(f""" + + {r.status} + {r.name} + {expected_val} + {actual_val} + {detail_val} + + """) + rows_html = "\n".join(rows) + + diag_items = [] + for k, v in diag.items(): + diag_items.append(f""" +
+ {k} + {v} +
+ """) + diag_html = "\n".join(diag_items) + + html = f""" + + + + OpenVerifiableLLM Verification Report + + + +
+
+

OpenVerifiableLLM Report

+
+
+ Status +
{verdict_text}
+
+
+ Model Reference + {ref} +
+
+ Timestamp + {time_ref()} +
+
+
+ +
+

Verification Check Results

+ + + + + + + + + + + + {rows_html} + +
StatusCheck NameExpectedActualDetails
+
+ +
+

System Diagnostics

+
+ {diag_html} +
+
+
+ + +""" + return html + + +def time_ref() -> str: + import time + return time.strftime("%Y-%m-%dT%H:%M:%S") + + +def print_report( + results: List[CheckResult], + format_type: str = "text", + output_path: Optional[str] = None, + ref: str = "", +) -> bool: + ok = all(result.ok for result in results) + + # 1. Generate text output + text_lines = [] for result in results: suffix = f" - {result.detail}" if result.detail else "" - print(f"[{result.status:<4}] {result.name}{suffix}") + text_lines.append(f"[{result.status:<4}] {result.name}{suffix}") if result.expected is not None or result.actual is not None: - print(f" expected: {result.expected}") - print(f" actual : {result.actual}") - ok = all(result.ok for result in results) - print("\nVERDICT:", "GREEN" if ok else "RED") + text_lines.append(f" expected: {result.expected}") + text_lines.append(f" actual : {result.actual}") + text_lines.append(f"\nVERDICT: {'GREEN' if ok else 'RED'}") + text_str = "\n".join(text_lines) + "\n" + + # 2. Gather diagnostics + diag = get_system_diagnostics() + + # 3. Handle reporting depending on format + if format_type == "text": + print(text_str, end="") + if output_path: + Path(output_path).write_text(text_str, encoding="utf-8") + elif format_type == "markdown": + md_str = generate_markdown_report(results, ok, ref, diag) + if output_path: + Path(output_path).write_text(md_str, encoding="utf-8") + else: + print(md_str, end="") + elif format_type == "html": + html_str = generate_html_report(results, ok, ref, diag) + if output_path: + Path(output_path).write_text(html_str, encoding="utf-8") + else: + print(html_str, end="") + return ok diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 46a4a58..fe5a4ae 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -96,7 +96,9 @@ def test_sigstore_verify_ignores_signature_file(self): manifest["sigstore_identity"] = "person@example.com" manifest["sigstore_identity_provider"] = "https://accounts.example.com" - with patch("verifier.subprocess.run") as mock_run: + with patch("verifier.importlib.util.find_spec") as mock_find_spec, \ + patch("verifier.subprocess.run") as mock_run: + mock_find_spec.return_value = "mock_spec" mock_run.return_value.returncode = 0 mock_run.return_value.stdout = "ok" mock_run.return_value.stderr = "" @@ -219,5 +221,50 @@ def snapshot_download(**kwargs): ) +class ReportFormattingTests(unittest.TestCase): + def test_text_markdown_html_report_generation(self): + from verifier import CheckResult, print_report, PASS, FAIL + + results = [ + CheckResult("test_pass", PASS, "Pass detail"), + CheckResult("test_fail", FAIL, "Fail detail", expected="123", actual="456"), + ] + + # Test Markdown formatting + with tempfile.TemporaryDirectory() as tmp: + md_path = Path(tmp) / "report.md" + print_report(results, format_type="markdown", output_path=str(md_path), ref="dummy-ref") + md_content = md_path.read_text(encoding="utf-8") + self.assertIn("# OpenVerifiableLLM Verification Report", md_content) + self.assertIn("RED", md_content) + self.assertIn("test_pass", md_content) + self.assertIn("test_fail", md_content) + self.assertIn("123", md_content) + self.assertIn("456", md_content) + + # Test HTML formatting + with tempfile.TemporaryDirectory() as tmp: + html_path = Path(tmp) / "report.html" + print_report(results, format_type="html", output_path=str(html_path), ref="dummy-ref") + html_content = html_path.read_text(encoding="utf-8") + self.assertIn("", html_content) + self.assertIn("OpenVerifiableLLM Report", html_content) + self.assertIn("VERDICT: RED", html_content) + self.assertIn("test_fail", html_content) + self.assertIn("status-fail", html_content) + + # Test CLI arguments verify execution for formats + with tempfile.TemporaryDirectory() as tmp: + weights = Path(tmp) / "model.safetensors" + save_file({"layer.weight": torch.arange(8, dtype=torch.float32).reshape(2, 4)}, str(weights)) + model_dir = prepare_publish_dir(weights=str(weights), output_dir=str(Path(tmp) / "publish")) + + md_path = Path(tmp) / "cli_report.md" + code = ovllm_main(["verify", str(model_dir), "--allow-unsigned", "--skip-replay", "--format", "markdown", "--output", str(md_path)]) + self.assertEqual(code, 0) + self.assertTrue(md_path.exists()) + self.assertIn("# OpenVerifiableLLM Verification Report", md_path.read_text(encoding="utf-8")) + + if __name__ == "__main__": unittest.main() From 28bce35af5521d9b92117f8577909f4f53e6c91f Mon Sep 17 00:00:00 2001 From: AbiramiR-27 Date: Thu, 9 Jul 2026 10:22:35 +0530 Subject: [PATCH 2/3] address maintainer bot review comments: escape pipes, XSS protection, and format validation --- src/verifier.py | 37 +++++++++++++++++++++++-------------- tests/test_verifier.py | 40 +++++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/src/verifier.py b/src/verifier.py index e176410..9439403 100644 --- a/src/verifier.py +++ b/src/verifier.py @@ -1,3 +1,4 @@ +import html import importlib import json import os @@ -339,6 +340,11 @@ def get_system_diagnostics() -> Dict[str, str]: return diag +def escape_markdown_cell(val: str) -> str: + """Escapes markdown pipe characters to prevent breaking table structure.""" + return val.replace("|", "\\|") + + def generate_markdown_report(results: List[CheckResult], ok: bool, ref: str, diag: Dict[str, str]) -> str: verdict_str = "🟢 **GREEN** (Passed)" if ok else "🔴 **RED** (Failed)" md = [] @@ -352,10 +358,11 @@ def generate_markdown_report(results: List[CheckResult], ok: bool, ref: str, dia md.append("| :--- | :--- | :--- | :--- | :--- |") for r in results: status_icon = "✅ PASS" if r.status == PASS else ("❌ FAIL" if r.status == FAIL else "⚠️ SKIP") - expected_val = f"`{r.expected}`" if r.expected else "-" - actual_val = f"`{r.actual}`" if r.actual else "-" - detail_val = r.detail.replace("\n", " ") if r.detail else "" - md.append(f"| {status_icon} | **{r.name}** | {expected_val} | {actual_val} | {detail_val} |") + name_val = escape_markdown_cell(r.name) + expected_val = f"`{escape_markdown_cell(r.expected)}`" if r.expected else "-" + actual_val = f"`{escape_markdown_cell(r.actual)}`" if r.actual else "-" + detail_val = escape_markdown_cell(r.detail.replace("\n", " ")) if r.detail else "" + md.append(f"| {status_icon} | **{name_val}** | {expected_val} | {actual_val} | {detail_val} |") md.append("\n## System Diagnostics\n") for k, v in diag.items(): @@ -371,13 +378,13 @@ def generate_html_report(results: List[CheckResult], ok: bool, ref: str, diag: D rows = [] for r in results: status_class = f"status-{r.status.lower()}" - expected_val = f"{r.expected}" if r.expected else "-" - actual_val = f"{r.actual}" if r.actual else "-" - detail_val = r.detail.replace("\n", "
") if r.detail else "" + expected_val = f"{html.escape(r.expected)}" if r.expected else "-" + actual_val = f"{html.escape(r.actual)}" if r.actual else "-" + detail_val = html.escape(r.detail).replace("\n", "
") if r.detail else "" rows.append(f""" - {r.status} - {r.name} + {html.escape(r.status)} + {html.escape(r.name)} {expected_val} {actual_val} {detail_val} @@ -389,13 +396,13 @@ def generate_html_report(results: List[CheckResult], ok: bool, ref: str, diag: D for k, v in diag.items(): diag_items.append(f"""
- {k} - {v} + {html.escape(k)} + {html.escape(v)}
""") diag_html = "\n".join(diag_items) - html = f""" + report_html = f""" @@ -586,7 +593,7 @@ def generate_html_report(results: List[CheckResult], ok: bool, ref: str, diag: D
Model Reference - {ref} + {html.escape(ref)}
Timestamp @@ -623,7 +630,7 @@ def generate_html_report(results: List[CheckResult], ok: bool, ref: str, diag: D """ - return html + return report_html def time_ref() -> str: @@ -670,5 +677,7 @@ def print_report( Path(output_path).write_text(html_str, encoding="utf-8") else: print(html_str, end="") + else: + raise ValueError(f"Unknown format_type: {format_type!r}") return ok diff --git a/tests/test_verifier.py b/tests/test_verifier.py index fe5a4ae..a37f829 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -228,6 +228,8 @@ def test_text_markdown_html_report_generation(self): results = [ CheckResult("test_pass", PASS, "Pass detail"), CheckResult("test_fail", FAIL, "Fail detail", expected="123", actual="456"), + CheckResult("test_pipe|", PASS, "detail | contains | pipes", expected="a|b", actual="c|d"), + CheckResult("", PASS, "some here", expected="", actual=""), ] # Test Markdown formatting @@ -241,29 +243,45 @@ def test_text_markdown_html_report_generation(self): self.assertIn("test_fail", md_content) self.assertIn("123", md_content) self.assertIn("456", md_content) + # Verify pipe escaping in Markdown + self.assertIn("test_pipe\\|", md_content) + self.assertIn("a\\|b", md_content) + self.assertIn("c\\|d", md_content) + self.assertIn("detail \\| contains \\| pipes", md_content) # Test HTML formatting with tempfile.TemporaryDirectory() as tmp: html_path = Path(tmp) / "report.html" - print_report(results, format_type="html", output_path=str(html_path), ref="dummy-ref") + print_report(results, format_type="html", output_path=str(html_path), ref="dummy-ref", PASS, "some here", expected="", actual=""), ] + # Test Text formatting + with tempfile.TemporaryDirectory() as tmp: + text_path = Path(tmp) / "report.txt" + with patch("builtins.print"): + print_report(results, format_type="text", output_path=str(text_path), ref="dummy-ref") + text_content = text_path.read_text(encoding="utf-8") + self.assertIn("VERDICT: RED", text_content) + self.assertIn("[PASS] test_pass", text_content) + self.assertIn("[FAIL] test_fail", text_content) + self.assertIn("expected: 123", text_content) + self.assertIn("actual : 456", text_content) + # Test Markdown formatting with tempfile.TemporaryDirectory() as tmp: md_path = Path(tmp) / "report.md"