From d29f9f773deb160c4967ad90db2a041d98633e92 Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Thu, 23 Jul 2026 23:27:38 +0800 Subject: [PATCH 1/2] refactor(flows): wire the semantic shape through PaperFlow.build and deprecate paper_flow PaperFlow.build now selects both paper knowledge shapes by cfg type: PaperStructureCfg -> PaperStructureTree (unchanged) and PaperFlowCfg -> PaperFlowResult (moved in from paper_flow, not duplicated). PaperFlow is generic over its result type, with constructor overloads that bind the result type from the cfg type, so each shape's callers keep a precise static return under basedpyright standard mode; a BaseFlowCfg fallback overload keeps other cfg types constructible and dispatches to NotImplementedError at runtime. paper_flow becomes a deprecated thin wrapper that emits a DeprecationWarning and delegates to PaperFlow(PaperFlowCfg(...)).build(input). Its signature is unchanged, so the magic-input introspection path is unaffected, and it stays grandfathered by the operation-naming contract rather than being removed. Tests, the paper example, the pdf-rag e2e slice, the paper flow design page, the naming contract, the library guide, and the component catalog row move to the config-bound entry point; a new test covers the deprecation warning and delegation. Co-Authored-By: Claude Fable 5 --- contexts/design/flow/paper.md | 10 +- contexts/design/operations/naming.md | 4 +- docs/README.md | 4 +- docs/library.md | 9 +- examples/flows/paper.py | 10 +- quantmind/flows/__init__.py | 8 +- quantmind/flows/paper/__init__.py | 215 ++++++++++++++++++--------- scripts/verify_pdf_rag_e2e.py | 10 +- tests/flows/test_paper.py | 71 ++++++--- tests/flows/test_structure.py | 11 +- 10 files changed, 231 insertions(+), 121 deletions(-) diff --git a/contexts/design/flow/paper.md b/contexts/design/flow/paper.md index 63cbda8..c90afe4 100644 --- a/contexts/design/flow/paper.md +++ b/contexts/design/flow/paper.md @@ -3,8 +3,8 @@ ## Quick Summary - **Purpose**: Define how one exact PDF revision becomes a durable page-aware chunk set and a cited global summary. -- **Read when**: Changing `paper_flow`, paper inputs, summarization limits, citation validation, or the end-to-end paper verifier. -- **Status**: Implemented by `quantmind.flows.paper_flow` for PDF-backed arXiv, HTTP, and local inputs. +- **Read when**: Changing the paper build (`PaperFlow(PaperFlowCfg).build` or the deprecated `paper_flow` wrapper), paper inputs, summarization limits, citation validation, or the end-to-end paper verifier. +- **Status**: Implemented by the config-bound `quantmind.flows.PaperFlow` — a `PaperFlowCfg` selects this source-first shape — for PDF-backed arXiv, HTTP, and local inputs. `paper_flow` remains as a deprecated thin wrapper that delegates to it. - **Core rule**: Preserve and validate the source revision and chunk set before any model-generated summary is accepted. - **Canonical models**: [Paper source and artifact design](../knowledge/paper.md). @@ -23,7 +23,7 @@ ## Contract -`paper_flow(input, *, cfg)` returns one validated `PaperFlowResult`: +`PaperFlow(PaperFlowCfg(...)).build(input)` returns one validated `PaperFlowResult`. `PaperFlow` binds the immutable `PaperFlowCfg` once and its cfg **type** selects this source-first shape, so `batch_run(flow.build, inputs)` runs a batch under one setting. The deprecated `paper_flow(input, *, cfg)` wrapper emits a `DeprecationWarning` and delegates to the same build. ```text PaperFlowResult @@ -61,7 +61,7 @@ The operation has a strict order: Steps 3, 5, 9, and 10 mint no IDs in the flow: the flow calls the knowledge-layer smart constructors `PaperSourceRevision.from_parsed`, `PaperChunkSet.from_parsed_chunks`, and `PaperGlobalSummary.from_draft`, which own every ID, content/producer hash, and citation resolution. The flow only fetches, parses, reads asset bytes, and maps those path-based artifacts into knowledge-native inputs. See [orchestration principles](../operations/orchestration.md). -A summarization failure occurs after source and chunks exist in memory, but `paper_flow` returns no partial success value. Persistence is a separate explicit operation. +A summarization failure occurs after source and chunks exist in memory, but the build returns no partial success value. Persistence is a separate explicit operation. ## Source Revision @@ -152,4 +152,4 @@ It fetches exact arXiv revision `1706.03762v7`, parses at least 15 physical page - DOI-to-open-PDF resolution; - question answering over search results; - hidden or unbounded model calls; -- implicit persistence from `paper_flow`. +- implicit persistence from the paper build. diff --git a/contexts/design/operations/naming.md b/contexts/design/operations/naming.md index 8362351..f4d60bd 100644 --- a/contexts/design/operations/naming.md +++ b/contexts/design/operations/naming.md @@ -71,8 +71,8 @@ A domain flow binds an immutable `cfg` at construction and applies it to each in - `collect_news` is a collection operation and follows these naming rules. - `batch_run` is a generic batch helper, not a news or paper operation. -- `paper_flow` is an existing extraction **function** with an old name; keep it as a thin compatibility wrapper. Do not copy the `*_flow` function pattern for new operations. -- `PaperFlow(cfg)` is the config-bound paper flow; `build(input)` produces a knowledge artifact whose shape is chosen by the cfg *type* (`PaperStructureCfg` → `PaperStructureTree` today). `Flow` as a noun here is deliberate and allowed. +- `paper_flow` is an existing extraction **function** with an old name, now a deprecated thin wrapper that emits a `DeprecationWarning` and delegates to `PaperFlow(PaperFlowCfg(...)).build(input)`. Do not copy the `*_flow` function pattern for new operations. +- `PaperFlow(cfg)` is the config-bound paper flow; `build(input)` produces a knowledge artifact whose shape is chosen by the cfg *type* (`PaperStructureCfg` → `PaperStructureTree`, `PaperFlowCfg` → `PaperFlowResult`). `Flow` as a noun here is deliberate and allowed. - `AgenticRetriever(cfg)` is the reasoning-retrieval service in `quantmind.mind`; `retrieve(structure, question)` returns evidence values. It has one behavior — an LLM agent reasons over the structure — so it binds `RetrievalCfg` but does no cfg-type dispatch. Mechanical semantic search is `quantmind.library.search`, a different layer, not a strategy here. The current `quantmind.flows` package continues to contain public operations in this release. Renaming it to `operations` or adding a separate `pipelines` package is outside this page. diff --git a/docs/README.md b/docs/README.md index 43a3ab0..04df584 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,7 @@ harness. | Operation | Import | Input and config | Result | Example | Design or guide | |---|---|---|---|---|---| -| Source-first paper flow | `quantmind.flows.paper_flow` | `PaperInput`, `PaperFlowCfg` | `PaperFlowResult` | [Persist and search a paper](../examples/flows/paper.py) | [Paper flow design](../contexts/design/flow/paper.md) | +| Source-first paper flow | `quantmind.flows.PaperFlow` | `PaperFlow(PaperFlowCfg)`; `build()`: `PaperInput` | `PaperFlowResult` | [Persist and search a paper](../examples/flows/paper.py) | [Paper flow design](../contexts/design/flow/paper.md) | | Paper structure build | `quantmind.flows.PaperFlow` | `PaperFlow(PaperStructureCfg)`; `build()`: `PaperInput` | `PaperStructureTree` (self-contained) | [Build and retrieve](../examples/mind/paper_structure_retrieval.py) | [Structure retrieval design](../contexts/design/mind/retrieval.md) | | Reasoning-based retrieval (agentic) | `quantmind.mind.AgenticRetriever` | `AgenticRetriever(RetrievalCfg)`; `retrieve()`: one `StructureTree` + question (no library) | `list[RetrievalEvidence]` | [Build and retrieve](../examples/mind/paper_structure_retrieval.py) | [Structure retrieval design](../contexts/design/mind/retrieval.md) | | News collection | `quantmind.flows.collect_news` | `NewsWindow`, `NewsCollectionCfg` | `NewsBatch` from `quantmind.preprocess` | [Collect news](../examples/flows/collect_news.py) | [News collection design](../contexts/design/flow/news.md) | @@ -31,7 +31,7 @@ Import result contracts from the canonical layer shown in the catalog. | Source | Source selection | Operation | Live-network component smoke test | |---|---|---|---| | PR Newswire | `NewsWindow(source="pr-newswire", ...)` | `collect_news` | `python scripts/verify_news_e2e.py` | -| arXiv Transformer PDF | `ArxivIdentifier(id="1706.03762v7")` | `paper_flow`, persistence, reopen, search, and resolution | `python scripts/verify_pdf_rag_e2e.py` | +| arXiv Transformer PDF | `ArxivIdentifier(id="1706.03762v7")` | `PaperFlow(PaperFlowCfg).build`, persistence, reopen, search, and resolution | `python scripts/verify_pdf_rag_e2e.py` | | Golden paper PDF (structure) | `LocalFilePath(...golden/paper.pdf)` | `PaperFlow.build`, standalone `put`/`open_structure`, and `AgenticRetriever.retrieve` | `python scripts/verify_structure_e2e.py` | The PR Newswire smoke test checks the public RSS feed, a complete preceding diff --git a/docs/library.md b/docs/library.md index 91ed03d..f763fec 100644 --- a/docs/library.md +++ b/docs/library.md @@ -4,13 +4,12 @@ ## Store a Paper Flow Result -Run `paper_flow()` first, then explicitly store its complete result: +Run `PaperFlow(PaperFlowCfg(...)).build()` first, then explicitly store its complete result: ```python -result = await paper_flow( - ArxivIdentifier(id="1706.03762v7"), - cfg=PaperFlowCfg(model="gpt-4o-mini"), -) +result = await PaperFlow( + PaperFlowCfg(model="gpt-4o-mini"), +).build(ArxivIdentifier(id="1706.03762v7")) library = await LocalKnowledgeLibrary.open( ".quantmind/library.db", diff --git a/examples/flows/paper.py b/examples/flows/paper.py index 2c1e4c4..971e4d9 100644 --- a/examples/flows/paper.py +++ b/examples/flows/paper.py @@ -9,7 +9,7 @@ from quantmind.configs import PaperFlowCfg from quantmind.configs.paper import ArxivIdentifier -from quantmind.flows import paper_flow +from quantmind.flows import PaperFlow from quantmind.knowledge import ( PaperArtifactKind, PaperChunk, @@ -62,13 +62,13 @@ async def main() -> None: workspace = Path(".quantmind") workspace.mkdir(exist_ok=True) - result = await paper_flow( - ArxivIdentifier(id=_ARXIV_ID), - cfg=PaperFlowCfg( + flow = PaperFlow( + PaperFlowCfg( model="gpt-4o-mini", output_dir=str(workspace / "attention-assets"), - ), + ) ) + result = await flow.build(ArxivIdentifier(id=_ARXIV_ID)) database = workspace / "library.db" library = await LocalKnowledgeLibrary.open( diff --git a/quantmind/flows/__init__.py b/quantmind/flows/__init__.py index faed970..bfe0dbb 100644 --- a/quantmind/flows/__init__.py +++ b/quantmind/flows/__init__.py @@ -7,10 +7,10 @@ - ``PaperFlow`` is the config-bound paper flow: ``PaperFlow(cfg)`` binds an immutable build config once and ``build(input)`` applies it per input, dispatching on the cfg **type** (``PaperStructureCfg`` → a self-contained - ``PaperStructureTree``). ``batch_run(flow.build, inputs)`` runs every input - under one unified setting. -- ``paper_flow`` is a thin compatibility function for the semantic - chunk/summary shape (``PaperFlowResult``). + ``PaperStructureTree``; ``PaperFlowCfg`` → a source-first ``PaperFlowResult``). + ``batch_run(flow.build, inputs)`` runs every input under one unified setting. +- ``paper_flow`` is a **deprecated** thin wrapper for the chunk/summary shape; + it warns and delegates to ``PaperFlow(PaperFlowCfg(...)).build(input)``. - ``batch_run`` runs any flow over a list of inputs with bounded concurrency and aggregated results. - ``BatchResult`` is the shape returned by ``batch_run``. diff --git a/quantmind/flows/paper/__init__.py b/quantmind/flows/paper/__init__.py index ca179b2..8aa0b6d 100644 --- a/quantmind/flows/paper/__init__.py +++ b/quantmind/flows/paper/__init__.py @@ -7,13 +7,21 @@ tree = await tree_flow.build(input) # -> tree trees = await batch_run(tree_flow.build, inputs) # one setting -``build(input)`` takes only the operand and dispatches on the cfg **type**: a -``PaperStructureCfg`` selects the self-contained ``PaperStructureTree`` shape -(fetch + parse, deterministic outline signals, one draft-structuring agent, then -the knowledge-layer ``from_draft`` constructor that mints identity and populates -each leaf node's page-cited text). Any other cfg type raises -``NotImplementedError`` — the chunk/summary (card) shape is not wired through -``build`` yet; it stays in the ``paper_flow`` compatibility function. +``build(input)`` takes only the operand and dispatches on the cfg **type**, so +one config-bound flow produces every paper shape: + +- ``PaperStructureCfg`` selects the self-contained ``PaperStructureTree`` shape + (fetch + parse, deterministic outline signals, one draft-structuring agent, + then the knowledge-layer ``from_draft`` constructor that mints identity and + populates each leaf node's page-cited text). +- ``PaperFlowCfg`` selects the source-first chunk/summary shape + (``PaperFlowResult``): fetch + parse, page-aware chunking, then a bounded + map-reduce summary whose citations the knowledge layer resolves. +- Any other cfg type raises ``NotImplementedError``. + +The bound cfg *type* determines ``build``'s result type: constructing with a +``PaperStructureCfg`` yields a ``PaperStructureTree``; a ``PaperFlowCfg`` yields +a ``PaperFlowResult`` (typed via constructor overloads over a generic result). ``build`` fetches and parses **per call**: the flow binds no source, no library, persists nothing, and retrieves nothing. Persistence (``library``) and retrieval @@ -26,16 +34,19 @@ knowledge models' ``from_*`` constructors, so this module imports no private ID helpers and computes no paper ID itself. -``paper_flow`` remains as a thin compatibility function for the semantic -chunk/summary shape (``PaperFlowResult``). +``paper_flow`` remains as a **deprecated** thin wrapper for the chunk/summary +shape: it emits a ``DeprecationWarning`` and delegates to +``PaperFlow(PaperFlowCfg(...)).build(input)``. It is grandfathered by the +operation-naming contract, not removed. """ import mimetypes +import warnings from collections.abc import Sequence from datetime import datetime, timezone from importlib.metadata import version from pathlib import Path -from typing import Literal +from typing import Generic, Literal, TypeVar, cast, overload from quantmind.configs import BaseFlowCfg, PaperFlowCfg, PaperStructureCfg from quantmind.configs.paper import ( @@ -108,48 +119,95 @@ class UnsupportedContentTypeError(ValueError): """The source is not a page-aware PDF supported by Paper Flow V1.""" -class PaperFlow: +# ``build``'s result type is selected by the bound cfg **type**; the +# constructor overloads below bind ``_ResultT`` accordingly. +_ResultT = TypeVar("_ResultT") + + +class PaperFlow(Generic[_ResultT]): """Config-bound flow producing a paper knowledge artifact per input. ``PaperFlow(cfg)`` binds an immutable copy of the build config; ``build`` applies it to each input, so a batch runs under one unified, reproducible setting and a config can never drift mid-run. The cfg **type** selects the - knowledge shape: ``PaperStructureCfg`` builds a self-contained - ``PaperStructureTree`` today. The flow binds no source, no library, and no - per-call state; ``build`` fetches and parses per call. + knowledge shape, and — through the constructor overloads — the static result + type of ``build``: + + - ``PaperStructureCfg`` builds a self-contained ``PaperStructureTree``; + - ``PaperFlowCfg`` builds a source-first ``PaperFlowResult`` (chunk set + + cited global summary). + + The flow binds no source, no library, and no per-call state; ``build`` + fetches and parses per call. """ __slots__ = ( "_cfg", "_structure_provider", + "_summary_provider", ) + @overload + def __init__( + self: "PaperFlow[PaperStructureTree]", + cfg: PaperStructureCfg, + *, + _structure_provider: _PaperStructureProvider | None = None, + _summary_provider: _PaperSummaryProvider | None = None, + ) -> None: ... + + @overload + def __init__( + self: "PaperFlow[PaperFlowResult]", + cfg: PaperFlowCfg, + *, + _structure_provider: _PaperStructureProvider | None = None, + _summary_provider: _PaperSummaryProvider | None = None, + ) -> None: ... + + @overload + def __init__( + self: "PaperFlow[PaperStructureTree | PaperFlowResult]", + cfg: BaseFlowCfg, + *, + _structure_provider: _PaperStructureProvider | None = None, + _summary_provider: _PaperSummaryProvider | None = None, + ) -> None: ... + def __init__( self, cfg: BaseFlowCfg, *, _structure_provider: _PaperStructureProvider | None = None, + _summary_provider: _PaperSummaryProvider | None = None, ) -> None: """Bind an immutable copy of the build config. Args: - cfg: The build config. Its **type** selects the knowledge shape - ``build`` produces (``PaperStructureCfg`` → structure tree). + cfg: The build config. Its **type** selects both the knowledge shape + ``build`` produces and ``build``'s static result type + (``PaperStructureCfg`` → ``PaperStructureTree``, ``PaperFlowCfg`` + → ``PaperFlowResult``). _structure_provider: Optional test seam for the structure draft. + _summary_provider: Optional test seam for the summary draft. """ self._cfg: BaseFlowCfg = cfg.model_copy(deep=True) self._structure_provider = _structure_provider + self._summary_provider = _summary_provider - async def build(self, input: PaperInput) -> PaperStructureTree: + async def build(self, input: PaperInput) -> _ResultT: """Build one self-contained knowledge artifact for ``input``. - Dispatches on the bound cfg **type**. A ``PaperStructureCfg`` runs the - structure pipeline: fetch and parse the input, extract deterministic - outline signals, seed a single draft-structuring agent, then call the - knowledge-layer constructor, which mints identity, resolves page - citations, and populates each leaf node's ``content`` from its cited - source pages. The returned tree is self-contained: it yields node text - without the source revision or a chunk set. + Dispatches on the bound cfg **type**: + + - ``PaperStructureCfg`` runs the structure pipeline (fetch + parse, + deterministic outline signals, one draft-structuring agent, then the + knowledge-layer constructor that mints identity, resolves page + citations, and populates each leaf node's ``content``), returning a + self-contained ``PaperStructureTree``. + - ``PaperFlowCfg`` runs the source-first chunk/summary pipeline (fetch + + parse, page-aware chunking, bounded map-reduce summary), returning a + ``PaperFlowResult``. Fetch and parse run **per call**; the flow keeps no shared source state. @@ -157,24 +215,27 @@ async def build(self, input: PaperInput) -> PaperStructureTree: input: Typed paper source. V1 requires a PDF-backed input. Returns: - A code-identified ``PaperStructureTree`` whose leaf nodes carry - cited page text. + The knowledge artifact for the bound cfg type: a + ``PaperStructureTree`` (``PaperStructureCfg``) or a + ``PaperFlowResult`` (``PaperFlowCfg``). Raises: - NotImplementedError: If the bound cfg is not a ``PaperStructureCfg`` - (the chunk/summary shape stays in ``paper_flow``). + NotImplementedError: If the bound cfg is neither a + ``PaperStructureCfg`` nor a ``PaperFlowCfg``. UnsupportedContentTypeError: If the resolved content is not a PDF. - PaperStructureError: If the model call exceeds its timeout. - StructureTreeValidationError: If the proposed page tree is invalid. + PaperStructureError: If a structure model call exceeds its timeout. + PaperCitationValidationError: If summary citations are invalid or do + not meet the configured source-coverage policy. """ cfg = self._cfg if isinstance(cfg, PaperStructureCfg): - return await self._build_structure(input, cfg) + return cast(_ResultT, await self._build_structure(input, cfg)) + if isinstance(cfg, PaperFlowCfg): + return cast(_ResultT, await self._build_semantic(input, cfg)) raise NotImplementedError( "PaperFlow.build does not support cfg type " f"{type(cfg).__name__!r}; only PaperStructureCfg (structure-tree " - "shape) is wired today. The chunk/summary shape stays in the " - "paper_flow() compatibility function." + "shape) and PaperFlowCfg (chunk/summary shape) are wired." ) async def _build_structure( @@ -202,6 +263,45 @@ async def _build_structure( draft=draft, ) + async def _build_semantic( + self, + input: PaperInput, + cfg: PaperFlowCfg, + ) -> PaperFlowResult: + """Fetch, parse, chunk, and summarize one input into a paper result. + + Builds a page-aware chunk set and one cited global summary. IDs, source + metadata, artifact membership, lineage, and citation links are minted + and validated by the knowledge-layer constructors; the model returns + only summary prose and chunk/page coordinates through a bounded seam. + """ + source, parsed = await _open_source(input, output_dir=cfg.output_dir) + parsed_chunks = chunk_parsed_document( + parsed, + config=SentenceSplitterConfig( + chunk_size=cfg.chunk_size, + chunk_overlap=cfg.chunk_overlap, + ), + ) + producer = PaperChunkingConfig( + splitter_version=version("llama-index-core"), + chunk_size=cfg.chunk_size, + chunk_overlap=cfg.chunk_overlap, + ) + chunk_set = PaperChunkSet.from_parsed_chunks( + source, + _adapt_chunks(parsed_chunks, source), + producer=producer, + ) + provider = self._summary_provider or _AgentsPaperSummaryProvider() + draft = await provider.summarize(source, chunk_set, cfg=cfg) + summary = _build_summary(chunk_set, draft, cfg) + return PaperFlowResult( + source_revision=source, + chunk_set=chunk_set, + global_summary=summary, + ) + async def paper_flow( input: PaperInput, @@ -209,17 +309,15 @@ async def paper_flow( cfg: PaperFlowCfg | None = None, _summary_provider: _PaperSummaryProvider | None = None, ) -> PaperFlowResult: - """Build a page-aware chunk set and one cited global summary. + """Build a page-aware chunk set and one cited global summary (deprecated). - Thin compatibility function for the semantic (chunk + summary) shape: it - fetches and parses the document once, chunks it, and runs the deterministic - map-reduce summary pipeline. IDs, source metadata, artifact membership, - lineage, and citation links are minted and validated by the knowledge-layer - constructors; the model returns only summary prose and chunk/page - coordinates through a bounded seam. + Deprecated thin wrapper for the source-first chunk/summary shape. It emits a + ``DeprecationWarning`` and delegates to + ``PaperFlow(PaperFlowCfg(...)).build(input)``, which now owns the semantic + pipeline. The wrapper is grandfathered by the operation-naming contract; + prefer the config-bound entry point in new code:: - ``PaperFlow(cfg).build(input)`` is the config-bound entry point for the - structure-tree shape; the chunk/summary shape is not wired through it yet. + result = await PaperFlow(PaperFlowCfg(...)).build(input) Args: input: Typed paper source. V1 requires a PDF-backed input. @@ -236,33 +334,14 @@ async def paper_flow( not meet the configured source-coverage policy. NotImplementedError: If a DOI input has no exact open PDF resolver. """ - cfg = cfg or PaperFlowCfg() - source, parsed = await _open_source(input, output_dir=cfg.output_dir) - parsed_chunks = chunk_parsed_document( - parsed, - config=SentenceSplitterConfig( - chunk_size=cfg.chunk_size, - chunk_overlap=cfg.chunk_overlap, - ), - ) - producer = PaperChunkingConfig( - splitter_version=version("llama-index-core"), - chunk_size=cfg.chunk_size, - chunk_overlap=cfg.chunk_overlap, - ) - chunk_set = PaperChunkSet.from_parsed_chunks( - source, - _adapt_chunks(parsed_chunks, source), - producer=producer, - ) - provider = _summary_provider or _AgentsPaperSummaryProvider() - draft = await provider.summarize(source, chunk_set, cfg=cfg) - summary = _build_summary(chunk_set, draft, cfg) - return PaperFlowResult( - source_revision=source, - chunk_set=chunk_set, - global_summary=summary, + warnings.warn( + "paper_flow() is deprecated; use " + "PaperFlow(PaperFlowCfg(...)).build(input) instead.", + DeprecationWarning, + stacklevel=2, ) + flow = PaperFlow(cfg or PaperFlowCfg(), _summary_provider=_summary_provider) + return await flow.build(input) async def _open_source( diff --git a/scripts/verify_pdf_rag_e2e.py b/scripts/verify_pdf_rag_e2e.py index 8eeb54c..d45ad87 100644 --- a/scripts/verify_pdf_rag_e2e.py +++ b/scripts/verify_pdf_rag_e2e.py @@ -11,7 +11,7 @@ from quantmind.configs import PaperFlowCfg from quantmind.configs.paper import ArxivIdentifier -from quantmind.flows import paper_flow +from quantmind.flows import PaperFlow from quantmind.knowledge import ( PaperArtifactKind, PaperChunk, @@ -52,9 +52,8 @@ async def _search_and_resolve( async def _run_vertical_slice() -> dict[str, Any]: with tempfile.TemporaryDirectory(prefix="quantmind-paper-v1-") as directory: root = Path(directory) - result = await paper_flow( - ArxivIdentifier(id=_ARXIV_ID), - cfg=PaperFlowCfg( + flow = PaperFlow( + PaperFlowCfg( model="gpt-4o-mini", output_dir=str(root / "assets"), timeout_seconds=240, @@ -63,8 +62,9 @@ async def _run_vertical_slice() -> dict[str, Any]: max_summary_output_tokens=4_096, min_summary_citations=3, min_summary_pages=2, - ), + ) ) + result = await flow.build(ArxivIdentifier(id=_ARXIV_ID)) database = root / "library.db" library = await LocalKnowledgeLibrary.open( database, diff --git a/tests/flows/test_paper.py b/tests/flows/test_paper.py index 8a48ecc..789453b 100644 --- a/tests/flows/test_paper.py +++ b/tests/flows/test_paper.py @@ -31,6 +31,7 @@ _validate_research_draft, ) from quantmind.flows.paper import ( + PaperFlow, UnsupportedContentTypeError, _build_summary, _fetch_paper_source, @@ -91,11 +92,10 @@ async def test_local_pdf_builds_chunks_before_summary(self) -> None: provider = _FakeSummaryProvider() cfg = PaperFlowCfg(chunk_size=256, chunk_overlap=32) - result = await paper_flow( - LocalFilePath(path=_FIXTURE), - cfg=cfg, + result = await PaperFlow( + cfg, _summary_provider=provider, - ) + ).build(LocalFilePath(path=_FIXTURE)) self.assertEqual(len(provider.calls), 1) source_seen, chunk_set_seen, _ = provider.calls[0] @@ -139,11 +139,10 @@ async def test_exact_arxiv_source_facts_are_code_owned(self) -> None: "quantmind.flows.paper.fetch_arxiv", new=AsyncMock(return_value=raw), ): - result = await paper_flow( - ArxivIdentifier(id="1706.03762v7"), - cfg=PaperFlowCfg(chunk_size=256, chunk_overlap=32), + result = await PaperFlow( + PaperFlowCfg(chunk_size=256, chunk_overlap=32), _summary_provider=_FakeSummaryProvider(), - ) + ).build(ArxivIdentifier(id="1706.03762v7")) self.assertEqual(result.source_revision.arxiv_id, "1706.03762v7") self.assertEqual(result.source_revision.source.kind, "arxiv") @@ -160,24 +159,21 @@ async def test_summary_failure_prevents_flow_success(self) -> None: provider = _FakeSummaryProvider(fail=RuntimeError("summary failed")) with self.assertRaisesRegex(RuntimeError, "summary failed"): - await paper_flow( - LocalFilePath(path=_FIXTURE), - cfg=PaperFlowCfg(chunk_size=256, chunk_overlap=32), + await PaperFlow( + PaperFlowCfg(chunk_size=256, chunk_overlap=32), _summary_provider=provider, - ) + ).build(LocalFilePath(path=_FIXTURE)) async def test_same_pdf_and_configs_have_idempotent_ids(self) -> None: cfg = PaperFlowCfg(chunk_size=256, chunk_overlap=32) - first = await paper_flow( - LocalFilePath(path=_FIXTURE), - cfg=cfg, + first = await PaperFlow( + cfg, _summary_provider=_FakeSummaryProvider(), - ) - second = await paper_flow( - LocalFilePath(path=_FIXTURE), - cfg=cfg, + ).build(LocalFilePath(path=_FIXTURE)) + second = await PaperFlow( + cfg, _summary_provider=_FakeSummaryProvider(), - ) + ).build(LocalFilePath(path=_FIXTURE)) self.assertEqual(first.source_revision.id, second.source_revision.id) self.assertEqual(first.chunk_set.id, second.chunk_set.id) @@ -188,6 +184,41 @@ async def test_same_pdf_and_configs_have_idempotent_ids(self) -> None: ) +class PaperFlowDeprecationTests(unittest.IsolatedAsyncioTestCase): + async def test_paper_flow_warns_and_delegates(self) -> None: + # The grandfathered ``paper_flow`` wrapper must emit a + # DeprecationWarning and delegate to ``PaperFlow.build`` for the same + # source-first result, forwarding the summary-provider seam. + provider = _FakeSummaryProvider() + + with self.assertWarns(DeprecationWarning): + result = await paper_flow( + LocalFilePath(path=_FIXTURE), + cfg=PaperFlowCfg(chunk_size=256, chunk_overlap=32), + _summary_provider=provider, + ) + + self.assertEqual(len(provider.calls), 1) + _, chunk_set_seen, _ = provider.calls[0] + self.assertIs(chunk_set_seen, result.chunk_set) + self.assertIs(result.source_revision, provider.calls[0][0]) + self.assertTrue(result.global_summary.summary) + self.assertEqual(len(result.source_revision.parsed.pages), 4) + + async def test_paper_flow_defaults_cfg_when_omitted(self) -> None: + # Omitting cfg still delegates under a default ``PaperFlowCfg``. + provider = _FakeSummaryProvider() + + with self.assertWarns(DeprecationWarning): + result = await paper_flow( + LocalFilePath(path=_FIXTURE), + _summary_provider=provider, + ) + + self.assertEqual(len(provider.calls), 1) + self.assertTrue(result.chunk_set.chunks) + + class SourceDispatchTests(unittest.IsolatedAsyncioTestCase): async def test_http_pdf_preserves_resolved_url_and_fetch_time(self) -> None: raw = Fetched( diff --git a/tests/flows/test_structure.py b/tests/flows/test_structure.py index 9dc9173..e28867a 100644 --- a/tests/flows/test_structure.py +++ b/tests/flows/test_structure.py @@ -9,7 +9,7 @@ from agents import ModelSettings from openai import BadRequestError -from quantmind.configs import PaperFlowCfg, PaperStructureCfg +from quantmind.configs import BaseFlowCfg, PaperStructureCfg from quantmind.configs.paper import LocalFilePath from quantmind.flows import PaperFlow, PaperStructureError from quantmind.flows.paper._structure import ( @@ -147,10 +147,11 @@ async def test_build_propagates_structure_error(self) -> None: self.assertEqual(parse_spy.await_count, 1) self.assertEqual(provider.calls, 1) - async def test_non_structure_cfg_raises_not_implemented(self) -> None: - # A non-PaperStructureCfg selects an unwired shape: build must reject - # it by cfg type, before any fetch or parse. - flow = PaperFlow(PaperFlowCfg()) + async def test_unwired_cfg_type_raises_not_implemented(self) -> None: + # A cfg type that is neither PaperStructureCfg nor PaperFlowCfg selects + # an unwired shape: build must reject it by cfg type, before any fetch + # or parse. + flow = PaperFlow(BaseFlowCfg()) with patch( "quantmind.flows.paper.parse_pdf", new=AsyncMock(side_effect=AssertionError("must not parse")), From c7ede03ed055b10ae56a7675c71eafbdea85db9f Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Fri, 24 Jul 2026 00:06:00 +0800 Subject: [PATCH 2/2] refactor(flows): rename semantic cfg/result and remove the paper_flow wrapper Per PR #140 owner feedback, this replaces the deprecation with an outright removal and clarifies the semantic naming. - Rename `PaperFlowCfg` -> `PaperSemanticCfg` and `PaperFlowResult` -> `PaperSemanticResult` across configs, knowledge, flows, library, magic surfaces, tests, examples, docs, and contexts. `PaperFlowCfg` read like a parent of `PaperStructureCfg`, which it is not; the result is renamed for coherence now that no `paper_flow` function survives to anchor the old name. - Remove the `paper_flow` wrapper entirely (owner overruled the deprecation): drop the function, its `__all__` export, the `warnings` import, and the deprecation tests. `PaperFlow(cfg).build` is now the single entry point for both paper shapes. Update the naming contract so it no longer grandfathers a deleted symbol, and sweep the remaining prose references (README, AGENTS, docs, contexts, labels). - Point the magic examples and tests at `collect_news`, the only remaining public flow with the `(input, *, cfg)` signature that magic introspects, since paper no longer exposes such a function. - Bump stale model ids touched by this change to `gpt-5.6-luna` (the `PaperSemanticCfg` default, the paper example, the pdf-rag e2e slice, the magic resolver default, and the surrounding docs), and rename the e2e paths-filter key `paper_flow` -> `paper_semantic`. - Document that embedding is a downstream (library) concern, not a build concern: the `PaperSemanticCfg`/module docstrings note that vectors are minted at `LocalKnowledgeLibrary.open(..., embedding_model=...)` (examples use `text-embedding-3-small`), so no embedding field is added to the cfg. - Add a real-run snapshot to the module docstring from an actual arXiv `1706.03762v7` build under `gpt-5.6-luna`: the structure shape yields 17 nodes (13 leaves); the semantic shape chunks to 15 pages / 33 chunks, but the cited-summary step's verbatim-quote check rejected every sampled model's paraphrase this run, so no summary line is shown. Co-Authored-By: Claude Fable 5 --- .github/workflows/e2e.yml | 8 +- AGENTS.md | 2 +- README.md | 49 ++++---- contexts/design/flow/paper.md | 16 +-- contexts/design/knowledge/paper.md | 10 +- contexts/design/library/local.md | 6 +- contexts/design/mind/retrieval.md | 2 +- contexts/design/operations/naming.md | 8 +- contexts/design/preprocess/pdf.md | 2 +- contexts/design/rag/document.md | 2 +- contexts/design/utils/usage.md | 4 +- contexts/dev/labels.md | 2 +- docs/README.md | 8 +- docs/library.md | 6 +- examples/flows/paper.py | 6 +- examples/library/README.md | 2 +- quantmind/configs/__init__.py | 4 +- quantmind/configs/paper.py | 17 ++- quantmind/flows/__init__.py | 11 +- quantmind/flows/_paper_summary.py | 12 +- quantmind/flows/paper/__init__.py | 106 ++++++------------ quantmind/knowledge/__init__.py | 4 +- quantmind/knowledge/paper.py | 4 +- .../library/_internal/retrieval_targets.py | 4 +- quantmind/library/_internal/sqlite_store.py | 12 +- quantmind/library/local.py | 6 +- quantmind/magic.py | 4 +- scripts/verify_pdf_rag_e2e.py | 6 +- tests/configs/test_base.py | 4 +- tests/configs/test_paper.py | 12 +- tests/flows/test_batch.py | 4 +- tests/flows/test_paper.py | 60 ++-------- tests/flows/test_runner.py | 10 +- tests/flows/test_structure.py | 2 +- tests/knowledge/test_base.py | 14 +-- tests/knowledge/test_paper.py | 6 +- tests/paper_helpers.py | 8 +- tests/test_magic.py | 85 ++++++++------ 38 files changed, 243 insertions(+), 285 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e7a98a3..e8c8c24 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest outputs: news: ${{ steps.filter.outputs.news }} - paper_flow: ${{ steps.filter.outputs.paper_flow }} + paper_semantic: ${{ steps.filter.outputs.paper_semantic }} structure: ${{ steps.filter.outputs.structure }} steps: @@ -73,12 +73,12 @@ jobs: - 'quantmind/preprocess/format/html.py' - 'quantmind/preprocess/fetch/**' - 'pyproject.toml' - paper_flow: + paper_semantic: - '.github/workflows/e2e.yml' - 'scripts/verify_pdf_rag_e2e.py' - 'quantmind/configs/paper.py' - 'quantmind/flows/_paper_summary.py' - - 'quantmind/flows/paper.py' + - 'quantmind/flows/paper/**' - 'quantmind/knowledge/paper.py' - 'quantmind/library/**' - 'quantmind/preprocess/fetch/arxiv.py' @@ -132,7 +132,7 @@ jobs: paper-flow: needs: changes - if: github.event_name != 'pull_request' || needs.changes.outputs.paper_flow == 'true' + if: github.event_name != 'pull_request' || needs.changes.outputs.paper_semantic == 'true' runs-on: ubuntu-latest timeout-minutes: 10 env: diff --git a/AGENTS.md b/AGENTS.md index f21ec5b..e940329 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ handoff all come from `openai-agents`. | `quantmind/configs/` | Operation cfg + typed input models or unions (`BaseFlowCfg`, `NewsWindow`, `PaperInput`) — depends only on `knowledge` | | `quantmind/preprocess/` | Deterministic fetch / format / clean / time utilities — depends only on `utils` | | `quantmind/rag/` | Opinionated LlamaIndex document chunking and retrieval — depends only on `preprocess` | -| `quantmind/flows/` | Apex layer: public library operations (`paper_flow`, `collect_news`, `batch_run`) | +| `quantmind/flows/` | Apex layer: public library operations (`PaperFlow`, `collect_news`, `batch_run`) | | `quantmind/magic.py` | `resolve_magic_input`: natural language → `(input, cfg)` | | `quantmind/mind/` | Pure-agentic reasoning layer — memory + agentic (reasoning-based) retrieval where an LLM decides; mechanical retrieval (similarity / BM25) lives in `rag` / `library` | | `quantmind/utils/` | Logger only — keep it that way | diff --git a/README.md b/README.md index e299554..7e72749 100644 --- a/README.md +++ b/README.md @@ -161,16 +161,15 @@ Component-specific guides and architecture notes live under [`docs/`](docs/). ```python import asyncio -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.configs.paper import ArxivIdentifier -from quantmind.flows import paper_flow +from quantmind.flows import PaperFlow async def main() -> None: - result = await paper_flow( - ArxivIdentifier(id="1706.03762v7"), - cfg=PaperFlowCfg(model="gpt-4o-mini"), - ) + result = await PaperFlow( + PaperSemanticCfg(model="gpt-5.6-luna"), + ).build(ArxivIdentifier(id="1706.03762v7")) print(result.global_summary.summary) print(result.source_revision.id, result.chunk_set.id) @@ -178,24 +177,34 @@ async def main() -> None: asyncio.run(main()) ``` +`PaperFlow` binds an immutable `PaperSemanticCfg` once; the cfg **type** selects +the shape (`PaperStructureCfg` → `PaperStructureTree`, `PaperSemanticCfg` → +`PaperSemanticResult`). + #### Fan out a batch with `batch_run` ```python import asyncio +from datetime import datetime, timedelta, timezone -from quantmind.configs import PaperFlowCfg -from quantmind.configs.paper import ArxivIdentifier -from quantmind.flows import batch_run, paper_flow +from quantmind.configs import NewsCollectionCfg, NewsWindow +from quantmind.flows import batch_run, collect_news async def main() -> None: - inputs = [ArxivIdentifier(id=aid) for aid in ( - "2401.12345", "2401.12346", "2401.12347", - )] + end = datetime.now(timezone.utc) + windows = [ + NewsWindow( + source="pr-newswire", + start=end - timedelta(days=day + 1), + end=end - timedelta(days=day), + ) + for day in range(3) + ] result = await batch_run( - paper_flow, - inputs, - cfg=PaperFlowCfg(model="gpt-4o-mini"), + collect_news, + windows, + cfg=NewsCollectionCfg(retain_raw_html=False), concurrency=3, on_error="skip", on_progress=lambda done, total: print(f"{done}/{total}"), @@ -211,17 +220,17 @@ asyncio.run(main()) ```python import asyncio -from quantmind.flows import paper_flow +from quantmind.flows import collect_news from quantmind.magic import resolve_magic_input async def main() -> None: inp, cfg = await resolve_magic_input( - "Pull arXiv 2401.12345 about cross-sectional momentum; use gpt-4o-mini.", - target_flow=paper_flow, + "Collect the last day of PR Newswire company news.", + target_flow=collect_news, ) - result = await paper_flow(inp, cfg=cfg) - print(result.global_summary.summary) + batch = await collect_news(inp, cfg=cfg) + print(f"documents={batch.success_count} complete={batch.complete}") asyncio.run(main()) diff --git a/contexts/design/flow/paper.md b/contexts/design/flow/paper.md index c90afe4..3f28289 100644 --- a/contexts/design/flow/paper.md +++ b/contexts/design/flow/paper.md @@ -3,8 +3,8 @@ ## Quick Summary - **Purpose**: Define how one exact PDF revision becomes a durable page-aware chunk set and a cited global summary. -- **Read when**: Changing the paper build (`PaperFlow(PaperFlowCfg).build` or the deprecated `paper_flow` wrapper), paper inputs, summarization limits, citation validation, or the end-to-end paper verifier. -- **Status**: Implemented by the config-bound `quantmind.flows.PaperFlow` — a `PaperFlowCfg` selects this source-first shape — for PDF-backed arXiv, HTTP, and local inputs. `paper_flow` remains as a deprecated thin wrapper that delegates to it. +- **Read when**: Changing the paper build (`PaperFlow(PaperSemanticCfg).build`), paper inputs, summarization limits, citation validation, or the end-to-end paper verifier. +- **Status**: Implemented by the config-bound `quantmind.flows.PaperFlow` — a `PaperSemanticCfg` selects this source-first shape — for PDF-backed arXiv, HTTP, and local inputs. - **Core rule**: Preserve and validate the source revision and chunk set before any model-generated summary is accepted. - **Canonical models**: [Paper source and artifact design](../knowledge/paper.md). @@ -23,10 +23,10 @@ ## Contract -`PaperFlow(PaperFlowCfg(...)).build(input)` returns one validated `PaperFlowResult`. `PaperFlow` binds the immutable `PaperFlowCfg` once and its cfg **type** selects this source-first shape, so `batch_run(flow.build, inputs)` runs a batch under one setting. The deprecated `paper_flow(input, *, cfg)` wrapper emits a `DeprecationWarning` and delegates to the same build. +`PaperFlow(PaperSemanticCfg(...)).build(input)` returns one validated `PaperSemanticResult`. `PaperFlow` binds the immutable `PaperSemanticCfg` once and its cfg **type** selects this source-first shape, so `batch_run(flow.build, inputs)` runs a batch under one setting. It is the single entry point for the semantic shape; there is no standalone `*_flow` function. ```text -PaperFlowResult +PaperSemanticResult ├── source_revision: PaperSourceRevision ├── chunk_set: PaperChunkSet └── global_summary: PaperGlobalSummary @@ -57,7 +57,7 @@ The operation has a strict order: 7. Fan out one bounded research agent per group (bounded concurrency), each returning typed findings for its own range only. 8. Run one reducer agent over the collected findings to synthesize the summary draft. 9. Resolve model-returned chunk/page coordinates into canonical citations in code. -10. Build and validate `PaperGlobalSummary` and the cross-artifact `PaperFlowResult`. +10. Build and validate `PaperGlobalSummary` and the cross-artifact `PaperSemanticResult`. Steps 3, 5, 9, and 10 mint no IDs in the flow: the flow calls the knowledge-layer smart constructors `PaperSourceRevision.from_parsed`, `PaperChunkSet.from_parsed_chunks`, and `PaperGlobalSummary.from_draft`, which own every ID, content/producer hash, and citation resolution. The flow only fetches, parses, reads asset bytes, and maps those path-based artifacts into knowledge-native inputs. See [orchestration principles](../operations/orchestration.md). @@ -103,7 +103,7 @@ Code accepts the draft only when: Summarization is a deterministic map-reduce, not an autonomous coordinator. Code — not a model — decides the decomposition: it tiles the chunk set into `summary_research_group_size` groups and runs one research agent per group, so complete chunk coverage is guaranteed by construction and never needs to be reconciled afterward. -`PaperFlowCfg` exposes only structural bounds: +`PaperSemanticCfg` exposes only structural bounds: - `summary_research_group_size` sets how many consecutive chunks each research agent receives; - `summary_concurrency` bounds simultaneous research-agent runs; @@ -124,7 +124,7 @@ Bounding is delegated to the Agents SDK (per-agent `max_tokens`, structured `out - A research finding that cites outside its assigned group is rejected in code; a reducer timeout raises `PaperSummaryError`. - Any canonical identity, content hash, membership, lineage, or cross-artifact mismatch fails Pydantic validation. -No failure is converted into a partially valid `PaperFlowResult`. Callers may retry with the same source and producer settings; stable IDs make successful repeated runs idempotent. +No failure is converted into a partially valid `PaperSemanticResult`. Callers may retry with the same source and producer settings; stable IDs make successful repeated runs idempotent. ## Persistence and Retrieval @@ -142,7 +142,7 @@ The bounded live slice is: python scripts/verify_pdf_rag_e2e.py ``` -It fetches exact arXiv revision `1706.03762v7`, parses at least 15 physical pages, creates multiple page-aware chunks, delegates complete coverage to bounded `gpt-4o-mini` research subagents, synthesizes a cited global summary, persists with `text-embedding-3-small`, closes and reopens the database, runs summary and chunk searches, resolves every returned locator, and prints useful summary, page, citation, and score diagnostics. The dedicated `paper-flow` job in `.github/workflows/e2e.yml` owns this non-required public-network check. +It fetches exact arXiv revision `1706.03762v7`, parses at least 15 physical pages, creates multiple page-aware chunks, delegates complete coverage to bounded `gpt-5.6-luna` research subagents, synthesizes a cited global summary, persists with `text-embedding-3-small`, closes and reopens the database, runs summary and chunk searches, resolves every returned locator, and prints useful summary, page, citation, and score diagnostics. The dedicated `paper-flow` job in `.github/workflows/e2e.yml` owns this non-required public-network check. ## Out of Scope diff --git a/contexts/design/knowledge/paper.md b/contexts/design/knowledge/paper.md index 7404ccc..7bc7643 100644 --- a/contexts/design/knowledge/paper.md +++ b/contexts/design/knowledge/paper.md @@ -28,7 +28,7 @@ Source-first paper handling separates four layers: | Semantic artifact | `PaperGlobalSummary` | `PaperCitation` | Store one independently versioned model summary with resolvable chunk/page evidence. | | Structural artifact | `PaperStructureTree` | `TreeNode`, `Citation` | Store one independently versioned natural-section hierarchy over exact source pages. | -`PaperFlowResult` validates one compatible source, chunk set, and summary combination. It is a transfer result, not a fourth stored artifact. +`PaperSemanticResult` validates one compatible source, chunk set, and summary combination. It is a transfer result, not a fourth stored artifact. All models are frozen Pydantic values with `extra="forbid"`. Canonical values contain no embedding vectors, provider node objects, or storage handles. @@ -53,7 +53,7 @@ These identities make an identical run idempotent. They also keep a changed spli The source's canonical JSON excludes blobs. This keeps canonical hashes stable and reviewable while allowing SQLite to store exact bytes in a normalized linked table. Rehydration checks both directions: stored blob bytes must match their table hashes, and table asset metadata must match the canonical source manifest. -Every chunk span is also checked against that manifest: its page must exist, its character range must fit the page text, and every visual asset ID must resolve to a screenshot or image from the same page. This check runs for a complete `PaperFlowResult` and when a stored chunk set is rehydrated independently. +Every chunk span is also checked against that manifest: its page must exist, its character range must fit the page text, and every visual asset ID must resolve to a screenshot or image from the same page. This check runs for a complete `PaperSemanticResult` and when a stored chunk set is rehydrated independently. ## Artifact Versioning @@ -69,13 +69,13 @@ Every chunk span is also checked against that manifest: its page must exist, its - per-agent output limit; - research group size. -Changing any producer field creates a distinct artifact ID. Multiple chunk sets and summaries may coexist for one source revision. Loading a complete `PaperFlowResult` without explicit artifact IDs is allowed only when one unambiguous linked pair exists. +Changing any producer field creates a distinct artifact ID. Multiple chunk sets and summaries may coexist for one source revision. Loading a complete `PaperSemanticResult` without explicit artifact IDs is allowed only when one unambiguous linked pair exists. `PaperStructureTree.producer` records model and prompt identity, the instructions hash, the bounded physical-page text input policy, and tree/output bounds. It deliberately records no splitter or chunk-set identity. Rechunking an unchanged source therefore does not create a different structure tree. ## Citation and Lineage Integrity -A `PaperCitation` identifies the exact chunk set, chunk, page, and optional verbatim quote. `PaperFlowResult` rejects citations to missing chunks, pages outside the cited chunk spans, or quotes absent from chunk text. +A `PaperCitation` identifies the exact chunk set, chunk, page, and optional verbatim quote. `PaperSemanticResult` rejects citations to missing chunks, pages outside the cited chunk spans, or quotes absent from chunk text. `PaperGlobalSummary.derived_from` contains `ArtifactLocator` values. At least one locator must point to its producer's exact input chunk set, with the same source revision and no member ID. The library stores this relationship explicitly so lineage can be checked independently from the summary JSON. @@ -95,6 +95,6 @@ Canonical paper models do not implement `embedding_text()` and do not select ret ## Compatibility Boundary -`LegacyPaper` retains the pre-V1 `TreeKnowledge` shape only so existing version-2 databases and the bundled legacy example can be opened. It is not exported as `Paper`, is not produced by `paper_flow`, and is not part of the V1 paper contract. +`LegacyPaper` retains the pre-V1 `TreeKnowledge` shape only so existing version-2 databases and the bundled legacy example can be opened. It is not exported as `Paper`, is not produced by `PaperFlow`, and is not part of the V1 paper contract. There is no nested `PaperTree` on the V1 result. `PaperStructureTree` is an independently versioned paper-artifact binding of the shared `StructureTree` base, derived directly from one exact source revision and independent of every chunk-set version; see [Build and retrieve from a page-preserving structure tree](../mind/retrieval.md). diff --git a/contexts/design/library/local.md b/contexts/design/library/local.md index f791682..e652a1f 100644 --- a/contexts/design/library/local.md +++ b/contexts/design/library/local.md @@ -30,7 +30,7 @@ |---|---| | `open()` | Open or migrate a SQLite library without network I/O. | | `put()` | Store one conventional `BaseKnowledge` item and its required projections. | -| `put_paper()` | Store one `PaperFlowResult`, including exact source assets, two artifacts, lineage, and required projections. | +| `put_paper()` | Store one `PaperSemanticResult`, including exact source assets, two artifacts, lineage, and required projections. | | `put_paper_structure_tree()` | Atomically store one exact source revision and validated structure tree without projections or embeddings. | | `get()` | Rehydrate one conventional knowledge item. | | `get_paper()` | Rehydrate one unambiguous source/chunk-set/summary result, or use explicit artifact IDs when versions coexist. | @@ -50,7 +50,7 @@ The public types are `LocalKnowledgeLibrary`, `SemanticQuery`, `SemanticHit`, an | `quantmind.flows` | Produce validated results and leave persistence explicit. | | Caller | Choose database and temporary parser-artifact locations and manage application lifecycle. | -For conventional `BaseKnowledge`, source references remain pointers and the caller retains external raw files. For `PaperFlowResult`, the exact source PDF, screenshots, and extracted images are part of the durable source revision and are copied into the library transaction. +For conventional `BaseKnowledge`, source references remain pointers and the caller retains external raw files. For `PaperSemanticResult`, the exact source PDF, screenshots, and extracted images are part of the durable source revision and are copied into the library transaction. ## Canonical Storage @@ -75,7 +75,7 @@ There is no single opaque paper JSON blob and no canonical vector field. Aggrega ## Paper Transaction -`put_paper()` first validates the complete `PaperFlowResult`, computes its source and artifact canonical forms, determines affected projections, and obtains every required embedding. Only then does it begin one `BEGIN IMMEDIATE` transaction. +`put_paper()` first validates the complete `PaperSemanticResult`, computes its source and artifact canonical forms, determines affected projections, and obtains every required embedding. Only then does it begin one `BEGIN IMMEDIATE` transaction. The transaction writes or reuses the source, asset blobs, artifacts, members, lineage, and all required projections. Any constraint, integrity, or write failure rolls back the transaction. An embedding-provider failure occurs before the transaction and therefore leaves no partial source or artifact rows. diff --git a/contexts/design/mind/retrieval.md b/contexts/design/mind/retrieval.md index dfb04b9..7da1f02 100644 --- a/contexts/design/mind/retrieval.md +++ b/contexts/design/mind/retrieval.md @@ -119,7 +119,7 @@ trees = await batch_run(tree_flow.build, inputs) Rules: - **Bind the config, not the input.** `PaperFlow(cfg)` stores the immutable cfg; `build(input)` takes only the operand. A batch therefore runs every input under one unified setting, and a config can never drift mid-run — a reproducibility requirement, not ergonomics. -- **The cfg type picks the shape.** `PaperStructureCfg` → `PaperStructureTree` (implemented now). A later `PaperCardCfg` → the chunk/summary shape reached through the same `build` seam; the existing `paper_flow(input, *, cfg)` function stays as a thin compatibility wrapper for that semantic shape until it lands. +- **The cfg type picks the shape.** `PaperStructureCfg` → `PaperStructureTree` and `PaperSemanticCfg` → `PaperSemanticResult`, both reached through the same `build` seam. A later `PaperCardCfg` would add a further shape through that seam. - **Pure processing.** `build` fetches, parses, and structures, producing the complete self-contained artifact. It binds **no** library, persists nothing, and does not retrieve. - A caller who wants only the parsed source uses `quantmind.preprocess` directly; "parse only" is a component seam, not a public pipeline. diff --git a/contexts/design/operations/naming.md b/contexts/design/operations/naming.md index f4d60bd..fea6f3f 100644 --- a/contexts/design/operations/naming.md +++ b/contexts/design/operations/naming.md @@ -4,7 +4,7 @@ - **Purpose**: Define clear names for public QuantMind callables and their input, config, and result types. - **Read when**: Adding or renaming a public function, service, operation type, or pipeline. -- **Status**: Use these rules for new names. Keep the existing `paper_flow` name until a separate change explains how callers will migrate. +- **Status**: Use these rules for new names. The legacy paper extraction function has been removed; the paper semantic build is now `PaperFlow(PaperSemanticCfg(...)).build(input)`, so no `*_flow` functions remain. - **Core rule**: Name a function or service method with a verb that says what it does. Use `pipeline` only when one callable combines several public operations. Do not use `flow` as a **verb**; `flow` as a **noun** naming a finished pipeline collection (`PaperFlow`) is allowed. ## Contents @@ -64,15 +64,15 @@ A domain flow binds an immutable `cfg` at construction and applies it to each in - Input types describe what the caller supplies, such as `NewsWindow` or `PaperInput`. - Config types name the domain and stage, such as `NewsCollectionCfg`, `PaperStructureCfg`, or `RetrievalCfg`. -- Result types describe returned data, such as `NewsBatch`, `PaperFlowResult`, or `PageIndex`. +- Result types describe returned data, such as `NewsBatch`, `PaperSemanticResult`, or `PageIndex`. - Keep provider names out of public function names unless callers are choosing provider-specific behavior. ## Current API - `collect_news` is a collection operation and follows these naming rules. - `batch_run` is a generic batch helper, not a news or paper operation. -- `paper_flow` is an existing extraction **function** with an old name, now a deprecated thin wrapper that emits a `DeprecationWarning` and delegates to `PaperFlow(PaperFlowCfg(...)).build(input)`. Do not copy the `*_flow` function pattern for new operations. -- `PaperFlow(cfg)` is the config-bound paper flow; `build(input)` produces a knowledge artifact whose shape is chosen by the cfg *type* (`PaperStructureCfg` → `PaperStructureTree`, `PaperFlowCfg` → `PaperFlowResult`). `Flow` as a noun here is deliberate and allowed. +- The `*_flow` function-name pattern is banned for new operations; the former paper extraction function was removed rather than kept as a wrapper. +- `PaperFlow(cfg)` is the config-bound paper flow and the single entry point for every paper shape; `build(input)` produces a knowledge artifact whose shape is chosen by the cfg *type* (`PaperStructureCfg` → `PaperStructureTree`, `PaperSemanticCfg` → `PaperSemanticResult`). `Flow` as a noun here is deliberate and allowed. - `AgenticRetriever(cfg)` is the reasoning-retrieval service in `quantmind.mind`; `retrieve(structure, question)` returns evidence values. It has one behavior — an LLM agent reasons over the structure — so it binds `RetrievalCfg` but does no cfg-type dispatch. Mechanical semantic search is `quantmind.library.search`, a different layer, not a strategy here. The current `quantmind.flows` package continues to contain public operations in this release. Renaming it to `operations` or adding a separate `pipelines` package is outside this page. diff --git a/contexts/design/preprocess/pdf.md b/contexts/design/preprocess/pdf.md index d921d43..be81bdb 100644 --- a/contexts/design/preprocess/pdf.md +++ b/contexts/design/preprocess/pdf.md @@ -30,7 +30,7 @@ The caller chooses an artifact directory. When supplied, parsing renders one PNG Preprocessing ends after producing `ParsedDocument`. It does not chunk, index, rank, or answer a query. - [`quantmind.rag`](../rag/document.md) converts the parsed value into LlamaIndex-backed chunks and page-aware retrieval evidence. -- [`paper_flow`](../flow/paper.md) uses the preserved source pages to build an exact source revision and a canonical chunk set before generating a cited summary. +- The [paper flow](../flow/paper.md) (`PaperFlow(PaperSemanticCfg).build`) uses the preserved source pages to build an exact source revision and a canonical chunk set before generating a cited summary. - `extract_outline_signals()` consumes the same ordered pages to emit deterministic table-of-contents, heading, and printed-page-offset hints for structure-tree construction. Flattened Markdown remains a compatibility view produced from the preserved pages. It is not the primary parsing result. diff --git a/contexts/design/rag/document.md b/contexts/design/rag/document.md index 56fcc95..3874401 100644 --- a/contexts/design/rag/document.md +++ b/contexts/design/rag/document.md @@ -48,7 +48,7 @@ LlamaIndex `Document`, node, retriever, index, and score-wrapper types remain pr ## Paper Flow Boundary -[`paper_flow`](../flow/paper.md) uses `chunk_parsed_document()` as the deterministic split stage. It converts `ParsedChunk` values into canonical `PaperChunk` members only after an exact `PaperSourceRevision` exists. +The [paper flow](../flow/paper.md) (`PaperFlow(PaperSemanticCfg).build`) uses `chunk_parsed_document()` as the deterministic split stage. It converts `ParsedChunk` values into canonical `PaperChunk` members only after an exact `PaperSourceRevision` exists. The conversion replaces parser paths with canonical source asset IDs and validates character spans against page evidence. The resulting `PaperChunkSet` is a durable, independently versioned artifact. `quantmind.rag` itself does not import or construct canonical paper models. diff --git a/contexts/design/utils/usage.md b/contexts/design/utils/usage.md index a0f43b4..58d2ca3 100644 --- a/contexts/design/utils/usage.md +++ b/contexts/design/utils/usage.md @@ -20,11 +20,11 @@ ## Motivation -Every flow LLM call returns a `RunResult` whose `context_wrapper.usage` already carries input/output/total tokens and a `requests` count, but the flow returns only its domain artifact (`PaperStructureTree`, `PaperFlowResult`) and drops the `RunResult`. So today there is no answer to a basic optimization question: for one `PaperFlow(cfg).build(input)` or one `paper_flow(input)`, how many tokens did it burn, how long did it take, and how many model calls did it make. +Every flow LLM call returns a `RunResult` whose `context_wrapper.usage` already carries input/output/total tokens and a `requests` count, but the flow returns only its domain artifact (`PaperStructureTree`, `PaperSemanticResult`) and drops the `RunResult`. So today there is no answer to a basic optimization question: for one `PaperFlow(cfg).build(input)`, how many tokens did it burn, how long did it take, and how many model calls did it make. Two facts make a naive `return result.context_wrapper.usage` insufficient: -- **One flow run is many SDK runs.** `paper_flow`'s summary fans out one research agent per chunk group with `asyncio.gather`, then runs one reducer — `N + 1` separate `Runner.run` calls. A structure build that hits the `json_schema` → `json_object` fallback is two calls. The usage of "one flow run" is a sum across several runs, so it must be aggregated, not read from a single result. +- **One flow run is many SDK runs.** The semantic build's summary fans out one research agent per chunk group with `asyncio.gather`, then runs one reducer — `N + 1` separate `Runner.run` calls. A structure build that hits the `json_schema` → `json_object` fallback is two calls. The usage of "one flow run" is a sum across several runs, so it must be aggregated, not read from a single result. - **Concurrency makes a summed duration lie.** Those `N` researchers run in parallel; summing their wall times over-reports latency several-fold. Real end-to-end time and summed busy time are different numbers, and optimization needs both. Cost is explicitly out of scope: this reports tokens, time, and steps only. Pricing changes per provider and per week; a caller who wants dollars multiplies tokens by their own rate table. See [Non-Enforcement](#layer-boundary-and-non-enforcement). diff --git a/contexts/dev/labels.md b/contexts/dev/labels.md index 2ad61d5..090c445 100644 --- a/contexts/dev/labels.md +++ b/contexts/dev/labels.md @@ -92,7 +92,7 @@ Body source formatting follows the [GitHub writing style](github-writing.md), in | Context framework in [#101](https://github.com/LLMQuant/quant-mind/issues/101) | `type: feature`, `area: contexts` | It added a new routing capability. | | CI/E2E consolidation in [#102](https://github.com/LLMQuant/quant-mind/issues/102) | `type: refactor`, `area: harness`, `impact: live-network` | It restructures repository verification, including live smoke tests. | | This label-system issue and its PR | `type: docs`, `area: contexts`, `area: harness` | The main result is context guidance routed through contributor controls. | -| Fix documented `paper_flow` behavior | `type: bug`, `area: flows` | An existing public operation is broken. | +| Fix documented `PaperFlow.build` behavior | `type: bug`, `area: flows` | An existing public operation is broken. | | Update only the news design document | `type: docs`, `area: preprocess` | The document belongs to the news preprocessing surface, not contexts. | | Add a public news source | `type: feature`, `area: preprocess`, `impact: live-network` | It adds acquisition behavior backed by a real external source. | diff --git a/docs/README.md b/docs/README.md index 04df584..862e00f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,12 +14,12 @@ harness. | Operation | Import | Input and config | Result | Example | Design or guide | |---|---|---|---|---|---| -| Source-first paper flow | `quantmind.flows.PaperFlow` | `PaperFlow(PaperFlowCfg)`; `build()`: `PaperInput` | `PaperFlowResult` | [Persist and search a paper](../examples/flows/paper.py) | [Paper flow design](../contexts/design/flow/paper.md) | +| Source-first paper flow | `quantmind.flows.PaperFlow` | `PaperFlow(PaperSemanticCfg)`; `build()`: `PaperInput` | `PaperSemanticResult` | [Persist and search a paper](../examples/flows/paper.py) | [Paper flow design](../contexts/design/flow/paper.md) | | Paper structure build | `quantmind.flows.PaperFlow` | `PaperFlow(PaperStructureCfg)`; `build()`: `PaperInput` | `PaperStructureTree` (self-contained) | [Build and retrieve](../examples/mind/paper_structure_retrieval.py) | [Structure retrieval design](../contexts/design/mind/retrieval.md) | | Reasoning-based retrieval (agentic) | `quantmind.mind.AgenticRetriever` | `AgenticRetriever(RetrievalCfg)`; `retrieve()`: one `StructureTree` + question (no library) | `list[RetrievalEvidence]` | [Build and retrieve](../examples/mind/paper_structure_retrieval.py) | [Structure retrieval design](../contexts/design/mind/retrieval.md) | | News collection | `quantmind.flows.collect_news` | `NewsWindow`, `NewsCollectionCfg` | `NewsBatch` from `quantmind.preprocess` | [Collect news](../examples/flows/collect_news.py) | [News collection design](../contexts/design/flow/news.md) | | Bounded fan-out | `quantmind.flows.batch_run` | Operation inputs and shared config | `BatchResult` | [README usage](../README.md#-usage-examples) | API docstrings | -| Local semantic search | `quantmind.library.LocalKnowledgeLibrary` | `BaseKnowledge` or `PaperFlowResult`, `SemanticQuery` | `list[SemanticHit]` | [Library example](../examples/library/README.md) | [Library guide](library.md) | +| Local semantic search | `quantmind.library.LocalKnowledgeLibrary` | `BaseKnowledge` or `PaperSemanticResult`, `SemanticQuery` | `list[SemanticHit]` | [Library example](../examples/library/README.md) | [Library guide](library.md) | | Page-aware document RAG | `quantmind.rag.chunk_parsed_document`, `quantmind.rag.retrieve_parsed_document` | `ParsedDocument`, splitter config, and query | `tuple[ParsedDocumentHit, ...]` | [Paper RAG](../examples/rag/paper.py) | [Document RAG design](../contexts/design/rag/document.md) | Import public inputs and configs from `quantmind.configs`, flow operations and @@ -31,7 +31,7 @@ Import result contracts from the canonical layer shown in the catalog. | Source | Source selection | Operation | Live-network component smoke test | |---|---|---|---| | PR Newswire | `NewsWindow(source="pr-newswire", ...)` | `collect_news` | `python scripts/verify_news_e2e.py` | -| arXiv Transformer PDF | `ArxivIdentifier(id="1706.03762v7")` | `PaperFlow(PaperFlowCfg).build`, persistence, reopen, search, and resolution | `python scripts/verify_pdf_rag_e2e.py` | +| arXiv Transformer PDF | `ArxivIdentifier(id="1706.03762v7")` | `PaperFlow(PaperSemanticCfg).build`, persistence, reopen, search, and resolution | `python scripts/verify_pdf_rag_e2e.py` | | Golden paper PDF (structure) | `LocalFilePath(...golden/paper.pdf)` | `PaperFlow.build`, standalone `put`/`open_structure`, and `AgenticRetriever.retrieve` | `python scripts/verify_structure_e2e.py` | The PR Newswire smoke test checks the public RSS feed, a complete preceding @@ -42,7 +42,7 @@ It is not a required merge check, so external PR Newswire availability cannot block unrelated changes. The `paper-flow` job fetches exact arXiv revision `1706.03762v7`, preserves at -least 15 pages, runs bounded `gpt-4o-mini` summarization, persists summary and +least 15 pages, runs bounded `gpt-5.6-luna` summarization, persists summary and chunk projections with `text-embedding-3-small`, reopens the database, searches both artifact kinds, and resolves every hit. It runs daily, manually, and on pull requests that change its dependency paths, and it remains non-required diff --git a/docs/library.md b/docs/library.md index f763fec..ec29aa0 100644 --- a/docs/library.md +++ b/docs/library.md @@ -4,11 +4,11 @@ ## Store a Paper Flow Result -Run `PaperFlow(PaperFlowCfg(...)).build()` first, then explicitly store its complete result: +Run `PaperFlow(PaperSemanticCfg(...)).build()` first, then explicitly store its complete result: ```python result = await PaperFlow( - PaperFlowCfg(model="gpt-4o-mini"), + PaperSemanticCfg(model="gpt-5.6-luna"), ).build(ArxivIdentifier(id="1706.03762v7")) library = await LocalKnowledgeLibrary.open( @@ -98,7 +98,7 @@ The bundled AI-infrastructure scenario contains primary-source-backed `News`, `E python examples/library/semantic_search.py ``` -`LegacyPaper` exists only so older databases and this auditable example remain readable. New paper ingestion uses `PaperFlowResult` and `put_paper()`. +`LegacyPaper` exists only so older databases and this auditable example remain readable. New paper ingestion uses `PaperSemanticResult` and `put_paper()`. Maintainers can regenerate the bundle after changing source data, projection rules, or the storage schema: diff --git a/examples/flows/paper.py b/examples/flows/paper.py index 971e4d9..9839b9a 100644 --- a/examples/flows/paper.py +++ b/examples/flows/paper.py @@ -7,7 +7,7 @@ from dotenv import load_dotenv -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.configs.paper import ArxivIdentifier from quantmind.flows import PaperFlow from quantmind.knowledge import ( @@ -63,8 +63,8 @@ async def main() -> None: workspace = Path(".quantmind") workspace.mkdir(exist_ok=True) flow = PaperFlow( - PaperFlowCfg( - model="gpt-4o-mini", + PaperSemanticCfg( + model="gpt-5.6-luna", output_dir=str(workspace / "attention-assets"), ) ) diff --git a/examples/library/README.md b/examples/library/README.md index f92599e..9f7ae7c 100644 --- a/examples/library/README.md +++ b/examples/library/README.md @@ -2,7 +2,7 @@ This example searches an auditable AI-infrastructure knowledge bundle containing `News`, `Earnings`, and a pre-V1 `LegacyPaper` tree. New paper ingestion uses -`PaperFlowResult` and `LocalKnowledgeLibrary.put_paper()`; the legacy type is +`PaperSemanticResult` and `LocalKnowledgeLibrary.put_paper()`; the legacy type is retained only so this existing bundle remains readable. ## Run diff --git a/quantmind/configs/__init__.py b/quantmind/configs/__init__.py index 3676f6f..9433743 100644 --- a/quantmind/configs/__init__.py +++ b/quantmind/configs/__init__.py @@ -11,7 +11,7 @@ from quantmind.configs.base import BaseFlowCfg, BaseInput from quantmind.configs.earnings import EarningsFlowCfg, EarningsInput from quantmind.configs.news import NewsCollectionCfg, NewsWindow -from quantmind.configs.paper import PaperFlowCfg, PaperInput +from quantmind.configs.paper import PaperInput, PaperSemanticCfg from quantmind.configs.retrieval import RetrievalCfg from quantmind.configs.structure import PaperStructureCfg @@ -22,7 +22,7 @@ "EarningsInput", "NewsCollectionCfg", "NewsWindow", - "PaperFlowCfg", + "PaperSemanticCfg", "PaperInput", "PaperStructureCfg", "RetrievalCfg", diff --git a/quantmind/configs/paper.py b/quantmind/configs/paper.py index ed6e842..193dee6 100644 --- a/quantmind/configs/paper.py +++ b/quantmind/configs/paper.py @@ -54,8 +54,12 @@ class DoiIdentifier(BaseInput): ] -class PaperFlowCfg(BaseFlowCfg): - """Chunking and summarization controls for ``paper_flow``. +class PaperSemanticCfg(BaseFlowCfg): + """Chunking and summarization controls for the semantic paper build. + + Selects the source-first chunk/summary shape when bound to + ``PaperFlow``: ``PaperFlow(PaperSemanticCfg(...)).build(input)`` returns a + ``PaperSemanticResult``. Summarization is a deterministic map-reduce: code tiles the chunk set into ``summary_research_group_size`` groups (so coverage is guaranteed by @@ -63,9 +67,14 @@ class PaperFlowCfg(BaseFlowCfg): ``summary_concurrency`` parallelism, then runs one reducer. Per-agent output is bounded by ``max_summary_output_tokens`` through ``ModelSettings``; there is no hand-rolled token accountant. + + This cfg carries no embedding setting: the build produces only source, chunk, + and summary text. Embeddings are minted downstream at persistence time, using + the ``embedding_model`` bound at ``LocalKnowledgeLibrary.open(...)`` (the + examples use ``text-embedding-3-small``). """ - model: str = "gpt-4o-mini" + model: str = "gpt-5.6-luna" max_turns: int = Field(default=16, ge=1) chunk_size: int = Field(default=512, gt=0) chunk_overlap: int = Field(default=64, ge=0) @@ -78,7 +87,7 @@ class PaperFlowCfg(BaseFlowCfg): min_summary_pages: int = Field(default=2, ge=1) @model_validator(mode="after") - def _validate_paper_bounds(self) -> "PaperFlowCfg": + def _validate_paper_bounds(self) -> "PaperSemanticCfg": if self.chunk_overlap >= self.chunk_size: raise ValueError("chunk_overlap must be smaller than chunk_size") if self.min_summary_pages > self.min_summary_citations: diff --git a/quantmind/flows/__init__.py b/quantmind/flows/__init__.py index bfe0dbb..71b75da 100644 --- a/quantmind/flows/__init__.py +++ b/quantmind/flows/__init__.py @@ -1,16 +1,15 @@ """Apex layer — composes configs / knowledge / preprocess on the SDK. -Semantic flows such as ``paper_flow`` return ``quantmind.knowledge`` values. +The config-bound ``PaperFlow`` returns ``quantmind.knowledge`` values. Deterministic operations such as ``collect_news`` return source-faithful ``quantmind.preprocess`` values. Cross-flow utilities live alongside: - ``PaperFlow`` is the config-bound paper flow: ``PaperFlow(cfg)`` binds an immutable build config once and ``build(input)`` applies it per input, dispatching on the cfg **type** (``PaperStructureCfg`` → a self-contained - ``PaperStructureTree``; ``PaperFlowCfg`` → a source-first ``PaperFlowResult``). - ``batch_run(flow.build, inputs)`` runs every input under one unified setting. -- ``paper_flow`` is a **deprecated** thin wrapper for the chunk/summary shape; - it warns and delegates to ``PaperFlow(PaperFlowCfg(...)).build(input)``. + ``PaperStructureTree``; ``PaperSemanticCfg`` → a source-first + ``PaperSemanticResult``). ``batch_run(flow.build, inputs)`` runs every input + under one unified setting. - ``batch_run`` runs any flow over a list of inputs with bounded concurrency and aggregated results. - ``BatchResult`` is the shape returned by ``batch_run``. @@ -26,7 +25,6 @@ PaperFlow, PaperStructureError, UnsupportedContentTypeError, - paper_flow, ) from quantmind.knowledge import PaperCitationValidationError @@ -38,5 +36,4 @@ "UnsupportedContentTypeError", "batch_run", "collect_news", - "paper_flow", ] diff --git a/quantmind/flows/_paper_summary.py b/quantmind/flows/_paper_summary.py index c7738bd..4bda58c 100644 --- a/quantmind/flows/_paper_summary.py +++ b/quantmind/flows/_paper_summary.py @@ -24,7 +24,7 @@ from agents import Agent, ModelSettings from pydantic import BaseModel, ConfigDict, Field, field_validator -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.flows._runner import run_with_observability from quantmind.knowledge import PaperChunkSet, PaperSourceRevision @@ -141,7 +141,7 @@ async def summarize( source: PaperSourceRevision, chunk_set: PaperChunkSet, *, - cfg: PaperFlowCfg, + cfg: PaperSemanticCfg, ) -> PaperSummaryDraft: """Create one bounded draft from the selected chunk set.""" ... @@ -241,7 +241,7 @@ def _reduce_payload( ) -def _summary_instructions(cfg: PaperFlowCfg) -> str: +def _summary_instructions(cfg: PaperSemanticCfg) -> str: instructions = _SUMMARY_INSTRUCTIONS if cfg.summary_instructions: instructions = ( @@ -251,7 +251,7 @@ def _summary_instructions(cfg: PaperFlowCfg) -> str: return instructions -def _summary_instructions_hash(cfg: PaperFlowCfg) -> str: +def _summary_instructions_hash(cfg: PaperSemanticCfg) -> str: payload = json.dumps( { "orchestration": _ORCHESTRATION_VERSION, @@ -267,7 +267,7 @@ def _summary_instructions_hash(cfg: PaperFlowCfg) -> str: return hashlib.sha256(payload.encode("utf-8")).hexdigest() -def _summary_model_settings(cfg: PaperFlowCfg) -> ModelSettings: +def _summary_model_settings(cfg: PaperSemanticCfg) -> ModelSettings: settings = cfg.model_settings or ModelSettings() configured = settings.max_tokens or cfg.max_summary_output_tokens return replace( @@ -284,7 +284,7 @@ async def summarize( source: PaperSourceRevision, chunk_set: PaperChunkSet, *, - cfg: PaperFlowCfg, + cfg: PaperSemanticCfg, ) -> PaperSummaryDraft: groups = _chunk_groups( len(chunk_set.chunks), cfg.summary_research_group_size diff --git a/quantmind/flows/paper/__init__.py b/quantmind/flows/paper/__init__.py index 8aa0b6d..f45c618 100644 --- a/quantmind/flows/paper/__init__.py +++ b/quantmind/flows/paper/__init__.py @@ -14,18 +14,30 @@ (fetch + parse, deterministic outline signals, one draft-structuring agent, then the knowledge-layer ``from_draft`` constructor that mints identity and populates each leaf node's page-cited text). -- ``PaperFlowCfg`` selects the source-first chunk/summary shape - (``PaperFlowResult``): fetch + parse, page-aware chunking, then a bounded +- ``PaperSemanticCfg`` selects the source-first chunk/summary shape + (``PaperSemanticResult``): fetch + parse, page-aware chunking, then a bounded map-reduce summary whose citations the knowledge layer resolves. - Any other cfg type raises ``NotImplementedError``. The bound cfg *type* determines ``build``'s result type: constructing with a -``PaperStructureCfg`` yields a ``PaperStructureTree``; a ``PaperFlowCfg`` yields -a ``PaperFlowResult`` (typed via constructor overloads over a generic result). +``PaperStructureCfg`` yields a ``PaperStructureTree``; a ``PaperSemanticCfg`` +yields a ``PaperSemanticResult`` (typed via constructor overloads over a generic +result). + +Real run — arXiv ``1706.03762v7``, model ``gpt-5.6-luna`` (2026-07-24): + +- structure: 17 nodes (13 leaves), root ``"Attention Is All You Need"``. +- semantic: 15 pages, 33 chunks. The cited-summary step asserts every research + quote verbatim against its chunk; the sampled models paraphrased, so that step + raised ``ValueError`` and produced no summary line on this run. ``build`` fetches and parses **per call**: the flow binds no source, no library, persists nothing, and retrieves nothing. Persistence (``library``) and retrieval -(``mind``) are downstream concerns a caller wires itself. +(``mind``) are downstream concerns a caller wires itself. Embeddings are not +computed here: a ``PaperSemanticResult`` carries only source, chunk, and summary +text; its vector projections are minted downstream when the result is persisted, +using the ``embedding_model`` bound at ``LocalKnowledgeLibrary.open(...)`` (the +examples use ``text-embedding-3-small``). The module keeps only what genuinely needs both the preprocess/rag value objects and IO: fetching bytes, parsing the PDF, reading page-asset bytes off disk, and @@ -33,22 +45,16 @@ identity (IDs, content and producer hashes, citation resolution) lives on the knowledge models' ``from_*`` constructors, so this module imports no private ID helpers and computes no paper ID itself. - -``paper_flow`` remains as a **deprecated** thin wrapper for the chunk/summary -shape: it emits a ``DeprecationWarning`` and delegates to -``PaperFlow(PaperFlowCfg(...)).build(input)``. It is grandfathered by the -operation-naming contract, not removed. """ import mimetypes -import warnings from collections.abc import Sequence from datetime import datetime, timezone from importlib.metadata import version from pathlib import Path from typing import Generic, Literal, TypeVar, cast, overload -from quantmind.configs import BaseFlowCfg, PaperFlowCfg, PaperStructureCfg +from quantmind.configs import BaseFlowCfg, PaperSemanticCfg, PaperStructureCfg from quantmind.configs.paper import ( ArxivIdentifier, DoiIdentifier, @@ -76,10 +82,10 @@ PaperChunkInput, PaperChunkSet, PaperCitationDraft, - PaperFlowResult, PaperGlobalSummary, PaperPageInput, PaperParsedBlock, + PaperSemanticResult, PaperSourceFacts, PaperSourceRevision, PaperStructureProducer, @@ -111,7 +117,6 @@ "PaperFlow", "PaperStructureError", "UnsupportedContentTypeError", - "paper_flow", ] @@ -134,7 +139,7 @@ class PaperFlow(Generic[_ResultT]): type of ``build``: - ``PaperStructureCfg`` builds a self-contained ``PaperStructureTree``; - - ``PaperFlowCfg`` builds a source-first ``PaperFlowResult`` (chunk set + + - ``PaperSemanticCfg`` builds a source-first ``PaperSemanticResult`` (chunk set + cited global summary). The flow binds no source, no library, and no per-call state; ``build`` @@ -158,8 +163,8 @@ def __init__( @overload def __init__( - self: "PaperFlow[PaperFlowResult]", - cfg: PaperFlowCfg, + self: "PaperFlow[PaperSemanticResult]", + cfg: PaperSemanticCfg, *, _structure_provider: _PaperStructureProvider | None = None, _summary_provider: _PaperSummaryProvider | None = None, @@ -167,7 +172,7 @@ def __init__( @overload def __init__( - self: "PaperFlow[PaperStructureTree | PaperFlowResult]", + self: "PaperFlow[PaperStructureTree | PaperSemanticResult]", cfg: BaseFlowCfg, *, _structure_provider: _PaperStructureProvider | None = None, @@ -186,8 +191,8 @@ def __init__( Args: cfg: The build config. Its **type** selects both the knowledge shape ``build`` produces and ``build``'s static result type - (``PaperStructureCfg`` → ``PaperStructureTree``, ``PaperFlowCfg`` - → ``PaperFlowResult``). + (``PaperStructureCfg`` → ``PaperStructureTree``, ``PaperSemanticCfg`` + → ``PaperSemanticResult``). _structure_provider: Optional test seam for the structure draft. _summary_provider: Optional test seam for the summary draft. """ @@ -205,9 +210,9 @@ async def build(self, input: PaperInput) -> _ResultT: knowledge-layer constructor that mints identity, resolves page citations, and populates each leaf node's ``content``), returning a self-contained ``PaperStructureTree``. - - ``PaperFlowCfg`` runs the source-first chunk/summary pipeline (fetch + + - ``PaperSemanticCfg`` runs the source-first chunk/summary pipeline (fetch + parse, page-aware chunking, bounded map-reduce summary), returning a - ``PaperFlowResult``. + ``PaperSemanticResult``. Fetch and parse run **per call**; the flow keeps no shared source state. @@ -217,11 +222,11 @@ async def build(self, input: PaperInput) -> _ResultT: Returns: The knowledge artifact for the bound cfg type: a ``PaperStructureTree`` (``PaperStructureCfg``) or a - ``PaperFlowResult`` (``PaperFlowCfg``). + ``PaperSemanticResult`` (``PaperSemanticCfg``). Raises: NotImplementedError: If the bound cfg is neither a - ``PaperStructureCfg`` nor a ``PaperFlowCfg``. + ``PaperStructureCfg`` nor a ``PaperSemanticCfg``. UnsupportedContentTypeError: If the resolved content is not a PDF. PaperStructureError: If a structure model call exceeds its timeout. PaperCitationValidationError: If summary citations are invalid or do @@ -230,12 +235,12 @@ async def build(self, input: PaperInput) -> _ResultT: cfg = self._cfg if isinstance(cfg, PaperStructureCfg): return cast(_ResultT, await self._build_structure(input, cfg)) - if isinstance(cfg, PaperFlowCfg): + if isinstance(cfg, PaperSemanticCfg): return cast(_ResultT, await self._build_semantic(input, cfg)) raise NotImplementedError( "PaperFlow.build does not support cfg type " f"{type(cfg).__name__!r}; only PaperStructureCfg (structure-tree " - "shape) and PaperFlowCfg (chunk/summary shape) are wired." + "shape) and PaperSemanticCfg (chunk/summary shape) are wired." ) async def _build_structure( @@ -266,8 +271,8 @@ async def _build_structure( async def _build_semantic( self, input: PaperInput, - cfg: PaperFlowCfg, - ) -> PaperFlowResult: + cfg: PaperSemanticCfg, + ) -> PaperSemanticResult: """Fetch, parse, chunk, and summarize one input into a paper result. Builds a page-aware chunk set and one cited global summary. IDs, source @@ -296,54 +301,13 @@ async def _build_semantic( provider = self._summary_provider or _AgentsPaperSummaryProvider() draft = await provider.summarize(source, chunk_set, cfg=cfg) summary = _build_summary(chunk_set, draft, cfg) - return PaperFlowResult( + return PaperSemanticResult( source_revision=source, chunk_set=chunk_set, global_summary=summary, ) -async def paper_flow( - input: PaperInput, - *, - cfg: PaperFlowCfg | None = None, - _summary_provider: _PaperSummaryProvider | None = None, -) -> PaperFlowResult: - """Build a page-aware chunk set and one cited global summary (deprecated). - - Deprecated thin wrapper for the source-first chunk/summary shape. It emits a - ``DeprecationWarning`` and delegates to - ``PaperFlow(PaperFlowCfg(...)).build(input)``, which now owns the semantic - pipeline. The wrapper is grandfathered by the operation-naming contract; - prefer the config-bound entry point in new code:: - - result = await PaperFlow(PaperFlowCfg(...)).build(input) - - Args: - input: Typed paper source. V1 requires a PDF-backed input. - cfg: Splitter, summary model, and usage/runtime limits. - _summary_provider: Optional test seam for the summary draft. - - Returns: - The exact source revision, one chunk-set artifact, and one cited - global-summary artifact. - - Raises: - UnsupportedContentTypeError: If the resolved content is not a PDF. - PaperCitationValidationError: If generated citations are invalid or do - not meet the configured source-coverage policy. - NotImplementedError: If a DOI input has no exact open PDF resolver. - """ - warnings.warn( - "paper_flow() is deprecated; use " - "PaperFlow(PaperFlowCfg(...)).build(input) instead.", - DeprecationWarning, - stacklevel=2, - ) - flow = PaperFlow(cfg or PaperFlowCfg(), _summary_provider=_summary_provider) - return await flow.build(input) - - async def _open_source( input: PaperInput, *, @@ -544,7 +508,7 @@ def _adapt_chunks( def _build_summary( chunk_set: PaperChunkSet, draft: PaperSummaryDraft, - cfg: PaperFlowCfg, + cfg: PaperSemanticCfg, ) -> PaperGlobalSummary: """Assemble the summary producer and delegate identity to the model.""" producer = PaperSummaryProducer( diff --git a/quantmind/knowledge/__init__.py b/quantmind/knowledge/__init__.py index fd6d8e1..7bb0002 100644 --- a/quantmind/knowledge/__init__.py +++ b/quantmind/knowledge/__init__.py @@ -45,12 +45,12 @@ PaperCitation, PaperCitationDraft, PaperCitationValidationError, - PaperFlowResult, PaperGlobalSummary, PaperPageInput, PaperParsedBlock, PaperParsedManifest, PaperParsedPage, + PaperSemanticResult, PaperSourceFacts, PaperSourceRevision, PaperSourceSpan, @@ -95,7 +95,7 @@ "PaperCitation", "PaperCitationDraft", "PaperCitationValidationError", - "PaperFlowResult", + "PaperSemanticResult", "PaperGlobalSummary", "PaperPageInput", "PaperParsedBlock", diff --git a/quantmind/knowledge/paper.py b/quantmind/knowledge/paper.py index ceeec0f..00fbcd4 100644 --- a/quantmind/knowledge/paper.py +++ b/quantmind/knowledge/paper.py @@ -1199,7 +1199,7 @@ def from_draft( ) -class PaperFlowResult(BaseModel): +class PaperSemanticResult(BaseModel): """Validated V1 result containing source, chunks, and cited summary.""" model_config = ConfigDict(extra="forbid", frozen=True) @@ -1209,7 +1209,7 @@ class PaperFlowResult(BaseModel): global_summary: PaperGlobalSummary @model_validator(mode="after") - def _validate_cross_artifact_links(self) -> "PaperFlowResult": + def _validate_cross_artifact_links(self) -> "PaperSemanticResult": source_id = self.source_revision.id if ( self.chunk_set.source_revision_id != source_id diff --git a/quantmind/library/_internal/retrieval_targets.py b/quantmind/library/_internal/retrieval_targets.py index d36d753..ef2c462 100644 --- a/quantmind/library/_internal/retrieval_targets.py +++ b/quantmind/library/_internal/retrieval_targets.py @@ -11,7 +11,7 @@ Factor, FlattenKnowledge, News, - PaperFlowResult, + PaperSemanticResult, Thesis, TreeKnowledge, ) @@ -90,7 +90,7 @@ def _project_knowledge(item: BaseKnowledge) -> list[_RetrievalTarget]: return targets -def _project_paper(result: PaperFlowResult) -> list[_RetrievalTarget]: +def _project_paper(result: PaperSemanticResult) -> list[_RetrievalTarget]: """Project a cited summary and every non-empty chunk for text search.""" source_id = result.source_revision.id summary = result.global_summary diff --git a/quantmind/library/_internal/sqlite_store.py b/quantmind/library/_internal/sqlite_store.py index 7dd6475..2b01553 100644 --- a/quantmind/library/_internal/sqlite_store.py +++ b/quantmind/library/_internal/sqlite_store.py @@ -21,8 +21,8 @@ News, PaperArtifact, PaperChunkSet, - PaperFlowResult, PaperGlobalSummary, + PaperSemanticResult, PaperSourceRevision, PaperStructureTree, ResolvedPaperArtifact, @@ -124,7 +124,7 @@ class _CanonicalPaperArtifact: class _PreparedPaperPut: """Validated source/artifact write plus reusable search projections.""" - result: PaperFlowResult + result: PaperSemanticResult source_payload: str source_canonical_hash: str artifacts: tuple[_CanonicalPaperArtifact, ...] @@ -784,7 +784,9 @@ def prepare_put(self, item: BaseKnowledge) -> _PreparedPut: existing_embeddings=existing, ) - def prepare_put_paper(self, result: PaperFlowResult) -> _PreparedPaperPut: + def prepare_put_paper( + self, result: PaperSemanticResult + ) -> _PreparedPaperPut: """Validate paper blobs and load projections eligible for reuse.""" source = result.source_revision source_payload, source_canonical_hash = _prepare_paper_source(source) @@ -1585,7 +1587,7 @@ def get_paper_result( *, chunk_set_id: UUID | None, summary_id: UUID | None, - ) -> PaperFlowResult: + ) -> PaperSemanticResult: """Resolve one unambiguous V1 source/chunk-set/summary combination.""" source = self.get_paper_source(source_revision_id) @@ -1622,7 +1624,7 @@ def select(kind: str, selected_id: UUID | None) -> PaperArtifact: summary, PaperGlobalSummary ): raise RuntimeError("Stored paper artifact types are inconsistent") - return PaperFlowResult( + return PaperSemanticResult( source_revision=source, chunk_set=chunk_set, global_summary=summary, diff --git a/quantmind/library/local.py b/quantmind/library/local.py index dc5600a..b1459aa 100644 --- a/quantmind/library/local.py +++ b/quantmind/library/local.py @@ -12,8 +12,8 @@ Citation, PaperArtifact, PaperChunkSet, - PaperFlowResult, PaperGlobalSummary, + PaperSemanticResult, PaperSourceRevision, PaperStructureTree, ResolvedPaperArtifact, @@ -168,7 +168,7 @@ async def put(self, item: BaseKnowledge | PaperStructureTree) -> None: ) self._index = None - async def put_paper(self, result: PaperFlowResult) -> None: + async def put_paper(self, result: PaperSemanticResult) -> None: """Persist one source-first paper result and all required projections. Embeddings are prepared before the SQLite transaction. A provider @@ -286,7 +286,7 @@ async def get_paper( *, chunk_set_id: UUID | None = None, summary_id: UUID | None = None, - ) -> PaperFlowResult: + ) -> PaperSemanticResult: """Return one unambiguous source/chunk-set/summary combination.""" async with self._lock: store = self._store diff --git a/quantmind/magic.py b/quantmind/magic.py index 819b96c..0d429cd 100644 --- a/quantmind/magic.py +++ b/quantmind/magic.py @@ -71,7 +71,7 @@ async def resolve_magic_input( natural_language: str, *, target_flow: Callable[..., Awaitable[Any]], - resolver_model: str = "gpt-4o-mini", + resolver_model: str = "gpt-5.6-luna", resolver_instructions: str | None = None, ) -> tuple[Any, Any]: """Parse ``natural_language`` into ``(input_obj, cfg_obj)`` for ``target_flow``. @@ -110,7 +110,7 @@ async def preview_resolve( natural_language: str, *, target_flow: Callable[..., Awaitable[Any]], - resolver_model: str = "gpt-4o-mini", + resolver_model: str = "gpt-5.6-luna", ) -> tuple[Any, Any]: """Resolve and pretty-print the result without invoking the flow.""" inp, cfg = await resolve_magic_input( diff --git a/scripts/verify_pdf_rag_e2e.py b/scripts/verify_pdf_rag_e2e.py index d45ad87..fbaa5a9 100644 --- a/scripts/verify_pdf_rag_e2e.py +++ b/scripts/verify_pdf_rag_e2e.py @@ -9,7 +9,7 @@ from dotenv import load_dotenv -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.configs.paper import ArxivIdentifier from quantmind.flows import PaperFlow from quantmind.knowledge import ( @@ -53,8 +53,8 @@ async def _run_vertical_slice() -> dict[str, Any]: with tempfile.TemporaryDirectory(prefix="quantmind-paper-v1-") as directory: root = Path(directory) flow = PaperFlow( - PaperFlowCfg( - model="gpt-4o-mini", + PaperSemanticCfg( + model="gpt-5.6-luna", output_dir=str(root / "assets"), timeout_seconds=240, summary_research_group_size=8, diff --git a/tests/configs/test_base.py b/tests/configs/test_base.py index 164e237..65213bd 100644 --- a/tests/configs/test_base.py +++ b/tests/configs/test_base.py @@ -51,10 +51,10 @@ def test_top_level_imports(self): from quantmind.configs import ( EarningsFlowCfg, NewsCollectionCfg, - PaperFlowCfg, + PaperSemanticCfg, ) - self.assertTrue(issubclass(PaperFlowCfg, BaseFlowCfgExport)) + self.assertTrue(issubclass(PaperSemanticCfg, BaseFlowCfgExport)) self.assertTrue(issubclass(NewsCollectionCfg, BaseFlowCfgExport)) self.assertTrue(issubclass(EarningsFlowCfg, BaseFlowCfgExport)) self.assertEqual(BaseInputExport.__name__, "BaseInput") diff --git a/tests/configs/test_paper.py b/tests/configs/test_paper.py index 6247521..28de376 100644 --- a/tests/configs/test_paper.py +++ b/tests/configs/test_paper.py @@ -10,16 +10,16 @@ DoiIdentifier, HttpUrl, LocalFilePath, - PaperFlowCfg, PaperInput, + PaperSemanticCfg, RawText, ) -class PaperFlowCfgTests(unittest.TestCase): +class PaperSemanticCfgTests(unittest.TestCase): def test_defaults(self): - cfg = PaperFlowCfg() - self.assertEqual(cfg.model, "gpt-4o-mini") + cfg = PaperSemanticCfg() + self.assertEqual(cfg.model, "gpt-5.6-luna") self.assertEqual(cfg.max_turns, 16) self.assertEqual(cfg.chunk_size, 512) self.assertEqual(cfg.chunk_overlap, 64) @@ -31,9 +31,9 @@ def test_defaults(self): def test_invalid_overlap_or_coverage_bounds_are_rejected(self): with self.assertRaises(ValidationError): - PaperFlowCfg(chunk_size=64, chunk_overlap=64) + PaperSemanticCfg(chunk_size=64, chunk_overlap=64) with self.assertRaises(ValidationError): - PaperFlowCfg(min_summary_citations=1, min_summary_pages=2) + PaperSemanticCfg(min_summary_citations=1, min_summary_pages=2) class PaperInputDiscriminatedTests(unittest.TestCase): diff --git a/tests/flows/test_batch.py b/tests/flows/test_batch.py index 6df8c5c..6017a01 100644 --- a/tests/flows/test_batch.py +++ b/tests/flows/test_batch.py @@ -4,7 +4,7 @@ import unittest from typing import Any -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.configs.paper import RawText from quantmind.flows.batch import BatchResult, batch_run @@ -143,7 +143,7 @@ async def flow(input: RawText, *, cfg: Any = None) -> str: seen_cfg.append(cfg) return "ok" - cfg = PaperFlowCfg(model="sentinel-model") + cfg = PaperSemanticCfg(model="sentinel-model") await batch_run(flow, [RawText(text="x")], cfg=cfg, concurrency=1) self.assertIs(seen_cfg[0], cfg) diff --git a/tests/flows/test_paper.py b/tests/flows/test_paper.py index 789453b..9c74562 100644 --- a/tests/flows/test_paper.py +++ b/tests/flows/test_paper.py @@ -1,4 +1,4 @@ -"""Offline tests for source-first ``paper_flow`` behavior.""" +"""Offline tests for the source-first ``PaperFlow`` semantic build.""" import asyncio import json @@ -9,7 +9,7 @@ from agents import ModelSettings -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.configs.paper import ( ArxivIdentifier, DoiIdentifier, @@ -35,7 +35,6 @@ UnsupportedContentTypeError, _build_summary, _fetch_paper_source, - paper_flow, ) from quantmind.knowledge import PaperCitationValidationError from quantmind.preprocess.fetch import Fetched, RawPaper @@ -90,7 +89,7 @@ async def summarize(self, source, chunk_set, *, cfg): class PaperFlowTests(unittest.IsolatedAsyncioTestCase): async def test_local_pdf_builds_chunks_before_summary(self) -> None: provider = _FakeSummaryProvider() - cfg = PaperFlowCfg(chunk_size=256, chunk_overlap=32) + cfg = PaperSemanticCfg(chunk_size=256, chunk_overlap=32) result = await PaperFlow( cfg, @@ -140,7 +139,7 @@ async def test_exact_arxiv_source_facts_are_code_owned(self) -> None: new=AsyncMock(return_value=raw), ): result = await PaperFlow( - PaperFlowCfg(chunk_size=256, chunk_overlap=32), + PaperSemanticCfg(chunk_size=256, chunk_overlap=32), _summary_provider=_FakeSummaryProvider(), ).build(ArxivIdentifier(id="1706.03762v7")) @@ -160,12 +159,12 @@ async def test_summary_failure_prevents_flow_success(self) -> None: with self.assertRaisesRegex(RuntimeError, "summary failed"): await PaperFlow( - PaperFlowCfg(chunk_size=256, chunk_overlap=32), + PaperSemanticCfg(chunk_size=256, chunk_overlap=32), _summary_provider=provider, ).build(LocalFilePath(path=_FIXTURE)) async def test_same_pdf_and_configs_have_idempotent_ids(self) -> None: - cfg = PaperFlowCfg(chunk_size=256, chunk_overlap=32) + cfg = PaperSemanticCfg(chunk_size=256, chunk_overlap=32) first = await PaperFlow( cfg, _summary_provider=_FakeSummaryProvider(), @@ -184,41 +183,6 @@ async def test_same_pdf_and_configs_have_idempotent_ids(self) -> None: ) -class PaperFlowDeprecationTests(unittest.IsolatedAsyncioTestCase): - async def test_paper_flow_warns_and_delegates(self) -> None: - # The grandfathered ``paper_flow`` wrapper must emit a - # DeprecationWarning and delegate to ``PaperFlow.build`` for the same - # source-first result, forwarding the summary-provider seam. - provider = _FakeSummaryProvider() - - with self.assertWarns(DeprecationWarning): - result = await paper_flow( - LocalFilePath(path=_FIXTURE), - cfg=PaperFlowCfg(chunk_size=256, chunk_overlap=32), - _summary_provider=provider, - ) - - self.assertEqual(len(provider.calls), 1) - _, chunk_set_seen, _ = provider.calls[0] - self.assertIs(chunk_set_seen, result.chunk_set) - self.assertIs(result.source_revision, provider.calls[0][0]) - self.assertTrue(result.global_summary.summary) - self.assertEqual(len(result.source_revision.parsed.pages), 4) - - async def test_paper_flow_defaults_cfg_when_omitted(self) -> None: - # Omitting cfg still delegates under a default ``PaperFlowCfg``. - provider = _FakeSummaryProvider() - - with self.assertWarns(DeprecationWarning): - result = await paper_flow( - LocalFilePath(path=_FIXTURE), - _summary_provider=provider, - ) - - self.assertEqual(len(provider.calls), 1) - self.assertTrue(result.chunk_set.chunks) - - class SourceDispatchTests(unittest.IsolatedAsyncioTestCase): async def test_http_pdf_preserves_resolved_url_and_fetch_time(self) -> None: raw = Fetched( @@ -260,7 +224,7 @@ async def test_doi_requires_an_exact_pdf_resolver(self) -> None: class CitationValidationTests(unittest.TestCase): def test_unknown_chunk_page_and_quote_are_rejected(self) -> None: result = build_paper_result() - cfg = PaperFlowCfg( + cfg = PaperSemanticCfg( min_summary_citations=1, min_summary_pages=1, ) @@ -320,7 +284,7 @@ def test_configured_citation_and_page_coverage_is_enforced(self) -> None: _build_summary( result.chunk_set, draft, - PaperFlowCfg(), + PaperSemanticCfg(), ) @@ -392,14 +356,14 @@ def test_research_finding_outside_its_group_is_rejected(self) -> None: def test_worker_and_reducer_output_is_capped(self) -> None: capped = _summary_model_settings( - PaperFlowCfg( + PaperSemanticCfg( max_summary_output_tokens=256, model_settings=ModelSettings(max_tokens=1024), ) ) self.assertEqual(capped.max_tokens, 256) lower = _summary_model_settings( - PaperFlowCfg( + PaperSemanticCfg( max_summary_output_tokens=256, model_settings=ModelSettings(max_tokens=128), ) @@ -408,7 +372,7 @@ def test_worker_and_reducer_output_is_capped(self) -> None: async def test_map_reduce_fans_out_one_worker_per_group(self) -> None: result = build_paper_result() - cfg = PaperFlowCfg( + cfg = PaperSemanticCfg( summary_research_group_size=1, summary_concurrency=2, min_summary_citations=1, @@ -440,7 +404,7 @@ async def test_map_reduce_fans_out_one_worker_per_group(self) -> None: async def test_summary_timeout_raises(self) -> None: result = build_paper_result() - cfg = PaperFlowCfg( + cfg = PaperSemanticCfg( summary_research_group_size=8, timeout_seconds=0.01, min_summary_citations=1, diff --git a/tests/flows/test_runner.py b/tests/flows/test_runner.py index 4ece594..b5a8a55 100644 --- a/tests/flows/test_runner.py +++ b/tests/flows/test_runner.py @@ -6,7 +6,7 @@ from agents import RunHooks -from quantmind.configs import PaperFlowCfg +from quantmind.configs import PaperSemanticCfg from quantmind.flows._runner import ( _archive_run_artifacts, _collect_hooks, @@ -113,7 +113,7 @@ def test_no_extras_returns_empty(self) -> None: class ArchiveStubTests(unittest.TestCase): def test_archive_is_no_op(self) -> None: - cfg = PaperFlowCfg() + cfg = PaperSemanticCfg() result = MagicMock() # Must not raise, must return None, must not touch result. self.assertIsNone(_archive_run_artifacts(cfg, None, result)) @@ -122,7 +122,7 @@ def test_archive_is_no_op(self) -> None: class RunWithObservabilityTests(unittest.IsolatedAsyncioTestCase): async def test_run_config_built_from_cfg(self) -> None: - cfg = PaperFlowCfg( + cfg = PaperSemanticCfg( model="gpt-test", max_turns=7, workflow_name="custom-name", @@ -160,7 +160,7 @@ async def test_run_config_built_from_cfg(self) -> None: self.assertIsNone(call.kwargs["hooks"]) async def test_workflow_name_falls_back_to_agent_name(self) -> None: - cfg = PaperFlowCfg() # workflow_name = None + cfg = PaperSemanticCfg() # workflow_name = None agent = MagicMock() agent.name = "paper_extractor" fake_result = MagicMock() @@ -178,7 +178,7 @@ async def test_workflow_name_falls_back_to_agent_name(self) -> None: ) async def test_extra_hooks_forwarded(self) -> None: - cfg = PaperFlowCfg() + cfg = PaperSemanticCfg() agent = MagicMock() agent.name = "x" fake_result = MagicMock() diff --git a/tests/flows/test_structure.py b/tests/flows/test_structure.py index e28867a..40ee22f 100644 --- a/tests/flows/test_structure.py +++ b/tests/flows/test_structure.py @@ -148,7 +148,7 @@ async def test_build_propagates_structure_error(self) -> None: self.assertEqual(provider.calls, 1) async def test_unwired_cfg_type_raises_not_implemented(self) -> None: - # A cfg type that is neither PaperStructureCfg nor PaperFlowCfg selects + # A cfg type that is neither PaperStructureCfg nor PaperSemanticCfg selects # an unwired shape: build must reject it by cfg type, before any fetch # or parse. flow = PaperFlow(BaseFlowCfg()) diff --git a/tests/knowledge/test_base.py b/tests/knowledge/test_base.py index d442922..e854dfe 100644 --- a/tests/knowledge/test_base.py +++ b/tests/knowledge/test_base.py @@ -67,9 +67,9 @@ def test_extra_forbidden(self): class ExtractionRefTests(unittest.TestCase): def test_minimal(self): e = ExtractionRef( - flow="paper_flow", model="gpt-4o", extracted_at=_now() + flow="paper_semantic", model="gpt-4o", extracted_at=_now() ) - self.assertEqual(e.flow, "paper_flow") + self.assertEqual(e.flow, "paper_semantic") self.assertEqual(e.model, "gpt-4o") self.assertIsNone(e.run_id) @@ -146,11 +146,11 @@ def test_extraction_optional(self): def test_extraction_round_trip(self): ext = ExtractionRef( - flow="paper_flow", model="gpt-4o", extracted_at=_now() + flow="paper_semantic", model="gpt-4o", extracted_at=_now() ) item = _ConcreteKnowledge(as_of=_now(), source=_src(), extraction=ext) assert item.extraction is not None - self.assertEqual(item.extraction.flow, "paper_flow") + self.assertEqual(item.extraction.flow, "paper_semantic") def test_is_extracted_false_when_hand_curated(self): item = _ConcreteKnowledge(as_of=_now(), source=_src()) @@ -158,7 +158,7 @@ def test_is_extracted_false_when_hand_curated(self): def test_is_extracted_true_when_extraction_set(self): ext = ExtractionRef( - flow="paper_flow", model="gpt-4o", extracted_at=_now() + flow="paper_semantic", model="gpt-4o", extracted_at=_now() ) item = _ConcreteKnowledge(as_of=_now(), source=_src(), extraction=ext) self.assertTrue(item.is_extracted()) @@ -201,8 +201,8 @@ def test_top_level_imports(self): LegacyPaper, News, PaperChunkSet, - PaperFlowResult, PaperGlobalSummary, + PaperSemanticResult, PaperSourceRevision, SourceRef, Thesis, @@ -221,7 +221,7 @@ def test_top_level_imports(self): self.assertEqual(PaperSourceRevision.__name__, "PaperSourceRevision") self.assertEqual(PaperChunkSet.__name__, "PaperChunkSet") self.assertEqual(PaperGlobalSummary.__name__, "PaperGlobalSummary") - self.assertEqual(PaperFlowResult.__name__, "PaperFlowResult") + self.assertEqual(PaperSemanticResult.__name__, "PaperSemanticResult") # Ensure side-imports are real classes self.assertEqual(Citation.__name__, "Citation") self.assertEqual(SourceRef.__name__, "SourceRef") diff --git a/tests/knowledge/test_paper.py b/tests/knowledge/test_paper.py index 5ab4312..2d4233b 100644 --- a/tests/knowledge/test_paper.py +++ b/tests/knowledge/test_paper.py @@ -7,7 +7,7 @@ from quantmind.knowledge import ( PaperCitation, - PaperFlowResult, + PaperSemanticResult, PaperSourceRevision, PaperSourceSpan, ) @@ -115,7 +115,7 @@ def test_result_rejects_unknown_model_owned_citation_identity(self) -> None: ) with self.assertRaisesRegex(ValidationError, "unknown chunk"): - PaperFlowResult( + PaperSemanticResult( source_revision=result.source_revision, chunk_set=result.chunk_set, global_summary=summary, @@ -161,7 +161,7 @@ def test_result_rejects_chunk_spans_outside_source_manifest(self) -> None: ) with self.assertRaisesRegex(ValidationError, "exceeds its source page"): - PaperFlowResult( + PaperSemanticResult( source_revision=result.source_revision, chunk_set=invalid_chunk_set, global_summary=result.global_summary, diff --git a/tests/paper_helpers.py b/tests/paper_helpers.py index cd5a384..2bd7b68 100644 --- a/tests/paper_helpers.py +++ b/tests/paper_helpers.py @@ -1,7 +1,7 @@ """Deterministic paper artifacts shared by offline tests. The fixture builds through the same knowledge-layer smart constructors that -``paper_flow`` uses, so tests exercise the real identity path instead of +``PaperFlow.build`` uses, so tests exercise the real identity path instead of re-deriving IDs and hashes with the private helpers. """ @@ -14,9 +14,9 @@ PaperChunkInput, PaperChunkSet, PaperCitationDraft, - PaperFlowResult, PaperGlobalSummary, PaperPageInput, + PaperSemanticResult, PaperSourceFacts, PaperSourceRevision, PaperStructureNodeDraft, @@ -48,7 +48,7 @@ def build_paper_result( "and improves translation quality with efficient training." ), when: datetime = _WHEN, -) -> PaperFlowResult: +) -> PaperSemanticResult: """Build one valid two-page result without parsing, network, or models. ``when`` drives the source revision timestamps (and therefore the derived @@ -129,7 +129,7 @@ def build_paper_result( min_citations=1, min_pages=1, ) - return PaperFlowResult( + return PaperSemanticResult( source_revision=source, chunk_set=chunk_set, global_summary=summary, diff --git a/tests/test_magic.py b/tests/test_magic.py index 80eba39..7ee4f52 100644 --- a/tests/test_magic.py +++ b/tests/test_magic.py @@ -4,14 +4,19 @@ import json import unittest from contextlib import redirect_stdout +from datetime import datetime, timezone from typing import Optional, Union from unittest.mock import AsyncMock, MagicMock, patch from pydantic import BaseModel -from quantmind.configs import PaperFlowCfg +from quantmind.configs import ( + NewsCollectionCfg, + NewsWindow, + PaperSemanticCfg, +) from quantmind.configs.paper import ArxivIdentifier, PaperInput -from quantmind.flows import paper_flow +from quantmind.flows import collect_news from quantmind.magic import ( ResolvedFlowConfig, _introspect_flow_signature, @@ -37,28 +42,36 @@ def test_union_without_none_unchanged(self) -> None: self.assertEqual(_strip_optional(anno), anno) +def _news_window() -> NewsWindow: + """A minimal valid collection window for resolver fixtures.""" + return NewsWindow( + source="pr-newswire", + start=datetime(2026, 1, 1, tzinfo=timezone.utc), + end=datetime(2026, 1, 2, tzinfo=timezone.utc), + ) + + class IntrospectFlowSignatureTests(unittest.TestCase): - def test_paper_flow_returns_paper_input_and_cfg(self) -> None: - input_type, cfg_type = _introspect_flow_signature(paper_flow) - self.assertIs(cfg_type, PaperFlowCfg) - # PaperInput is the Annotated[Union[...]] alias; pass through. - self.assertEqual(input_type, PaperInput) + def test_collect_news_returns_window_input_and_cfg(self) -> None: + input_type, cfg_type = _introspect_flow_signature(collect_news) + self.assertIs(cfg_type, NewsCollectionCfg) + self.assertIs(input_type, NewsWindow) def test_resolves_postponed_annotations(self) -> None: async def postponed_flow( input: "ArxivIdentifier", *, - cfg: "PaperFlowCfg | None" = None, + cfg: "PaperSemanticCfg | None" = None, ) -> None: return None input_type, cfg_type = _introspect_flow_signature(postponed_flow) self.assertIs(input_type, ArxivIdentifier) - self.assertIs(cfg_type, PaperFlowCfg) + self.assertIs(cfg_type, PaperSemanticCfg) def test_missing_input_param_raises(self) -> None: - async def bad(*, cfg: PaperFlowCfg | None = None) -> None: + async def bad(*, cfg: PaperSemanticCfg | None = None) -> None: return None with self.assertRaises(TypeError): @@ -81,11 +94,11 @@ async def bad(input: ArxivIdentifier, *, cfg: int = 0) -> None: class PydanticSchemaStrTests(unittest.TestCase): def test_basemodel_renders_json_schema(self) -> None: - # PaperFlowCfg embeds ModelSettings which has callable fields; + # PaperSemanticCfg embeds ModelSettings which has callable fields; # the renderer falls back to a fields summary in that case. - out = _pydantic_schema_str(PaperFlowCfg) + out = _pydantic_schema_str(PaperSemanticCfg) parsed = json.loads(out) - self.assertEqual(parsed.get("title"), "PaperFlowCfg") + self.assertEqual(parsed.get("title"), "PaperSemanticCfg") self.assertIn("model", parsed["fields"]) def test_basemodel_with_clean_schema(self) -> None: @@ -126,9 +139,9 @@ def _capture_agent(*_a: object, **kwargs: object) -> object: # Build a fake resolver result whose final_output is a populated # ResolvedFlowConfig. - resolved = ResolvedFlowConfig[PaperInput, PaperFlowCfg]( - input_obj=ArxivIdentifier(id="2604.12345"), - cfg_obj=PaperFlowCfg(model="gpt-test"), + resolved = ResolvedFlowConfig[NewsWindow, NewsCollectionCfg]( + input_obj=_news_window(), + cfg_obj=NewsCollectionCfg(retain_raw_html=True), ) fake_result = MagicMock() fake_result.final_output = resolved @@ -140,14 +153,14 @@ def _capture_agent(*_a: object, **kwargs: object) -> object: ), ): inp, cfg = await resolve_magic_input( - "fetch arxiv 2604.12345 about momentum", - target_flow=paper_flow, + "collect the last day of pr-newswire", + target_flow=collect_news, ) self.assertIs(inp, resolved.input_obj) self.assertIs(cfg, resolved.cfg_obj) # Resolver agent was given a name derived from the flow. - self.assertEqual(captured["name"], "magic_resolver_paper_flow") - self.assertEqual(captured["model"], "gpt-4o-mini") + self.assertEqual(captured["name"], "magic_resolver_collect_news") + self.assertEqual(captured["model"], "gpt-5.6-luna") async def test_custom_resolver_instructions(self) -> None: captured: dict[str, object] = {} @@ -156,9 +169,9 @@ def _capture_agent(*_a: object, **kwargs: object) -> object: captured.update(kwargs) return MagicMock() - resolved = ResolvedFlowConfig[PaperInput, PaperFlowCfg]( - input_obj=ArxivIdentifier(id="x"), - cfg_obj=PaperFlowCfg(), + resolved = ResolvedFlowConfig[NewsWindow, NewsCollectionCfg]( + input_obj=_news_window(), + cfg_obj=NewsCollectionCfg(), ) fake_result = MagicMock() fake_result.final_output = resolved @@ -172,12 +185,12 @@ def _capture_agent(*_a: object, **kwargs: object) -> object: ): await resolve_magic_input( "x", - target_flow=paper_flow, + target_flow=collect_news, resolver_instructions=template, ) instructions = captured["instructions"] assert isinstance(instructions, str) - self.assertTrue(instructions.startswith("FLOW=paper_flow")) + self.assertTrue(instructions.startswith("FLOW=collect_news")) self.assertIn("INPUT=", instructions) self.assertIn("CFG=", instructions) @@ -188,9 +201,9 @@ def _capture_agent(*_a: object, **kwargs: object) -> object: captured.update(kwargs) return MagicMock() - resolved = ResolvedFlowConfig[PaperInput, PaperFlowCfg]( - input_obj=ArxivIdentifier(id="x"), - cfg_obj=PaperFlowCfg(), + resolved = ResolvedFlowConfig[NewsWindow, NewsCollectionCfg]( + input_obj=_news_window(), + cfg_obj=NewsCollectionCfg(), ) fake_result = MagicMock() fake_result.final_output = resolved @@ -203,17 +216,17 @@ def _capture_agent(*_a: object, **kwargs: object) -> object: ): await resolve_magic_input( "x", - target_flow=paper_flow, - resolver_model="claude-3-5-sonnet", + target_flow=collect_news, + resolver_model="custom-resolver-model", ) - self.assertEqual(captured["model"], "claude-3-5-sonnet") + self.assertEqual(captured["model"], "custom-resolver-model") class PreviewResolveTests(unittest.IsolatedAsyncioTestCase): async def test_prints_and_returns_tuple(self) -> None: - resolved = ResolvedFlowConfig[PaperInput, PaperFlowCfg]( - input_obj=ArxivIdentifier(id="2604.12345"), - cfg_obj=PaperFlowCfg(), + resolved = ResolvedFlowConfig[NewsWindow, NewsCollectionCfg]( + input_obj=_news_window(), + cfg_obj=NewsCollectionCfg(), ) fake_result = MagicMock() fake_result.final_output = resolved @@ -226,10 +239,10 @@ async def test_prints_and_returns_tuple(self) -> None: ): buf = io.StringIO() with redirect_stdout(buf): - inp, cfg = await preview_resolve("x", target_flow=paper_flow) + inp, cfg = await preview_resolve("x", target_flow=collect_news) self.assertIs(inp, resolved.input_obj) self.assertIs(cfg, resolved.cfg_obj) out = buf.getvalue() self.assertIn("input_obj:", out) self.assertIn("cfg_obj:", out) - self.assertIn("2604.12345", out) + self.assertIn("pr-newswire", out)