diff --git a/api.md b/api.md index 0652d10..633045a 100644 --- a/api.md +++ b/api.md @@ -78,13 +78,20 @@ from landingai_ade.types.v2 import ( V2ExtractResult, V2FileUploadResponse, V2ParseBilling, + V2ParseElement, + V2ParseGrounding, + V2ParseGroundingElement, + V2ParseGroundingEntry, + V2ParseGroundingPage, V2ParseMetadata, + V2ParsePage, V2ParseResponse, + V2ParseStructure, ) ``` - Job -- unified job shape: `job_id`, `status` (JobStatus: `pending` / `processing` / `completed` / `failed` / `cancelled`), `created_at`, `completed_at`, `progress`, `result` (a `V2ParseResponse` for parse jobs, a `V2ExtractResult` for extract jobs, or `None` until completion), `error` (JobError), `raw` (the full original envelope as a `dict`), and the `.is_terminal` property. -- V2ParseResponse -- `markdown`, `structure`, `grounding`, `metadata` (V2ParseMetadata, which nests V2ParseBilling). Loosely typed pending a published gateway schema; unknown fields are retained. +- V2ParseResponse -- `markdown`, `structure`, `grounding`, `metadata` (V2ParseMetadata, which nests V2ParseBilling). `structure` is a typed V2ParseStructure tree (`document` → V2ParsePageV2ParseElement); `grounding` is the parallel V2ParseGrounding tree (V2ParseGroundingPageV2ParseGroundingElementV2ParseGroundingEntry) carrying boxes and parts. Element `type`/page `status` are permissive strings and unknown keys are retained. - V2ExtractResult -- `extraction`, `extraction_metadata`, `markdown`, `metadata` (V2ExtractMetadata, which nests V2ExtractBilling). - V2FileUploadResponse -- `file_ref`. diff --git a/src/landingai_ade/types/v2/__init__.py b/src/landingai_ade/types/v2/__init__.py index 7502512..95b6f85 100644 --- a/src/landingai_ade/types/v2/__init__.py +++ b/src/landingai_ade/types/v2/__init__.py @@ -2,9 +2,16 @@ from .job import Job as Job, JobError as JobError, JobStatus as JobStatus from .parse_response import ( + V2ParsePage as V2ParsePage, V2ParseBilling as V2ParseBilling, + V2ParseElement as V2ParseElement, V2ParseMetadata as V2ParseMetadata, V2ParseResponse as V2ParseResponse, + V2ParseGrounding as V2ParseGrounding, + V2ParseStructure as V2ParseStructure, + V2ParseGroundingPage as V2ParseGroundingPage, + V2ParseGroundingEntry as V2ParseGroundingEntry, + V2ParseGroundingElement as V2ParseGroundingElement, ) from .extract_response import ( V2ExtractResult as V2ExtractResult, diff --git a/src/landingai_ade/types/v2/parse_response.py b/src/landingai_ade/types/v2/parse_response.py index 88c3eda..f5e433f 100644 --- a/src/landingai_ade/types/v2/parse_response.py +++ b/src/landingai_ade/types/v2/parse_response.py @@ -4,7 +4,18 @@ from ..._models import BaseModel -__all__ = ["V2ParseBilling", "V2ParseMetadata", "V2ParseResponse"] +__all__ = [ + "V2ParseBilling", + "V2ParseMetadata", + "V2ParseElement", + "V2ParsePage", + "V2ParseStructure", + "V2ParseGroundingEntry", + "V2ParseGroundingElement", + "V2ParseGroundingPage", + "V2ParseGrounding", + "V2ParseResponse", +] class V2ParseBilling(BaseModel): @@ -23,12 +34,90 @@ class V2ParseMetadata(BaseModel): billing: Optional[V2ParseBilling] = None +# --- `structure`: the logical document tree (no spatial data) ------------------ +# +# `type` and `status` are typed as permissive `str` (not `Literal`) so a novel +# element/status value from the gateway never fails deserialization; unknown keys +# are retained via `BaseModel`'s `extra="allow"`. + + +class V2ParseElement(BaseModel): + """A document element (text, table, table_cell, figure, ...). Keys off `type`; + optional fields are only present for the relevant element types.""" + + type: str + id: str + # Unicode code-point offsets `[start, end)` into the top-level `markdown`. + span: List[int] + # The cells of a `table` element (each a `table_cell`); only set for tables. + children: Optional[List["V2ParseElement"]] = None + # Table-cell geometry; only set on `table_cell` elements. + row: Optional[int] = None + col: Optional[int] = None + colspan: Optional[int] = None + rowspan: Optional[int] = None + + +class V2ParsePage(BaseModel): + type: str = "page" + # 0-indexed page number in the source document. + page: int + span: List[int] + width: Optional[int] = None + height: Optional[int] = None + dpi: Optional[int] = None + status: str = "ok" + reason: Optional[str] = None + children: List[V2ParseElement] = [] + + +class V2ParseStructure(BaseModel): + """Root of the `structure` tree (`document -> page -> element`).""" + + type: str = "document" + children: List[V2ParsePage] = [] + + +# --- `grounding`: mirrors `structure`, carrying the spatial data --------------- + + +class V2ParseGroundingEntry(BaseModel): + """One fine-grained grounding segment (line-level or finer).""" + + span: List[int] + # Bounding box `[left, top, right, bottom]` in source-page pixels. + box: List[int] + + +class V2ParseGroundingElement(BaseModel): + type: str + id: str + span: List[int] + box: List[int] + parts: List[V2ParseGroundingEntry] = [] + # The cells of a `table` element; only set for tables. + children: Optional[List["V2ParseGroundingElement"]] = None + + +class V2ParseGroundingPage(BaseModel): + type: str = "page" + page: int + span: List[int] + children: List[V2ParseGroundingElement] = [] + + +class V2ParseGrounding(BaseModel): + """Root of the `grounding` tree, mirroring `structure` with spatial data.""" + + type: str = "document" + children: List[V2ParseGroundingPage] = [] + + class V2ParseResponse(BaseModel): - """V2 parse result. The gateway spec types this loosely; fields are permissive - and extra keys are retained. Re-verify against the typed schema when the gateway - publishes one.""" + """V2 parse result. `structure` and `grounding` are typed trees mirroring the + published gateway schema; unknown element types and extra keys are retained.""" markdown: Optional[str] = None - structure: Optional[object] = None - grounding: Optional[object] = None + structure: Optional[V2ParseStructure] = None + grounding: Optional[V2ParseGrounding] = None metadata: Optional[V2ParseMetadata] = None diff --git a/tests/api_resources/v2/test_parse.py b/tests/api_resources/v2/test_parse.py index 4d81e89..ed81581 100644 --- a/tests/api_resources/v2/test_parse.py +++ b/tests/api_resources/v2/test_parse.py @@ -15,10 +15,63 @@ APIKEY = "My Apikey" PARSE_BODY: Dict[str, Any] = { "markdown": "# Hello", - "structure": [{"type": "text"}], + "structure": {"type": "document", "children": []}, + "grounding": {"type": "document", "children": []}, "metadata": {"req_id": "r1", "job_id": "j1", "model_version": "dpt-3", "page_count": 1, "failed_pages": []}, } +# A fully-populated ParseResponse mirroring the typed gateway schema: a document +# with one page holding a text element and a table (with a table_cell child), plus +# the parallel grounding tree carrying boxes/parts. +RICH_PARSE_BODY: Dict[str, Any] = { + "markdown": "# Invoice", + "metadata": {"req_id": "r1", "job_id": "j1", "model_version": "dpt-3", "page_count": 1, "failed_pages": []}, + "structure": { + "type": "document", + "children": [ + { + "type": "page", + "page": 0, + "span": [0, 100], + "width": 800, + "height": 1000, + "dpi": 150, + "status": "ok", + "children": [ + {"type": "text", "id": "e1", "span": [0, 20]}, + { + "type": "table", + "id": "e2", + "span": [20, 80], + "children": [ + {"type": "table_cell", "id": "e3", "span": [25, 30], + "row": 0, "col": 1, "colspan": 2, "rowspan": 1}, + ], + }, + ], + } + ], + }, + "grounding": { + "type": "document", + "children": [ + { + "type": "page", + "page": 0, + "span": [0, 100], + "children": [ + {"type": "text", "id": "e1", "span": [0, 20], "box": [10, 10, 200, 30], + "parts": [{"span": [0, 20], "box": [10, 10, 200, 30]}]}, + {"type": "table", "id": "e2", "span": [20, 80], "box": [10, 40, 400, 300], "parts": [], + "children": [ + {"type": "table_cell", "id": "e3", "span": [25, 30], "box": [12, 42, 100, 80], "parts": []}, + ]}, + ], + } + ], + }, +} + @respx.mock def test_parse_sync_ok_routes_to_v2_and_sends_options_json() -> None: @@ -159,6 +212,62 @@ def test_parse_sync_206_returns_response_with_failed_pages() -> None: assert result.metadata is not None and result.metadata.failed_pages == [3] +@respx.mock +def test_parse_sync_typed_structure_and_grounding() -> None: + # The gateway now returns a typed ParseResponse; `structure` and `grounding` + # deserialize into typed models with attribute access, not raw dicts. + from landingai_ade.types.v2 import V2ParseGrounding, V2ParseStructure + + client = LandingAIADE(apikey=APIKEY) + respx.post("https://api.ade.landing.ai/v2/parse").mock( + return_value=httpx.Response(200, json=RICH_PARSE_BODY) + ) + result = client.v2.parse(document=b"pdf") + + assert isinstance(result.structure, V2ParseStructure) + page = result.structure.children[0] + assert page.page == 0 and page.width == 800 and page.dpi == 150 and page.status == "ok" + text, table = page.children[0], page.children[1] + assert text.type == "text" and text.id == "e1" + assert table.type == "table" and table.children is not None + cell = table.children[0] + assert cell.type == "table_cell" and cell.row == 0 and cell.col == 1 and cell.colspan == 2 + + assert isinstance(result.grounding, V2ParseGrounding) + g_text = result.grounding.children[0].children[0] + assert g_text.box == [10, 10, 200, 30] + assert g_text.parts[0].span == [0, 20] and g_text.parts[0].box == [10, 10, 200, 30] + g_table = result.grounding.children[0].children[1] + assert g_table.children is not None and g_table.children[0].box == [12, 42, 100, 80] + + +@respx.mock +def test_parse_sync_tolerates_unknown_element_type_and_extra_keys() -> None: + # Novel element `type` values and extra keys must not break deserialization + # (`type` is a permissive str; BaseModel retains extra keys). + client = LandingAIADE(apikey=APIKEY) + body: Dict[str, Any] = { + "markdown": "x", + "metadata": {"req_id": "r1", "job_id": "j1", "model_version": "m", "page_count": 1, "failed_pages": []}, + "structure": { + "type": "document", + "children": [ + {"type": "page", "page": 0, "span": [0, 5], "future_field": 7, "children": [ + {"type": "some_future_kind", "id": "e9", "span": [0, 5]}, + ]}, + ], + }, + "grounding": {"type": "document", "children": []}, + } + respx.post("https://api.ade.landing.ai/v2/parse").mock(return_value=httpx.Response(200, json=body)) + result = client.v2.parse(document=b"pdf") + assert result.structure is not None + page = result.structure.children[0] + assert page.children[0].type == "some_future_kind" + # the unknown `future_field` key must survive deserialization (extra="allow") + assert page.to_dict()["future_field"] == 7 + + @respx.mock def test_parse_sync_504_raises_v2_sync_timeout() -> None: client = LandingAIADE(apikey=APIKEY, max_retries=0) diff --git a/tests/test_v2_types.py b/tests/test_v2_types.py index 1ea4cb0..faf7082 100644 --- a/tests/test_v2_types.py +++ b/tests/test_v2_types.py @@ -76,10 +76,12 @@ def test_extract_result_parses_nested_metadata() -> None: assert r.metadata.credit_usage == 0.0 # default -def test_parse_response_tolerates_loose_shape() -> None: +def test_parse_response_builds_from_dicts() -> None: + # `structure`/`grounding` are typed trees; `metadata` (with nested `billing`) + # and the whole response build cleanly from plain dicts. r = V2ParseResponse( markdown="# hi", - structure=[{"type": "text"}], + structure={"type": "document", "children": [{"type": "page", "page": 0, "span": [0, 4]}]}, # type: ignore[arg-type] metadata={ # type: ignore[arg-type] "req_id": "r1", "job_id": "j1", @@ -90,6 +92,7 @@ def test_parse_response_tolerates_loose_shape() -> None: }, ) assert r.markdown == "# hi" + assert r.structure is not None and r.structure.children[0].page == 0 assert r.metadata is not None and r.metadata.billing is not None assert r.metadata.billing.total_credits == 3.0