diff --git a/pyproject.toml b/pyproject.toml index 7848ecd..3b78975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "fairscape-cli" -version = "1.2.8" +version = "1.2.9" description = "A utility for packaging objects and validating metadata for FAIRSCAPE" readme = "README.md" requires-python = ">=3.8" @@ -39,9 +39,8 @@ dependencies = [ "prettytable>=3.9.0", "jsonschema>=4.20.0", "sqids>=0.4.1", - "fairscape-models[schema-infer]>=1.2.0", + "fairscape-models[schema-tabular]>=1.2.2", "pyyaml", - "h5py", "frictionless>=5.0,<6.0", "beautifulsoup4", "pandas", @@ -101,3 +100,8 @@ test = [ shacl = [ "pyshacl>=0.27" ] +# Optional per-format schema inference/validation. +schema-hdf5 = ["fairscape-models[schema-hdf5]"] +schema-signal = ["fairscape-models[schema-signal]"] +schema-image = ["fairscape-models[schema-image]"] +schema-all = ["fairscape-models[schema-infer]"] diff --git a/src/fairscape_cli/commands/schema_commands.py b/src/fairscape_cli/commands/schema_commands.py index 867d570..45b6815 100644 --- a/src/fairscape_cli/commands/schema_commands.py +++ b/src/fairscape_cli/commands/schema_commands.py @@ -5,12 +5,11 @@ ValidationError ) -from fairscape_models.schema import Schema - from fairscape_cli.config import DEFAULT_CONTEXT, DEFAULT_SCHEMA_TYPE from fairscape_cli.models import ReadROCrateMetadata, AppendCrate from fairscape_cli.models.schema import ( + TabularSchema, infer_schema, validate_schema, load_schema, @@ -52,7 +51,7 @@ def create_tabular_schema( """Initialize a Tabular Schema. """ try: - schema_model = Schema.model_validate({ + schema_model = TabularSchema.model_validate({ "@id": guid or generate_schema_guid(name), "@context": DEFAULT_CONTEXT, "@type": DEFAULT_SCHEMA_TYPE, @@ -289,7 +288,8 @@ def validate(ctx, schema, data): def infer_schema_rocrate(ctx, name, description, guid, input_file, rocrate_path, schema_file): """Infer a schema from a file and optionally append it to an RO-Crate. - INPUT_FILE: File to infer schema from (CSV, TSV, Parquet, or HDF5) + INPUT_FILE: File to infer schema from. Supported: CSV, TSV, Parquet + (tabular); HDF5 (.h5/.hdf5); WFDB header (.hea); DICOM (.dcm). SCHEMA_FILE: Path to save the schema file """ try: @@ -322,6 +322,12 @@ def infer_schema_rocrate(ctx, name, description, guid, input_file, rocrate_path, except ValueError as e: click.echo(f"Error with file type: {str(e)}") ctx.exit(code=1) + except ImportError as e: + click.echo( + f"Missing optional dependency for this file type ({str(e)}). " + "Install with: pip install 'fairscape-models[schema-infer]'" + ) + ctx.exit(code=1) except Exception as e: click.echo(f"Error inferring schema: {str(e)}") ctx.exit(code=1) diff --git a/src/fairscape_cli/datasheet_builder/rocrate/datasheet_generator.py b/src/fairscape_cli/datasheet_builder/rocrate/datasheet_generator.py index 945de18..a5ce7b3 100644 --- a/src/fairscape_cli/datasheet_builder/rocrate/datasheet_generator.py +++ b/src/fairscape_cli/datasheet_builder/rocrate/datasheet_generator.py @@ -30,6 +30,7 @@ SUBCRATE_MAPPING_CONFIGURATION, PREVIEW_MAPPING_CONFIGURATION ) +from fairscape_models.conversion.mapping.subcrate_utils import enrich_preview_computations from fairscape_cli.utils.rocrate_helpers import get_root_entity @@ -394,6 +395,7 @@ def process_subcrates(self): ) preview = converter.convert() + enrich_preview_computations(preview, subcrate, self.global_metadata_index) preview_html = self.preview_generator.generate(preview, self.published) diff --git a/src/fairscape_cli/datasheet_builder/rocrate/section_generators.py b/src/fairscape_cli/datasheet_builder/rocrate/section_generators.py index 35609ef..74306ce 100644 --- a/src/fairscape_cli/datasheet_builder/rocrate/section_generators.py +++ b/src/fairscape_cli/datasheet_builder/rocrate/section_generators.py @@ -226,6 +226,7 @@ def _prepare_subcrate_context(self, item: SubCrateItem, published: bool) -> Dict 'computations_count': details.computations_count, 'schemas_count': details.schemas_count, 'other_count': details.other_count, + 'datasets_with_provenance_count': details.datasets_with_provenance_count, # Format and access summaries - convert to dict for template 'file_formats': dict(details.file_formats) if details.file_formats else {}, @@ -327,7 +328,8 @@ def _prepare_items(self, items: List) -> List[Dict[str, Any]]: 'content_status': item.content_status or "Not specified", 'experimentType': getattr(item, 'experimentType', None) or "N/A", 'manufacturer': getattr(item, 'manufacturer', None) or "N/A", - 'schema_properties': getattr(item, 'schema_properties', None) + 'schema_properties': getattr(item, 'schema_properties', None), + 'computation_details': getattr(item, 'computation_details', None) } prepared.append(item_context) return prepared diff --git a/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.html.j2 b/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.html.j2 index eea851b..1b27103 100644 --- a/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.html.j2 +++ b/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.html.j2 @@ -104,6 +104,94 @@ line-height: 1.3; box-sizing: border-box; } + .node-info-button { + position: absolute; + top: 0; + right: 0; + width: 22px; + height: 22px; + color: #333; + border: none; + font-size: 12px; + font-weight: bold; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + z-index: 10; + padding: 0; + margin: 0; + border-bottom-left-radius: 4px; + opacity: 0.8; + transition: opacity 0.2s ease; + } + .node-info-button:hover { + opacity: 1; + } + .node-info-popover { + position: fixed; + max-height: 70vh; + overflow-y: auto; + background: #fff; + border: 1px solid #ccc; + border-radius: 4px; + box-shadow: 0 4px 12px rgba(0,0,0,0.15); + padding: 10px 12px; + font-size: 13px; + color: #333; + text-align: left; + z-index: 50; + box-sizing: border-box; + cursor: default; + user-select: text; + } + .node-info-popover h4 { + margin: 0 22px 8px 0; + font-size: 1.1em; + color: #000; + border-bottom: 1px solid #eee; + padding-bottom: 4px; + word-break: break-word; + } + .node-info-popover .popover-section { + margin-bottom: 8px; + padding-bottom: 8px; + } + .node-info-popover .popover-section:not(:last-child) { + border-bottom: 1px dotted #eee; + } + .node-info-popover .prop-item { + display: flex; + margin: 4px 0; + line-height: 1.4; + } + .node-info-popover .prop-key { + font-weight: bold; + min-width: 100px; + flex-shrink: 0; + margin-right: 8px; + color: #555; + } + .node-info-popover .prop-value { + word-break: break-word; + } + .node-info-popover .popover-close { + position: absolute; + top: 6px; + right: 6px; + width: 20px; + height: 20px; + border: none; + background: transparent; + font-size: 16px; + line-height: 1; + color: #888; + cursor: pointer; + padding: 0; + } + .node-info-popover .popover-close:hover { + color: #333; + } .svg-layer { position: absolute; top: 0; diff --git a/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.js b/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.js index e4603d1..36188bf 100644 --- a/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.js +++ b/src/fairscape_cli/datasheet_builder/templates/evidence_graph/evidence_graph.js @@ -530,9 +530,33 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; } } + function formatPropertyValue(value) { + if (value === null || value === undefined) return ""; + if (Array.isArray(value)) { + return value.map(formatPropertyValue).filter(Boolean).join(", "); + } + if (typeof value === "object") { + return value["@id"] || value.name || JSON.stringify(value); + } + return String(value); + } + + // Properties from the entity's source data worth showing in the info + // popover, beyond the ones the popover displays explicitly. + function getDisplayableProperties(sourceData) { + const EXCLUDED = new Set(["@id", "@type", "@context", "name", "label", "description", "count"]); + const props = {}; + Object.entries(sourceData || {}).forEach(([key, value]) => { + if (key.startsWith("_") || EXCLUDED.has(key)) return; + const formatted = formatPropertyValue(value); + if (formatted) props[key] = formatted; + }); + return props; + } + const { createElement, useState, useCallback, useEffect, useRef, memo } = React; - const EvidenceNode = memo(({ nodeData, onClick }) => { + const EvidenceNode = memo(({ nodeData, onClick, onInfoClick }) => { const { id, type, label, displayName, description, expandable, x, y, width, height } = nodeData; const nodeColor = getNodeColor(type); // DatasetGroup (graph-condensation node) and DatasetCollection (synthetic UI grouping @@ -548,6 +572,14 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; } }, [id, expandable, onClick]); + const handleInfoClick = useCallback((event) => { + // Keep the 'i' click from triggering node expansion. + event.stopPropagation(); + if (onInfoClick) { + onInfoClick(id, event.currentTarget.getBoundingClientRect()); + } + }, [id, onInfoClick]); + const style = { left: `${x}px`, top: `${y}px`, @@ -567,11 +599,62 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; createElement('div', { key: 'header', className: 'node-header', style: { backgroundColor: nodeColor } }, displayType), createElement('div', { key: 'content', className: 'node-content' }, createElement('div', { style: { width: '100%', textAlign: 'center' } }, displayName) - ) + ), + createElement('button', { + key: 'info', + className: 'node-info-button', + style: { backgroundColor: nodeColor }, + onClick: handleInfoClick, + onMouseDown: (e) => e.stopPropagation(), + 'aria-label': `Details for ${displayName || label || id}` + }, 'i') ]) ); }); + // Details panel opened by a node's 'i' button — the standalone equivalent + // of the web client's Tippy tooltip on EvidenceNode. + const NodeInfoPopover = ({ node, anchorRect, onClose }) => { + const { id, type, label, displayName, description } = node; + const sourceData = node._sourceData || {}; + const otherProps = getDisplayableProperties(sourceData); + + const PANEL_WIDTH = 360; + let left = anchorRect.right + 8; + if (left + PANEL_WIDTH > window.innerWidth - 10) { + left = Math.max(10, anchorRect.left - PANEL_WIDTH - 8); + } + const top = Math.max(10, Math.min(anchorRect.top, window.innerHeight - 200)); + + const propRow = (key, value) => createElement('div', { key: key, className: 'prop-item' }, [ + createElement('span', { key: 'k', className: 'prop-key' }, `${key}:`), + createElement('span', { key: 'v', className: 'prop-value' }, value) + ]); + + const metaRows = [ + propRow('@id', id), + propRow('@type', type), + ]; + if (description) metaRows.push(propRow('description', description)); + if (type === 'DatasetCollection' && sourceData.count !== undefined) { + metaRows.push(propRow('Items', String(sourceData.count))); + } + + return createElement('div', { + className: 'node-info-popover', + style: { left: `${left}px`, top: `${top}px`, width: `${PANEL_WIDTH}px` }, + onMouseDown: (e) => e.stopPropagation(), + onClick: (e) => e.stopPropagation() + }, [ + createElement('button', { key: 'close', className: 'popover-close', onClick: onClose, 'aria-label': 'Close details' }, '×'), + createElement('h4', { key: 'title' }, label || displayName || 'Node Details'), + createElement('div', { key: 'meta', className: 'popover-section' }, metaRows), + Object.keys(otherProps).length > 0 && createElement('div', { key: 'props', className: 'popover-section' }, + Object.entries(otherProps).map(([key, value]) => propRow(key, value)) + ) + ]); + }; + const Edge = memo(({ edgeData, sourceNode, targetNode }) => { if (!sourceNode || !targetNode || sourceNode.x === undefined || targetNode.x === undefined) { return null; @@ -659,6 +742,7 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; const [translate, setTranslate] = useState({ x: 0, y: 0 }); const [isDragging, setIsDragging] = useState(false); const [startDragPos, setStartDragPos] = useState({ x: 0, y: 0 }); + const [infoPopover, setInfoPopover] = useState(null); // { nodeId, anchorRect } const rootRef = useRef(null); @@ -709,7 +793,15 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; }, [graphData, applyLayout]); + const handleInfoClick = useCallback((nodeId, anchorRect) => { + // Toggle: clicking the same node's 'i' again closes the popover. + setInfoPopover(current => + current && current.nodeId === nodeId ? null : { nodeId, anchorRect } + ); + }, []); + const handleNodeClick = useCallback((nodeId) => { + setInfoPopover(null); const clickedNodeIndex = nodes.findIndex(n => n.id === nodeId); if (clickedNodeIndex === -1) return; @@ -775,9 +867,10 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; const handleMouseDown = useCallback((event) => { if (event.button !== 0) return; - if (event.target.closest('.controls-container') || event.target.closest('.node-wrapper')) { + if (event.target.closest('.controls-container') || event.target.closest('.node-wrapper') || event.target.closest('.node-info-popover')) { return; } + setInfoPopover(null); setIsDragging(true); setStartDragPos({ x: event.clientX - translate.x, y: event.clientY - translate.y }); if(rootRef.current) rootRef.current.classList.add('dragging'); @@ -864,12 +957,25 @@ const evidenceGraphData = window.__EVIDENCE_GRAPH_DATA__; nodes.map(node => createElement(EvidenceNode, { key: node.id, nodeData: node, - onClick: handleNodeClick + onClick: handleNodeClick, + onInfoClick: handleInfoClick })) ) ] ) ), + (() => { + if (!infoPopover) return null; + const infoNode = nodeMap.get(infoPopover.nodeId); + if (!infoNode) return null; + return createElement(NodeInfoPopover, { + key: 'info-popover', + node: infoNode, + anchorRect: infoPopover.anchorRect, + onClose: () => setInfoPopover(null) + }); + })(), + isLoading && createElement('div', { key: 'loading', className: 'loading-overlay' }, 'Loading...'), createElement('div', {key: 'controls', className: 'controls-container'}, [ diff --git a/src/fairscape_cli/datasheet_builder/templates/preview.html b/src/fairscape_cli/datasheet_builder/templates/preview.html index f715102..a3b9bfa 100644 --- a/src/fairscape_cli/datasheet_builder/templates/preview.html +++ b/src/fairscape_cli/datasheet_builder/templates/preview.html @@ -224,10 +224,42 @@ .schema-details-container { padding: 0; } - .toggle-schema { + .toggle-schema, + .toggle-computation { cursor: pointer; color: var(--color-accent); text-decoration: underline; + white-space: nowrap; + } + .computation-meta { + margin-bottom: 10px; + } + .computation-meta div { + margin-bottom: 4px; + } + .io-heading { + font-weight: 600; + margin: 12px 0 6px; + } + .io-crate { + color: var(--color-text-muted); + font-size: 0.85em; + } + .mono-id { + font-family: monospace; + font-size: 0.85em; + word-break: break-all; + } + .computation-command { + display: block; + padding: 8px 12px; + background-color: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + font-family: monospace; + font-size: 0.85em; + white-space: pre-wrap; + word-break: break-all; } .properties-table { margin: 0; @@ -639,6 +671,156 @@

RO-Crate Summary

No {{ tab_id }} found in this RO-Crate.

{% endif %} + {% endmacro %} {% macro render_computation_table(items, tab_id, is_active) %} +
+ {% if items %} + + + + + + + + + + + + {% for item in items %} + + + + + + + + {% if item.computation_details %} {% set details = + item.computation_details %} + + + + {% endif %} {% endfor %} + +
NameDescriptionAccessDate CreatedDetails
{{ item.name }} + {{ item.description_display }} + {{ item.content_status | safe }}{{ item.date }} + {% if item.computation_details %} + Show Details + {% else %} No details found {% endif %} +
+ {% else %} +

No {{ tab_id }} found in this RO-Crate.

+ {% endif %} +
{% endmacro %} {% macro render_other_table(items, tab_id, is_active) %}
RO-Crate Summary 'Description', 'Access', 'Release Date'], 'date') }} {{ render_table(software, 'software', not datasets and software, ['Name', 'Description', 'Access', 'Release Date'], 'date') }} {{ - render_table(computations, 'computations', not datasets and not software - and computations, ['Name', 'Description', 'Access', 'Date Created'], - 'date') }} {{ render_table(samples, 'samples', not datasets and not + render_computation_table(computations, 'computations', not datasets and + not software and computations) }} {{ render_table(samples, 'samples', not datasets and not software and not computations and samples, ['Name', 'Description', 'Identifier', 'Date Created'], 'date') }} {{ render_table(experiments, 'experiments', not datasets and not software and not computations and not @@ -779,6 +960,23 @@

RO-Crate Summary

} }); }); + + document.querySelectorAll(".toggle-computation").forEach((button) => { + button.addEventListener("click", function () { + const computationId = this.getAttribute("data-computation-id"); + const detailsRow = document.getElementById( + `computation-details-row-${computationId}` + ); + + if (detailsRow.style.display === "table-row") { + detailsRow.style.display = "none"; + this.textContent = "Show Details"; + } else { + detailsRow.style.display = "table-row"; + this.textContent = "Hide Details"; + } + }); + }); }); diff --git a/src/fairscape_cli/datasheet_builder/templates/sections/subcrates.html b/src/fairscape_cli/datasheet_builder/templates/sections/subcrates.html index 364f9f6..8de0398 100644 --- a/src/fairscape_cli/datasheet_builder/templates/sections/subcrates.html +++ b/src/fairscape_cli/datasheet_builder/templates/sections/subcrates.html @@ -24,7 +24,8 @@

Datasets at a glance

Dataset Size Files - Computations + Activities + Datasets (withProv) Schemas Formats @@ -43,7 +44,8 @@

Datasets at a glance

{{ subcrate.size if subcrate.size else "—" }} {{ subcrate.files_count }} - {{ subcrate.computations_count }} + {{ subcrate.computations_count + subcrate.experiments_count }} + {{ subcrate.datasets_with_provenance_count }} / {{ subcrate.files_count }} {{ subcrate.schemas_count }} {% if subcrate.file_formats %}{% for fmt in subcrate.file_formats.keys()|list %}{{ fmt }}{% if not loop.last %}, {% endif %}{% endfor %}{% else %}—{% endif %} @@ -331,6 +333,11 @@

Content Summary

{% for pattern in subcrate.computation_patterns %}
{{ pattern | safe }}
{% endfor %}
{% endif %} + {% if subcrate.computations_count %} +
+ Full computation details (inputs, outputs, identifiers) available in the {% if subcrate.preview_url %}full dataset{% else %}full dataset{% endif %}. +
+ {% endif %}
Schemas: {{ subcrate.schemas_count }} diff --git a/src/fairscape_cli/models/schema/__init__.py b/src/fairscape_cli/models/schema/__init__.py index 99dc230..58721ed 100644 --- a/src/fairscape_cli/models/schema/__init__.py +++ b/src/fairscape_cli/models/schema/__init__.py @@ -1,21 +1,32 @@ -from typing import List, Optional +""" +fairscape_cli.models.schema — CLI-facing schema surface. -from fairscape_models.schema import Schema, Property +The typed schema hierarchy, per-format infer/validate, dispatch, and legacy +loading live in `fairscape_models.schema`; this package re-exports them together +with the CLI-only property builders and the file writer. +""" -from fairscape_cli.models.schema.core import ( - SchemaHandler, +from fairscape_models.schema import ( + Schema, + Property, + TabularSchema, + HDF5Schema, + SignalSchema, + ImageSchema, + NDArraySchema, ValidationErrorRecord, - frictionless_type_to_json_schema, generate_schema_guid, SOURCE_TYPE_KEY, - register_handler, - get_handler, - supported_extensions, + frictionless_type_to_json_schema, load_schema, - write_schema, normalize_schema_document, + schema_class_for_file, + infer_schema, + validate_schema, ) + from fairscape_cli.models.schema.properties import ( + write_schema, DatatypeEnum, Items, NullProperty, @@ -28,35 +39,22 @@ ClickAppendProperty, ) -# Importing handlers registers the built-in csv/tsv, hdf5, and parquet handlers. -import fairscape_cli.models.schema.handlers # noqa: F401,E402 - - -def infer_schema(filepath: str, name: str, description: str, - guid: Optional[str] = None) -> Schema: - """Infer a canonical Schema from a data file, dispatching by extension.""" - return get_handler(filepath).infer(filepath, name, description, guid=guid) - - -def validate_schema(schema: Schema, filepath: str) -> List[ValidationErrorRecord]: - """Validate a data file against a canonical Schema, dispatching by extension.""" - return get_handler(filepath).validate(schema, filepath) - - __all__ = [ 'Schema', 'Property', - 'SchemaHandler', + 'TabularSchema', + 'HDF5Schema', + 'SignalSchema', + 'ImageSchema', + 'NDArraySchema', 'ValidationErrorRecord', - 'frictionless_type_to_json_schema', 'generate_schema_guid', 'SOURCE_TYPE_KEY', - 'register_handler', - 'get_handler', - 'supported_extensions', + 'frictionless_type_to_json_schema', 'load_schema', 'write_schema', 'normalize_schema_document', + 'schema_class_for_file', 'infer_schema', 'validate_schema', 'DatatypeEnum', diff --git a/src/fairscape_cli/models/schema/core.py b/src/fairscape_cli/models/schema/core.py deleted file mode 100644 index 468a1dd..0000000 --- a/src/fairscape_cli/models/schema/core.py +++ /dev/null @@ -1,252 +0,0 @@ -import json -import pathlib -import re -from abc import ABC, abstractmethod -from typing import ClassVar, Dict, List, Optional, Tuple, Union - -from pydantic import BaseModel - -from fairscape_models.schema import Schema, Property - -from fairscape_cli.config import ( - DEFAULT_CONTEXT, - DEFAULT_SCHEMA_TYPE, - NAAN, -) -from fairscape_cli.models.guid_utils import GenerateDatetimeSquid -from fairscape_cli.utils.serialization import model_dump_pruned, write_json_atomic - -CANONICAL_TYPES = {'integer', 'number', 'string', 'array', 'boolean', 'object'} -SOURCE_TYPE_KEY = 'source-type' - - -def frictionless_type_to_json_schema(field_type: str) -> str: - """Convert Frictionless types to JSON Schema types""" - type_mapping = { - 'string': 'string', - 'integer': 'integer', - 'number': 'number', - 'boolean': 'boolean', - 'date': 'string', - 'datetime': 'string', - 'year': 'integer', - 'yearmonth': 'string', - 'duration': 'string', - 'geopoint': 'array', - 'geojson': 'object', - 'array': 'array', - 'object': 'object', - 'time': 'string' - } - return type_mapping.get(field_type, 'string') - - -def generate_schema_guid(name: str) -> str: - """Generate a unique identifier for a schema""" - prefix = f"schema-{name.lower().replace(' ', '-')}" - return f"ark:{NAAN}/{prefix}-{GenerateDatetimeSquid()}" - - -# --------------------------------------------------------------------------- # -# Validation error record + frictionless adapter -# --------------------------------------------------------------------------- # - -class ValidationErrorRecord(BaseModel): - message: str - failed_keyword: str = "error" - error_type: str = "ValidationError" - field: Optional[str] = None - row: Optional[int] = None - path: Optional[str] = None - - @property - def location(self) -> str: - parts = [ - part for part in ( - self.path, - f"row {self.row}" if self.row is not None else None, - self.field, - ) if part - ] - return " / ".join(parts) or "-" - - -def frictionless_error_to_record(error, path: Optional[str] = None) -> ValidationErrorRecord: - """Convert a frictionless error object to a ValidationErrorRecord. - - Frictionless 5.x exposes row/field attributes under both snake_case and - camelCase depending on error class and minor version, so check both. - """ - row = getattr(error, 'row_number', None) - if row is None: - row = getattr(error, 'rowNumber', None) - field = getattr(error, 'field_name', None) - if field is None: - field = getattr(error, 'fieldName', None) - return ValidationErrorRecord( - message=error.message, - row=row, - field=field, - failed_keyword=getattr(error, 'type', 'error') or 'error', - path=path, - ) - - -# --------------------------------------------------------------------------- # -# Handler contract + registry -# --------------------------------------------------------------------------- # - -class SchemaHandler(ABC): - """Contract for per-format schema inference and validation. - - Implementations may use any machinery internally, but infer() must return - the canonical fairscape_models Schema and validate() must consume it. - Register implementations with @register_handler('ext', ...). - """ - - extensions: ClassVar[Tuple[str, ...]] = () - - @abstractmethod - def infer(self, filepath: str, name: str, description: str, - guid: Optional[str] = None) -> Schema: - """Infer a canonical Schema from a data file.""" - - @abstractmethod - def validate(self, schema: Schema, filepath: str) -> List[ValidationErrorRecord]: - """Validate a data file against a canonical Schema.""" - - def new_schema(self, *, name: str, description: str, - properties: Dict[str, Property], required: List[str], - guid: Optional[str] = None, separator: str = ",", - header: bool = True, **extra) -> Schema: - return Schema.model_validate({ - "@id": guid or generate_schema_guid(name), - "@context": DEFAULT_CONTEXT, - "@type": DEFAULT_SCHEMA_TYPE, - "name": name, - "description": description, - "properties": properties, - "required": required, - "separator": separator, - "header": header, - **extra, - }) - - -_HANDLERS: Dict[str, SchemaHandler] = {} - - -def register_handler(*extensions: str): - """Class decorator: instantiate the handler and map each extension to it. - - Third parties can add formats by subclassing SchemaHandler and decorating - with @register_handler('myext') in code that imports fairscape_cli. - """ - def decorator(cls): - handler = cls() - for ext in extensions: - _HANDLERS[ext.lower().lstrip('.')] = handler - return cls - return decorator - - -def get_handler(filepath: str) -> SchemaHandler: - ext = pathlib.Path(filepath).suffix.lower().lstrip('.') - try: - return _HANDLERS[ext] - except KeyError: - raise ValueError( - f"Unsupported file extension '{ext}'. " - f"Supported extensions: {', '.join(supported_extensions())}" - ) - - -def supported_extensions() -> List[str]: - return sorted(_HANDLERS) - - -# --------------------------------------------------------------------------- # -# Loading (with backward-compat normalization) + writing -# --------------------------------------------------------------------------- # - -# Same index pattern the canonical Property model accepts -_INDEX_PATTERN = re.compile(r'^\d+$|^-?\d+::|^-?\d+::-?\d+$|^::-?\d+') - - -def _valid_index(value) -> bool: - if isinstance(value, int): - return True - if isinstance(value, str): - return bool(_INDEX_PATTERN.match(value)) - return False - - -def normalize_property(name: str, prop: dict, index_fallback: int) -> dict: - """Normalize a single property dict to canonical Property shape. - - Handles legacy HDF5 schema JSON where each dataset entry is a full - TabularValidationSchema dump (nested 'properties' but no 'index'), and - legacy non-canonical type strings like 'datetime'/'year'. - """ - if not isinstance(prop, dict): - return prop - - nested = prop.get('properties') - if isinstance(nested, dict) and not _valid_index(prop.get('index')): - # Legacy HDF5 dataset entry: rebuild as an object Property, dropping - # the schema-level wrapper keys the old dump carried. - return { - 'type': 'object', - 'index': index_fallback, - 'description': prop.get('description') or f"Dataset at {name}", - 'hdf5-path': name, - 'properties': { - child_name: normalize_property(child_name, child, i) - for i, (child_name, child) in enumerate(nested.items()) - }, - } - - normalized = dict(prop) - prop_type = normalized.get('type') - if prop_type is None: - normalized['type'] = 'string' - elif prop_type not in CANONICAL_TYPES: - normalized.setdefault(SOURCE_TYPE_KEY, prop_type) - normalized['type'] = frictionless_type_to_json_schema(prop_type) - - if not _valid_index(normalized.get('index')): - normalized['index'] = index_fallback - if not normalized.get('description'): - normalized['description'] = f"Column {name}" - - if isinstance(nested, dict): - normalized['properties'] = { - child_name: normalize_property(child_name, child, i) - for i, (child_name, child) in enumerate(nested.items()) - } - return normalized - - -def normalize_schema_document(data: dict) -> dict: - """Normalize a schema JSON document so legacy files validate canonically.""" - data = dict(data) - data.setdefault('@type', DEFAULT_SCHEMA_TYPE) - data.setdefault('separator', ',') - data.setdefault('header', True) - data['properties'] = { - name: normalize_property(name, prop, i) - for i, (name, prop) in enumerate(data.get('properties', {}).items()) - } - return data - - -def load_schema(path: Union[str, pathlib.Path]) -> Schema: - """Read a schema JSON file into the canonical Schema model.""" - with open(path) as f: - data = json.load(f) - return Schema.model_validate(normalize_schema_document(data)) - - -def write_schema(schema: Schema, output_file: Union[str, pathlib.Path]) -> None: - """Write a canonical Schema to a JSON file.""" - write_json_atomic(output_file, model_dump_pruned(schema, by_alias=True)) diff --git a/src/fairscape_cli/models/schema/handlers/__init__.py b/src/fairscape_cli/models/schema/handlers/__init__.py deleted file mode 100644 index c50d9b6..0000000 --- a/src/fairscape_cli/models/schema/handlers/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Importing these modules registers the built-in handlers as a side effect. -from fairscape_cli.models.schema.handlers import tabular -from fairscape_cli.models.schema.handlers import hdf5 -from fairscape_cli.models.schema.handlers import parquet - -__all__ = ['tabular', 'hdf5', 'parquet'] diff --git a/src/fairscape_cli/models/schema/handlers/hdf5.py b/src/fairscape_cli/models/schema/handlers/hdf5.py deleted file mode 100644 index aa0a92c..0000000 --- a/src/fairscape_cli/models/schema/handlers/hdf5.py +++ /dev/null @@ -1,131 +0,0 @@ -from typing import List, Optional - -import h5py -import pandas as pd -from frictionless import Resource, describe - -from fairscape_models.schema import Schema, Property - -from fairscape_cli.models.schema.core import ( - SchemaHandler, - ValidationErrorRecord, - frictionless_error_to_record, - frictionless_type_to_json_schema, - register_handler, - SOURCE_TYPE_KEY, -) -from fairscape_cli.models.schema.handlers.tabular import build_frictionless_schema - - -def dataset_to_dataframe(dataset: h5py.Dataset) -> pd.DataFrame: - """Convert an HDF5 dataset to a pandas DataFrame""" - data = dataset[()] - - if dataset.dtype.fields: # Structured array - return pd.DataFrame(data) - elif len(dataset.shape) > 1: # Multi-dimensional array - n_cols = dataset.shape[1] - columns = [f"column_{i}" for i in range(n_cols)] - return pd.DataFrame(data, columns=columns) - else: # 1D array - return pd.DataFrame(data, columns=['value']) - - -@register_handler('h5', 'hdf5') -class HDF5Handler(SchemaHandler): - extensions = ('h5', 'hdf5') - - def infer(self, filepath: str, name: str, description: str, - guid: Optional[str] = None) -> Schema: - properties = {} - - with h5py.File(filepath, 'r') as f: - def process_group(group, parent_path=""): - for key, item in group.items(): - path = f"{parent_path}/{key}" if parent_path else key - - if isinstance(item, h5py.Dataset): - try: - df = dataset_to_dataframe(item) - resource = describe(df) - - columns = {} - for i, field in enumerate(resource.schema.fields): - json_schema_type = frictionless_type_to_json_schema(field.type) - extra = {} - if json_schema_type != field.type: - extra[SOURCE_TYPE_KEY] = field.type - columns[field.name] = Property( - type=json_schema_type, - description=field.description or f"Column {field.name}", - index=i, - **extra, - ) - - # The int index is a placeholder required by the - # canonical Property index validator; validation - # keys datasets on the path, never this index. - properties[path] = Property( - type='object', - index=len(properties), - description=f"Dataset at {path}", - properties=columns, - **{'hdf5-path': path}, - ) - - except Exception as e: - print(f"Warning: Could not process dataset {path}: {str(e)}") - - elif isinstance(item, h5py.Group): - process_group(item, path) - - process_group(f) - - return self.new_schema( - name=name, - description=description, - guid=guid, - properties=properties, - required=list(properties.keys()), - ) - - def validate(self, schema: Schema, filepath: str) -> List[ValidationErrorRecord]: - errors = [] - - with h5py.File(filepath, 'r') as f: - for path, prop in schema.properties.items(): - if prop.type != 'object' or not prop.properties: - continue - try: - dataset = f[path] - if isinstance(dataset, h5py.Dataset): - df = dataset_to_dataframe(dataset) - frictionless_schema = build_frictionless_schema(prop.properties) - resource = Resource(data=df, schema=frictionless_schema) - report = resource.validate() - - for task in report.tasks: - for error in task.errors: - # Skip string type errors: h5py returns bytes - # for string data, which frictionless flags as - # a type mismatch against string fields. - if (hasattr(error, 'type') and error.type == 'type-error' and - hasattr(error, 'note') and 'type is "string' in error.note): - continue - - errors.append(frictionless_error_to_record(error, path=path)) - - except KeyError: - errors.append(ValidationErrorRecord( - message=f"Dataset {path} not found", - failed_keyword="required", - path=path, - )) - except Exception as e: - errors.append(ValidationErrorRecord( - message=f"Error validating dataset {path}: {str(e)}", - failed_keyword="format", - path=path, - )) - - return errors diff --git a/src/fairscape_cli/models/schema/handlers/parquet.py b/src/fairscape_cli/models/schema/handlers/parquet.py deleted file mode 100644 index be77c73..0000000 --- a/src/fairscape_cli/models/schema/handlers/parquet.py +++ /dev/null @@ -1,111 +0,0 @@ -from typing import List, Optional - -import pyarrow as pa -import pyarrow.parquet as pq - -from fairscape_models.schema import Schema, Property - -from fairscape_cli.models.schema.core import ( - SchemaHandler, - ValidationErrorRecord, - register_handler, - SOURCE_TYPE_KEY, -) - - -def arrow_type_to_json_schema(arrow_type: pa.DataType) -> str: - """Map an Arrow type to one of the six canonical JSON Schema types.""" - if pa.types.is_boolean(arrow_type): - return 'boolean' - if pa.types.is_integer(arrow_type): - return 'integer' - if pa.types.is_floating(arrow_type) or pa.types.is_decimal(arrow_type): - return 'number' - if (pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type) - or pa.types.is_fixed_size_list(arrow_type)): - return 'array' - if pa.types.is_struct(arrow_type) or pa.types.is_map(arrow_type): - return 'object' - # string/binary/timestamp/date/time/duration/dictionary/... - return 'string' - - -@register_handler('parquet') -class ParquetHandler(SchemaHandler): - """Native Parquet handler. - - Reads the file's embedded Arrow schema directly — no full-table read and - no pandas round-trip — so exact source types (int64 vs float64, - timestamp[us] vs date32) are preserved in each Property's 'source-type'. - Validation compares the embedded schema against the declared one without - scanning rows. - """ - extensions = ('parquet',) - - def infer(self, filepath: str, name: str, description: str, - guid: Optional[str] = None) -> Schema: - arrow_schema = pq.read_schema(filepath) - - properties = {} - required_fields = [] - - for i, field in enumerate(arrow_schema): - properties[field.name] = Property( - type=arrow_type_to_json_schema(field.type), - description=f"Column {field.name}", - index=i, - **{SOURCE_TYPE_KEY: str(field.type)}, - ) - required_fields.append(field.name) - - return self.new_schema( - name=name, - description=description, - guid=guid, - properties=properties, - required=required_fields, - ) - - def validate(self, schema: Schema, filepath: str) -> List[ValidationErrorRecord]: - arrow_schema = pq.read_schema(filepath) - file_fields = {field.name: field for field in arrow_schema} - - errors = [] - - for name, prop in schema.properties.items(): - field = file_fields.get(name) - if field is None: - errors.append(ValidationErrorRecord( - message=f"Column {name} not found in parquet file", - failed_keyword="required", - field=name, - )) - continue - - actual_source = str(field.type) - actual_canonical = arrow_type_to_json_schema(field.type) - declared_source = (prop.model_extra or {}).get(SOURCE_TYPE_KEY) - - if declared_source is not None and declared_source == actual_source: - continue - # Fall back to comparing canonical categories so hand-authored - # schemas (no source-type) or compatible physical-type rewrites - # still validate. - if prop.type != actual_canonical: - declared = declared_source or prop.type - errors.append(ValidationErrorRecord( - message=f"Column {name} type mismatch: schema declares {declared}, file has {actual_source}", - failed_keyword="type", - field=name, - )) - - if schema.additionalProperties is False: - for name in file_fields: - if name not in schema.properties: - errors.append(ValidationErrorRecord( - message=f"Column {name} present in parquet file but not declared in schema", - failed_keyword="additionalProperties", - field=name, - )) - - return errors diff --git a/src/fairscape_cli/models/schema/handlers/tabular.py b/src/fairscape_cli/models/schema/handlers/tabular.py deleted file mode 100644 index e4e0ffb..0000000 --- a/src/fairscape_cli/models/schema/handlers/tabular.py +++ /dev/null @@ -1,166 +0,0 @@ -import pathlib -from typing import Dict, List, Optional - -from frictionless import ( - Schema as FrictionlessSchema, - Resource, - Dialect, - Report, - describe, - fields, - formats, -) - -from fairscape_models.schema import Schema, Property - -from fairscape_cli.models.schema.core import ( - SchemaHandler, - ValidationErrorRecord, - frictionless_error_to_record, - frictionless_type_to_json_schema, - register_handler, - SOURCE_TYPE_KEY, -) - -_TYPE_TO_FRICTIONLESS_FIELD = { - 'string': fields.StringField, - 'integer': fields.IntegerField, - 'number': fields.NumberField, - 'boolean': fields.BooleanField, -} - - -def _get_either(prop_details: dict, *keys): - for key in keys: - if prop_details.get(key) is not None: - return prop_details[key] - return None - - -def build_frictionless_schema(properties: Dict[str, Property]) -> FrictionlessSchema: - """Rebuild a frictionless Schema from canonical Properties for row validation.""" - properties_input = { - name: prop.model_dump(by_alias=True, exclude_none=True) - for name, prop in properties.items() - } - - frictionless_schema_obj = FrictionlessSchema() - - sorted_prop_items = [] - spanning_array_prop_name = None - spanning_array_prop_details = None - - for name, prop_details in properties_input.items(): - index_val = prop_details.get("index") - if prop_details.get("type") == "array" and isinstance(index_val, str) and "::" in index_val: - if spanning_array_prop_name is not None: - raise ValueError("Multiple spanning array properties (index: 'X::') are not supported.") - spanning_array_prop_name = name - spanning_array_prop_details = prop_details - elif isinstance(index_val, int): - sorted_prop_items.append((name, prop_details, index_val)) - else: - sorted_prop_items.append((name, prop_details, float('inf'))) - - sorted_prop_items.sort(key=lambda x: x[2]) - - for name, prop_details, _ in sorted_prop_items: - field_class = _TYPE_TO_FRICTIONLESS_FIELD.get(prop_details.get('type', 'string'), fields.StringField) - - constraints = {} - if 'minimum' in prop_details: constraints['minimum'] = prop_details['minimum'] - if 'maximum' in prop_details: constraints['maximum'] = prop_details['maximum'] - if 'pattern' in prop_details: constraints['pattern'] = prop_details['pattern'] - if 'minLength' in prop_details: constraints['minLength'] = prop_details['minLength'] - if 'maxLength' in prop_details: constraints['maxLength'] = prop_details['maxLength'] - - if prop_details.get('type') == 'array': - field = fields.ArrayField(name=name, description=prop_details.get('description', ''), constraints=constraints) - else: - field = field_class(name=name, description=prop_details.get('description', ''), constraints=constraints) - frictionless_schema_obj.add_field(field) - - if spanning_array_prop_name and spanning_array_prop_details: - prop_name_original = spanning_array_prop_name - details = spanning_array_prop_details - item_details = details.get('items', {}) - item_type = item_details.get('type', 'number') - item_field_class = _TYPE_TO_FRICTIONLESS_FIELD.get(item_type, fields.NumberField) - - # min/max item counts appear as 'min-items'/'max-items' (canonical alias) - # or 'minItems'/'maxItems' (CLI property models) depending on origin - num_items = _get_either(details, 'min-items', 'minItems') - max_items = _get_either(details, 'max-items', 'maxItems') - if num_items is None or num_items != max_items: - raise ValueError(f"Spanning array '{prop_name_original}' must have equal and defined minItems and maxItems.") - - for i in range(num_items): - field_name_for_frictionless = f"{prop_name_original}_{i}" # e.g., embed_0, embed_1, ... - field = item_field_class(name=field_name_for_frictionless, description=f"Element {i} of {prop_name_original}") - frictionless_schema_obj.add_field(field) - - return frictionless_schema_obj - - -@register_handler('csv', 'tsv') -class TabularHandler(SchemaHandler): - extensions = ('csv', 'tsv') - - def infer(self, filepath: str, name: str, description: str, - guid: Optional[str] = None) -> Schema: - ext = pathlib.Path(filepath).suffix.lower().lstrip('.') - separator = '\t' if ext == 'tsv' else ',' - - resource = describe(filepath) - - properties = {} - required_fields = [] - - for i, field in enumerate(resource.schema.fields): - json_schema_type = frictionless_type_to_json_schema(field.type) - extra = {} - if json_schema_type != field.type: - extra[SOURCE_TYPE_KEY] = field.type - - properties[field.name] = Property( - type=json_schema_type, - description=field.description or f"Column {field.name}", - index=i, - **extra, - ) - required_fields.append(field.name) - - return self.new_schema( - name=name, - description=description, - guid=guid, - properties=properties, - required=required_fields, - separator=separator, - header=True, - ) - - def validate(self, schema: Schema, filepath: str) -> List[ValidationErrorRecord]: - frictionless_schema = build_frictionless_schema(schema.properties) - - file_dialect = Dialect() - file_dialect.header = schema.header if schema.header is not None else True - - if schema.separator: - csv_control = formats.csv.CsvControl(delimiter=schema.separator) - file_dialect.add_control(csv_control) - - resource = Resource( - path=filepath, - schema=frictionless_schema, - dialect=file_dialect, - ) - - report: Report = resource.validate() - errors_list = [] - if not report.valid: - for task in report.tasks: - for error_detail in task.errors: - errors_list.append(frictionless_error_to_record(error_detail)) - - return errors_list diff --git a/src/fairscape_cli/models/schema/properties.py b/src/fairscape_cli/models/schema/properties.py index 784c938..00f5afd 100644 --- a/src/fairscape_cli/models/schema/properties.py +++ b/src/fairscape_cli/models/schema/properties.py @@ -9,13 +9,18 @@ model_validator, ) -from fairscape_models.schema import Property +from fairscape_models.schema import Schema, Property, load_schema -from fairscape_cli.models.schema.core import load_schema, write_schema from fairscape_cli.models.schema.utils import ( PropertyNameException, ColumnIndexException, ) +from fairscape_cli.utils.serialization import model_dump_pruned, write_json_atomic + + +def write_schema(schema: Schema, output_file: Union[str, pathlib.Path]) -> None: + """Write a schema to a JSON file (canonical aliased form, empty fields pruned).""" + write_json_atomic(output_file, model_dump_pruned(schema, by_alias=True)) class DatatypeEnum(str, Enum): diff --git a/src/fairscape_cli/utils/build_utils.py b/src/fairscape_cli/utils/build_utils.py index 7eab8e1..7cfb4ed 100644 --- a/src/fairscape_cli/utils/build_utils.py +++ b/src/fairscape_cli/utils/build_utils.py @@ -280,6 +280,7 @@ def process_preview(crate_path: Path, published: bool = False) -> bool: from fairscape_models.rocrate import ROCrateV1_2 from fairscape_models.conversion.converter import ROCToTargetConverter from fairscape_models.conversion.mapping.FairscapeDatasheet import PREVIEW_MAPPING_CONFIGURATION + from fairscape_models.conversion.mapping.subcrate_utils import enrich_preview_computations from fairscape_cli.datasheet_builder.rocrate.section_generators import PreviewGenerator from jinja2 import Environment, FileSystemLoader @@ -308,6 +309,13 @@ def process_preview(crate_path: Path, published: bool = False) -> bool: ) preview = converter.convert() + # Standalone crate: a per-crate index (no rocrateName stamps) still resolves + # input/output names and formats from within this crate. + local_index = { + entity.guid: entity.model_dump() + for entity in crate.metadataGraph if hasattr(entity, 'guid') + } + enrich_preview_computations(preview, crate, local_index) preview_html = preview_generator.generate(preview, published) with open(output_path, 'w', encoding='utf-8') as f: diff --git a/tests/commands/schema/test_schema_handlers.py b/tests/commands/schema/test_schema_handlers.py index dc901e1..c8fbde4 100644 --- a/tests/commands/schema/test_schema_handlers.py +++ b/tests/commands/schema/test_schema_handlers.py @@ -39,7 +39,7 @@ def sample_h5(tmp_path: pathlib.Path) -> pathlib.Path: class TestParquetHandler: - def test_infer_preserves_exact_types(self, runner, tmp_path, sample_parquet): + def test_infer_maps_frictionless_types(self, runner, tmp_path, sample_parquet): schema_path = tmp_path / "s_pq.json" result = runner.invoke(fairscape_cli_app, [ "schema", "infer", @@ -51,14 +51,16 @@ def test_infer_preserves_exact_types(self, runner, tmp_path, sample_parquet): data = json.loads(schema_path.read_text()) props = data["properties"] + # Parquet is inferred through frictionless (same engine as csv/tsv). assert props["count"]["type"] == "integer" - assert props["count"]["source-type"] == "int64" assert props["value"]["type"] == "number" - assert props["value"]["source-type"] == "double" assert props["label"]["type"] == "string" - # timestamp collapses to the canonical 'string' but records the exact source + # a datetime column collapses to canonical 'string' but keeps its source assert props["ts"]["type"] == "string" - assert props["ts"]["source-type"] == "timestamp[us]" + assert props["ts"]["source-type"] == "datetime" + # parquet has no delimiter/header row -> those fields are pruned + assert "separator" not in data or data["separator"] is None + assert data["EVI:schemaType"] == "tabular" def test_validate_success(self, runner, tmp_path, sample_parquet): schema_path = tmp_path / "s_pq.json" @@ -72,29 +74,27 @@ def test_validate_success(self, runner, tmp_path, sample_parquet): assert result.exit_code == 0, result.output assert "Validation Success" in result.output - def test_validate_type_and_missing_column(self, runner, tmp_path, sample_parquet): + def test_validate_missing_column(self, runner, tmp_path, sample_parquet): schema_path = tmp_path / "s_pq.json" runner.invoke(fairscape_cli_app, [ "schema", "infer", "--name", "PQ", "--description", "parquet schema test", str(sample_parquet), str(schema_path), ]) data = json.loads(schema_path.read_text()) - # break the count column's declared source type + add a nonexistent column - data["properties"]["count"]["type"] = "number" - data["properties"]["count"]["source-type"] = "float64" + # declare a column the file does not contain data["properties"]["missing_col"] = {"type": "string", "index": 9, "description": "not present"} schema_path.write_text(json.dumps(data)) result = runner.invoke(fairscape_cli_app, [ "schema", "validate", "--schema", str(schema_path), "--data", str(sample_parquet), ]) + # frictionless surfaces the undeclared column as a missing-label error assert result.exit_code != 0 - assert "type" in result.output - assert "required" in result.output + assert "missing_col" in result.output class TestHDF5Handler: - def test_infer_maps_datasets_to_object_properties(self, runner, tmp_path, sample_h5): + def test_infer_maps_datasets_to_structural_properties(self, runner, tmp_path, sample_h5): schema_path = tmp_path / "s_h5.json" result = runner.invoke(fairscape_cli_app, [ "schema", "infer", @@ -105,19 +105,25 @@ def test_infer_maps_datasets_to_object_properties(self, runner, tmp_path, sample assert result.exit_code == 0, result.output data = json.loads(schema_path.read_text()) + assert data["EVI:schemaType"] == "hdf5" props = data["properties"] assert "grp/ds" in props assert "root_data" in props + + # a plain dataset is described structurally (array + shape/dtype), no columns + root = props["root_data"] + assert root["type"] == "array" + assert root["shape"] == [10] + assert root["dtype"] == "float64" + assert "properties" not in root or root["properties"] is None + + # a compound (structured) dataset exposes its fields as nested properties ds = props["grp/ds"] assert ds["type"] == "object" assert ds["hdf5-path"] == "grp/ds" + assert ds["shape"] == [2] assert ds["properties"]["id"]["type"] == "integer" assert ds["properties"]["score"]["type"] == "number" - # every nested column type is one of the six canonical types - canonical = {"integer", "number", "string", "array", "boolean", "object"} - for dataset_prop in props.values(): - for col in dataset_prop["properties"].values(): - assert col["type"] in canonical def test_validate_success(self, runner, tmp_path, sample_h5): schema_path = tmp_path / "s_h5.json" @@ -211,9 +217,11 @@ def test_legacy_hdf5_schema_loads_validates_and_registers(self, runner, tmp_path schema_path = tmp_path / "legacy_hdf5_schema.json" schema_path.write_text(json.dumps(legacy, indent=2)) - # the legacy non-canonical 'year' type must normalize to 'integer' on load - from fairscape_cli.models.schema import load_schema + # a legacy per-dataset document (no EVI:schemaType) resolves to HDF5Schema + from fairscape_cli.models.schema import load_schema, HDF5Schema normalized = load_schema(schema_path) + assert isinstance(normalized, HDF5Schema) + # the legacy non-canonical 'year' type must normalize to 'integer' on load id_prop = normalized.properties["grp/ds"].properties["id"] assert id_prop.type == "integer" assert (id_prop.model_extra or {}).get("source-type") == "year"