diff --git a/.coverage b/.coverage index 7ac305f..22ba402 100644 Binary files a/.coverage and b/.coverage differ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc132ba..0c37857 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,19 @@ on: branches: [ main ] jobs: + codemeta-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check codemeta.json version matches pyproject.toml + run: | + python3 -c " + import json, tomllib + cm = json.load(open('codemeta.json'))['version'] + py = tomllib.load(open('pyproject.toml','rb'))['project']['version'] + assert cm == py, f'codemeta.json version {cm} != pyproject {py}' + " + build-and-test: runs-on: ubuntu-latest strategy: @@ -26,9 +39,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest pytest-cov flake8 + pip install pytest pytest-cov flake8 numpy pip install -r requirements.txt - pip install -e . + # schema-infer extra (frictionless/pyarrow/pandas/h5py/wfdb/pydicom) so the + # schema infer/validate tests actually run instead of importorskip-skipping, + # which is what keeps coverage above the threshold below. + pip install -e ".[schema-infer]" - name: Lint with flake8 run: | @@ -37,4 +53,4 @@ jobs: - name: Run tests and check coverage run: | - pytest --cov=fairscape_models --cov-report=term-missing --cov-fail-under=100 \ No newline at end of file + pytest --cov=fairscape_models --cov-report=term-missing --cov-fail-under=95 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index fa9a909..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,23 +0,0 @@ -# Changelog - - -## [1.1.7] - 2026-06-30 - -Corrected regex for `fairscape_models.fairscape_base.IdentifierPattern`. Added base class `fairscape_models.fairscape_base.Identifier` to all other EVI models. Corrected `Identifier.isPartOf` preprocessing for instances of `fairscape_models.fairscape_base.IdentifierValue`. - -## [1.1.6] - 2026-06-30 - -Uses the Base Class `fairscape_models.fairscape_base.Identifier` to preprocess ARK identifiers in the `guid` and `isPartOf` properties. This Base Class also uses a regex to validate ARK identifiers. - -## [1.0.29] - 2026-03-20 - -Added D4DConverter class for converting ROCrateV1_2 into LinkML D4D yaml. -Added dependency PyYaml - -Class for ROCrates now has default context set by default. Default context contained inside `fairscape_models.fairscape_base.DEFAULT_CONTEXT` is default for `fairscape_models.rocrate.ROCrateV1_2` property `context`. - -Added a bound method `generateFileElem` to `fairscape_models.rocrate.ROCrateMetadataElem` to generate the `fairscape_models.rocrate.ROCrateFileElem` required in an ROCrate. - -## [1.0.24] - 2026-02-16 - -Added a property `fairscapeVersion` to all classes which is set to the version of this models package. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/codemeta.json b/codemeta.json new file mode 100644 index 0000000..fb94108 --- /dev/null +++ b/codemeta.json @@ -0,0 +1,46 @@ +{ + "@context": "https://w3id.org/codemeta/3.0", + "@type": "SoftwareSourceCode", + "name": "fairscape-models", + "description": "Pydantic models defining the canonical data shape for FAIRSCAPE RO-Crate releases. The JSON Schemas, TypeScript types, and the EVI OWL/RDFS vocabulary are all generated from these classes, which implement the Fairscape Release RO-Crate Profile.", + "version": "1.2.2", + "license": "https://spdx.org/licenses/Apache-2.0", + "codeRepository": "https://github.com/fairscape/fairscape_models", + "issueTracker": "https://github.com/fairscape/fairscape_models/issues", + "url": "https://fairscape.net", + "downloadUrl": "https://pypi.org/project/fairscape-models/", + "programmingLanguage": "Python", + "runtimePlatform": "Python >= 3.8", + "developmentStatus": "active", + "keywords": [ + "fairscape", + "FAIR", + "RO-Crate", + "EVI", + "provenance", + "pydantic" + ], + "author": [ + { + "@type": "Person", + "@id": "https://orcid.org/0000-0002-1103-3882", + "givenName": "Justin", + "familyName": "Niestroy", + "email": "jniestroy@gmail.com", + "affiliation": { + "@type": "Organization", + "name": "University of Virginia" + } + } + ], + "softwareRequirements": [ + {"@type": "SoftwareApplication", "name": "pydantic", "version": ">=2.5"}, + {"@type": "SoftwareApplication", "name": "pymongo"}, + {"@type": "SoftwareApplication", "name": "pyyaml", "version": ">=6.0.3"} + ], + "funder": { + "@type": "Organization", + "name": "National Institutes of Health" + }, + "funding": "NIH Bridge2AI: OT2OD032742 (Bridge2AI: Cell Maps for AI (CM4AI) Data Generation Project) and OT2OD032701 (Bridge2AI: Patient-Focused Collaborative Hospital Repository Uniting Standards (CHoRUS) for Equitable AI); Frederick Thomas Fund of the University of Virginia" +} diff --git a/fairscape_models/computation.py b/fairscape_models/computation.py index 314f94a..6e3fcf5 100644 --- a/fairscape_models/computation.py +++ b/fairscape_models/computation.py @@ -11,6 +11,7 @@ class Computation(Activity): dateCreated: str additionalDocumentation: Optional[str] = Field(default=None) command: Optional[Union[List[str], str]] = Field(default=None) + parameter: Optional[Union[List[str], str]] = Field(default=None) usedSoftware: Optional[List[IdentifierValue]] = Field(default=[]) usedMLModel: Optional[List[IdentifierValue]] = Field(default=[]) usedDataset: Optional[List[IdentifierValue]] = Field(default=[]) diff --git a/fairscape_models/conversion/mapping/AIReady.py b/fairscape_models/conversion/mapping/AIReady.py index 2c319bb..103e747 100644 --- a/fairscape_models/conversion/mapping/AIReady.py +++ b/fairscape_models/conversion/mapping/AIReady.py @@ -71,6 +71,21 @@ def _get_type(entity: Dict[str, Any]) -> List[str]: return type_val return type_val[-1] +def _parse_content_size_bytes(value: Any) -> float: + """Parse a contentSize value ("1.2 TB", "500 MB", raw bytes) into bytes; 0 if unparseable.""" + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str) or not value.strip(): + return 0 + size = value.strip().upper() + try: + for unit, multiplier in (("TB", 1e12), ("GB", 1e9), ("MB", 1e6), ("KB", 1e3), ("B", 1)): + if size.endswith(unit): + return float(size[:-len(unit)].strip()) * multiplier + return float(size) + except ValueError: + return 0 + def _get_format(entity: Dict[str, Any]) -> str: """Get format from either fileFormat field.""" return entity.get("format", "") @@ -218,29 +233,29 @@ def _score_characterization(characterization: CharacterizationScore, root_data: total_size = total_size_bytes stats_count = stats_count_agg else: - # Fall back to iterating metadata_graph - total_size = 0 + # Fall back to the metadata_graph. The root's own contentSize is + # authoritative when present (it already covers the entities below it); + # only sum the child entities when the root doesn't declare a size. stats_count = 0 + summed_size = 0 + root_id = root_data.get("@id") + seen_ids = set() for entity in metadata_graph: entity_type = _get_type(entity) if "Dataset" in entity_type or "ROCrate" in entity_type: - size = entity.get("contentSize", "") - if size: - try: - if isinstance(size, str): - if "TB" in size: - total_size += float(size.replace("TB", "").strip()) * 1e12 - elif "GB" in size: - total_size += float(size.replace("GB", "").strip()) * 1e9 - elif "MB" in size: - total_size += float(size.replace("MB", "").strip()) * 1e6 - except: - pass - if entity.get("hasSummaryStatistics"): stats_count += 1 + + entity_id = entity.get("@id") + if entity_id == root_id or entity_id in seen_ids: + continue + if entity_id: + seen_ids.add(entity_id) + summed_size += _parse_content_size_bytes(entity.get("contentSize")) + + total_size = _parse_content_size_bytes(root_data.get("contentSize")) or summed_size details = [] if total_size > 0: diff --git a/fairscape_models/conversion/mapping/FairscapeDatasheet.py b/fairscape_models/conversion/mapping/FairscapeDatasheet.py index 8fe6f35..70e12a2 100644 --- a/fairscape_models/conversion/mapping/FairscapeDatasheet.py +++ b/fairscape_models/conversion/mapping/FairscapeDatasheet.py @@ -19,6 +19,20 @@ def _as_list_str(value: Any) -> List[str]: return out return [] +def _related_publications(value: Any) -> List[str]: + items = value if isinstance(value, list) else ([] if value is None else [value]) + out, seen = [], set() + for it in items: + if isinstance(it, dict): + s = it.get("name") or it.get("@id") or it.get("identifier") or "" + else: + s = str(it or "") + s = s.strip() + if s and s not in seen: + seen.add(s) + out.append(s) + return out + def _list_to_str(value: Any) -> str: if isinstance(value, list): return ', '.join(str(v).strip() for v in value if str(v).strip()) @@ -28,6 +42,15 @@ def _list_to_str(value: Any) -> str: return '' return str(value).strip() +def _list_to_str_hyphen(value: Any) -> str: + if isinstance(value, list): + return '- '.join(str(v).strip() for v in value if str(v).strip()) + if isinstance(value, str): + return value.strip() + if value is None: + return '' + return str(value).strip() + def from_additional_property(name: str, default: Optional[str] = None): def _parser(prop_list: Any) -> Optional[str]: if isinstance(prop_list, list): @@ -57,6 +80,46 @@ def _extract_id(value: Any) -> Optional[str]: return value.get("@id") return None +def _resolve_authors(converter_instance, source_entity_model) -> List[str]: + """Resolve the root crate's author field into an ordered list of name strings. + + Authors may be plain name strings or reference stubs ({"@id": "..."}) that + point to Person entities elsewhere in the @graph (e.g. ORCID URIs). Plain + strings are kept as-is; reference stubs are resolved to the referenced + entity's name. References that cannot be resolved from the @graph are + skipped so they are not dropped silently as blanks. + """ + raw = source_entity_model.model_dump(by_alias=True).get("author") + if raw is None: + return [] + + # Build a guid -> name index from the @graph. + name_by_id: Dict[str, str] = {} + for item in converter_instance.source_crate.metadataGraph: + guid = getattr(item, "guid", None) + name = getattr(item, "name", None) + if guid and name: + name_by_id[guid] = name + + entries = raw if isinstance(raw, list) else [raw] + resolved: List[str] = [] + for entry in entries: + if isinstance(entry, str): + s = entry.strip() + if s: + resolved.append(s) + elif isinstance(entry, dict): + # Inline Person object carries its own name. + inline_name = entry.get("name") + if isinstance(inline_name, str) and inline_name.strip(): + resolved.append(inline_name.strip()) + continue + ref_id = entry.get("@id") + if ref_id and ref_id in name_by_id: + resolved.append(name_by_id[ref_id]) + # Unresolved reference: skip per spec (no external lookup). + return resolved + OVERVIEW_MAPPING: Dict[str, Dict[str, Any]] = { # identity @@ -73,7 +136,7 @@ def _extract_id(value: Any) -> Optional[str]: "updated_date": {"source_key": "dateModified"}, # people/orgs - "authors": {"source_key": "author", "parser": _as_list_str}, + "authors": {"builder_func": _resolve_authors}, "publisher": {"source_key": "publisher"}, "principal_investigator":{"source_key": "principalInvestigator"}, "contact_email": {"source_key": "contactEmail"}, @@ -101,7 +164,7 @@ def _extract_id(value: Any) -> Optional[str]: "completeness": {"source_key": "completeness", "fallback_source_key": "additionalProperty", "fallback_parser": from_additional_property("Completeness")}, # related pubs - "related_publications": {"source_key": "associatedPublication", "parser": _as_list_str}, + "related_publications": {"source_key": "associatedPublication", "parser": _related_publications}, } OVERVIEW_MAPPING_CONFIGURATION = { @@ -134,7 +197,7 @@ def _extract_id(value: Any) -> Optional[str]: "data_collection_type": {"source_key": "rai:dataCollectionType", "parser": _list_to_str}, "data_collection_missing_data": {"source_key": "rai:dataCollectionMissingData"}, "data_collection_raw_data": {"source_key": "rai:dataCollectionRawData"}, - "data_collection_timeframe": {"source_key": "rai:dataCollectionTimeframe", "parser": _list_to_str}, + "data_collection_timeframe": {"source_key": "rai:dataCollectionTimeframe", "parser": _list_to_str_hyphen}, "data_imputation_protocol": {"source_key": "rai:dataImputationProtocol"}, "data_manipulation_protocol": {"source_key": "rai:dataManipulationProtocol"}, "data_preprocessing_protocol": {"source_key": "rai:dataPreprocessingProtocol", "parser": _list_to_str}, @@ -203,7 +266,7 @@ def _extract_id(value: Any) -> Optional[str]: "composition_details": {"builder_func": build_composition_details}, # publications - "related_publications": {"source_key": "associatedPublication", "parser": _as_list_str}, + "related_publications": {"source_key": "associatedPublication", "parser": _related_publications}, } @@ -229,20 +292,6 @@ def _keywords_as_list(v: Any) -> List[str]: return [p.strip() for p in v.split(sep) if p.strip()] return [] -def _related_publications(value: Any) -> List[str]: - items = value if isinstance(value, list) else ([] if value is None else [value]) - out, seen = [], set() - for it in items: - if isinstance(it, dict): - s = it.get("name") or it.get("@id") or it.get("identifier") or "" - else: - s = str(it or "") - s = s.strip() - if s and s not in seen: - seen.add(s) - out.append(s) - return out - def _join_authors(value: Any) -> Optional[str]: names: List[str] = [] if isinstance(value, list): diff --git a/fairscape_models/conversion/mapping/subcrate_utils.py b/fairscape_models/conversion/mapping/subcrate_utils.py index c16fbb7..0090fa4 100644 --- a/fairscape_models/conversion/mapping/subcrate_utils.py +++ b/fairscape_models/conversion/mapping/subcrate_utils.py @@ -1,7 +1,56 @@ +import html +import re from typing import Any, Dict, List, Optional, Set from fairscape_models.conversion.models.FairscapeDatasheet import CompositionDetails from collections import Counter +# Computation/experiment patterns render as a small "equation": input RO-Crates combine to +# make an output. These build the HTML fragments (rendered with the Jinja `| safe` filter); +# all dynamic text is html-escaped here. +_ARROW = '→' + + +def _fmt_tokens(fmt: str) -> str: + """Escape a (possibly comma-joined) format string and join its parts with ' + '.""" + return ' + '.join(html.escape(p.strip()) for p in fmt.split(',') if p.strip()) + + +def _input_fragment(rocrate_name: Optional[str], fmt: str) -> str: + """One input term: '**RO-Crate name**: fmt' when cross-crate, else just the format(s).""" + fmt_disp = _fmt_tokens(fmt) + if rocrate_name: + return f"{html.escape(rocrate_name)}: {fmt_disp}" + return fmt_disp + + +def normalize_formats(raw) -> List[str]: + """Canonicalize a raw format value (str or list) into dotless, lowercase tokens. + + Splits combined values on ``/ , ;`` (e.g. ".cx / .tsv" -> ["cx", "tsv"]), strips + surrounding whitespace and a leading dot, lowercases, and drops empty/"unknown" + tokens. This collapses redundant variants like "tsv" / ".tsv" / "TSV" to a single + canonical "tsv" so the datasheet doesn't list the same format multiple times. + """ + out: List[str] = [] + for item in (raw if isinstance(raw, list) else [raw]): + if not isinstance(item, str): + continue + for part in re.split(r'[\/,;]', item): + tok = part.strip().lstrip('.').strip().lower() + if tok and tok != "unknown": + out.append(tok) + return out + + +def _normalize_format_str(raw) -> str: + """Return the normalized format token(s) for a single entity as one display string. + + Joins multiple canonical tokens with ", " and returns "unknown" when nothing usable + remains, so callers that gate on ``!= "unknown"`` keep working. + """ + tokens = normalize_formats(raw) + return ", ".join(tokens) if tokens else "unknown" + def build_composition_details(converter_instance, source_entity_model) -> CompositionDetails: graph = converter_instance.source_crate.metadataGraph global_index = converter_instance.global_index @@ -29,6 +78,8 @@ def build_composition_details(converter_instance, source_entity_model) -> Compos if item_type == "Dataset": details.files_count += 1 + if _has_provenance(item): + details.datasets_with_provenance_count += 1 _process_dataset(item, file_formats, file_access_types) elif item_type == "Software": @@ -61,8 +112,9 @@ def build_composition_details(converter_instance, source_entity_model) -> Compos else: details.other_count += 1 - details.file_formats = dict(Counter(file_formats)) - details.software_formats = dict(Counter(software_formats)) + # Drop blank/unknown formats so the Files card only lists real formats. + details.file_formats = {fmt: count for fmt, count in Counter(file_formats).items() if fmt and fmt != "unknown"} + details.software_formats = {fmt: count for fmt, count in Counter(software_formats).items() if fmt and fmt != "unknown"} details.file_access = dict(Counter(file_access_types)) details.software_access = dict(Counter(software_access_types)) details.computation_patterns = list(set(computation_patterns)) @@ -79,6 +131,24 @@ def build_composition_details(converter_instance, source_entity_model) -> Compos return details +# Provenance link keys, in the forms they appear on parsed graph entities: the EVI +# generatedBy field (https://w3id.org/EVI#generatedBy) parsed or raw, and PROV-O +# prov:wasGeneratedBy (http://www.w3.org/ns/prov#wasGeneratedBy) aliased or raw. +_PROVENANCE_KEYS = ( + 'generatedBy', + 'EVI:generatedBy', + 'evi:generatedBy', + 'https://w3id.org/EVI#generatedBy', + 'wasGeneratedBy', + 'prov:wasGeneratedBy', + 'http://www.w3.org/ns/prov#wasGeneratedBy', +) + + +def _has_provenance(item) -> bool: + return any(getattr(item, key, None) for key in _PROVENANCE_KEYS) + + def _normalize_type(item) -> str: type_field = getattr(item, 'metadataType', None) or getattr(item, '@type', None) @@ -110,7 +180,7 @@ def _normalize_type(item) -> str: def _process_dataset(item, formats: List[str], access_types: List[str]): format_val = getattr(item, 'fileFormat', 'unknown') - formats.append(format_val) + formats.extend(normalize_formats(format_val)) content_url = getattr(item, 'contentUrl', '') if not content_url: @@ -123,7 +193,7 @@ def _process_dataset(item, formats: List[str], access_types: List[str]): def _process_software(item, formats: List[str], access_types: List[str]): format_val = getattr(item, 'fileFormat', 'unknown') - formats.append(format_val) + formats.extend(normalize_formats(format_val)) content_url = getattr(item, 'contentUrl', '') if not content_url: @@ -177,14 +247,14 @@ def _extract_experiment_pattern(item, global_index: Dict[str, Any]) -> Optional[ for dataset_ref in generated: dataset_id = _get_ref_id(dataset_ref) if dataset_id: - format_val = _lookup_format(dataset_id, global_index) + format_val = _normalize_format_str(_lookup_format(dataset_id, global_index)) if format_val and format_val != "unknown": output_formats.append(format_val) - + if output_formats: - output_str = ", ".join(sorted(set(output_formats))) - return f"Sample → {output_str}" - + output_str = " + ".join(sorted(set(_fmt_tokens(f) for f in output_formats))) + return f"Sample {_ARROW} {output_str}" + return None @@ -207,14 +277,14 @@ def _extract_computation_pattern(item, global_index: Dict[str, Any]) -> Optional dataset_id = _get_ref_id(dataset_ref) if dataset_id and dataset_id in global_index: dataset_info = global_index[dataset_id] - format_val = dataset_info.get('fileFormat', 'unknown') + format_val = _normalize_format_str(dataset_info.get('fileFormat', 'unknown')) dataset_rocrate_name = dataset_info.get('rocrateName') if format_val and format_val != "unknown": if dataset_rocrate_name and dataset_rocrate_name != current_rocrate_name: - input_formats.append(f"{dataset_rocrate_name} {format_val}") + input_formats.append(_input_fragment(dataset_rocrate_name, format_val)) else: - input_formats.append(format_val) + input_formats.append(_input_fragment(None, format_val)) generated = getattr(item, 'generated', []) if generated: @@ -224,15 +294,15 @@ def _extract_computation_pattern(item, global_index: Dict[str, Any]) -> Optional for dataset_ref in generated: dataset_id = _get_ref_id(dataset_ref) if dataset_id: - format_val = _lookup_format(dataset_id, global_index) + format_val = _normalize_format_str(_lookup_format(dataset_id, global_index)) if format_val and format_val != "unknown": output_formats.append(format_val) - + if input_formats and output_formats: - input_str = ", ".join(sorted(set(input_formats))) - output_str = ", ".join(sorted(set(output_formats))) - return f"{input_str} → {output_str}" - + input_str = " + ".join(sorted(set(input_formats))) + output_str = " + ".join(sorted(set(_fmt_tokens(f) for f in output_formats))) + return f"{input_str} {_ARROW} {output_str}" + return None @@ -253,6 +323,97 @@ def _lookup_format(dataset_id: str, global_index: Dict[str, Any]) -> str: return 'unknown' +def _resolve_ref_entry(ref, global_index: Dict[str, Any], current_rocrate_name: Optional[str] = None, + include_format: bool = True) -> Optional[Dict[str, Any]]: + """Resolve one usedDataset/generated/usedSoftware ref into {id, name, format[, crate]}. + + Unresolvable refs fall back to the raw @id as the name so no input/output is dropped. + ``crate`` is set only when the referenced entity lives in a different RO-Crate than + the computation (same convention as _extract_computation_pattern). + """ + ref_id = _get_ref_id(ref) + if not ref_id: + return None + info = global_index.get(ref_id) + entry: Dict[str, Any] = { + 'id': ref_id, + 'name': (info.get('name') if info else None) or ref_id, + } + if include_format: + fmt = _normalize_format_str(info.get('fileFormat', 'unknown')) if info else 'unknown' + entry['format'] = fmt if fmt != 'unknown' else None + ref_crate = info.get('rocrateName') if info else None + entry['crate'] = ref_crate if ref_crate and ref_crate != current_rocrate_name else None + return entry + + +def _as_ref_list(refs) -> List[Any]: + if not refs: + return [] + return refs if isinstance(refs, list) else [refs] + + +def extract_computation_details(item, global_index: Dict[str, Any]) -> Dict[str, Any]: + """Structured provenance for one Computation: source crate, inputs, outputs, software, command. + + Unlike _extract_computation_pattern (which reduces to a 'format -> format' string), this + keeps names and @ids so the preview can render an expandable details view. + """ + guid = getattr(item, 'guid', None) + source_crate = None + if guid and guid in global_index: + source_crate = global_index[guid].get('rocrateName') + + command = getattr(item, 'command', None) + if isinstance(command, list): + command = ' '.join(str(part) for part in command) + + details: Dict[str, Any] = { + 'id': guid, + 'source_crate': source_crate, + 'command': command or None, + 'inputs': [], + 'outputs': [], + 'software': [], + } + + for ref in _as_ref_list(getattr(item, 'usedDataset', None)): + entry = _resolve_ref_entry(ref, global_index, source_crate) + if entry: + details['inputs'].append(entry) + + for ref in _as_ref_list(getattr(item, 'generated', None)): + entry = _resolve_ref_entry(ref, global_index, source_crate) + if entry: + details['outputs'].append(entry) + + for ref in _as_ref_list(getattr(item, 'usedSoftware', None)): + entry = _resolve_ref_entry(ref, global_index, source_crate, include_format=False) + if entry: + details['software'].append(entry) + + return details + + +def enrich_preview_computations(preview, source_crate, global_index: Dict[str, Any]) -> None: + """Attach extract_computation_details() output to each computation PreviewItem. + + PreviewItem allows extra fields, so details land on ``item.computation_details`` + without a schema change. Safe to call with a bare per-crate index (no rocrateName + stamps): names/formats still resolve within the crate. + """ + if not preview or not getattr(preview, 'computations', None): + return + + items_by_id = {item.id: item for item in preview.computations if getattr(item, 'id', None)} + for entity in source_crate.metadataGraph: + if _normalize_type(entity) != "Computation": + continue + target = items_by_id.get(getattr(entity, 'guid', None)) + if target is not None: + target.computation_details = extract_computation_details(entity, global_index) + + def _calculate_input_datasets(root_dataset, global_index: Dict[str, Any]) -> Dict[str, int]: input_counts = Counter() @@ -287,11 +448,11 @@ def _calculate_input_datasets(root_dataset, global_index: Dict[str, Any]) -> Dic output_id = _get_ref_id(output_ref) if output_id and output_id in global_index: output_info = global_index[output_id] - format_val = output_info.get('fileFormat', 'unknown') + format_val = _normalize_format_str(output_info.get('fileFormat', 'unknown')) key = f"{rocrate_name} ({format_val})" input_counts[key] += 1 else: - format_val = entity_info.get('fileFormat', 'unknown') + format_val = _normalize_format_str(entity_info.get('fileFormat', 'unknown')) if format_val == 'unknown': format_val = 'Sample' diff --git a/fairscape_models/conversion/models/FairscapeDatasheet.py b/fairscape_models/conversion/models/FairscapeDatasheet.py index d5a72d0..9110fab 100644 --- a/fairscape_models/conversion/models/FairscapeDatasheet.py +++ b/fairscape_models/conversion/models/FairscapeDatasheet.py @@ -24,10 +24,10 @@ class OverviewSection(BaseModel): principal_investigator: Optional[str] = None contact_email: Optional[str] = None - # Legal + # Legal copyright: Optional[str] = None - terms_of_use: Optional[str] = None - citation: Optional[str] = None + terms_of_use: Optional[str] = None + citation: Optional[Union[str, List[str]]] = None # Versioning version: Optional[str] = None @@ -148,6 +148,7 @@ class CompositionDetails(BaseModel): computations_count: int = 0 schemas_count: int = 0 other_count: int = 0 + datasets_with_provenance_count: int = 0 # formats & access summaries file_formats: Dict[str, int] = {} @@ -258,7 +259,7 @@ class Preview(BaseModel): # Misc keywords: List[str] = Field(default_factory=list) - citation: Optional[str] = None + citation: Optional[Union[str, List[str]]] = None related_publications: List[str] = Field(default_factory=list) # Linked QC/summary stats report diff --git a/fairscape_models/fairscape_base.py b/fairscape_models/fairscape_base.py index c6ad651..17c7981 100644 --- a/fairscape_models/fairscape_base.py +++ b/fairscape_models/fairscape_base.py @@ -45,7 +45,7 @@ # TODO fully specify default context "usedSoftware": { - "@id": "https://w3id.org/EVI#usefdSoftware", + "@id": "https://w3id.org/EVI#usedSoftware", "@type": "@id" }, "usedDataset": { @@ -126,7 +126,7 @@ class IdentifierPropertyValue(BaseModel): name: str -def extractGUID(input: str | IdentifierValue | None) -> str|None: +def extractGUID(input: Optional[Union[str, IdentifierValue, dict]]) -> Optional[Union[str, IdentifierValue, dict]]: """ Given an input ARK extract the normalized ARK, if validation fails return the input. """ @@ -150,6 +150,19 @@ def extractGUID(input: str | IdentifierValue | None) -> str|None: return input except AttributeError: return input + elif isinstance(input, dict): + guid = input.get("@id") + if isinstance(guid, str): + try: + input["@id"] = re.search( + pattern="ark:[0-9]{5}/.+$", + string=guid + ).group() + except AttributeError: + # not an ARK; leave as-is + pass + return input + return input class Identifier(BaseModel): diff --git a/fairscape_models/medical_condition.py b/fairscape_models/medical_condition.py index 7bc7418..43219ba 100644 --- a/fairscape_models/medical_condition.py +++ b/fairscape_models/medical_condition.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ConfigDict from typing import Optional, List from fairscape_models.fairscape_base import Identifier, IdentifierValue, IdentifierPropertyValue @@ -9,6 +9,7 @@ class MedicalCondition(Identifier): This class represents any condition of the human body that affects the normal functioning of a person, whether physically or mentally. Includes diseases, injuries, disabilities, disorders, syndromes, etc. """ + model_config = ConfigDict(extra="allow") guid: str = Field(alias="@id") metadataType: Optional[str] = Field(default="MedicalCondition", alias="@type") name: str diff --git a/fairscape_models/patient.py b/fairscape_models/patient.py index ab0f545..ed38aa1 100644 --- a/fairscape_models/patient.py +++ b/fairscape_models/patient.py @@ -23,7 +23,7 @@ class Patient(Identifier): diagnosis: Optional[List[IdentifierValue]] = Field(default=[]) drug: Optional[List[IdentifierValue]] = Field(default=[]) healthCondition: Optional[List[IdentifierValue]] = Field(default=[]) - gender: Optional[str] + gender: Optional[str] = Field(default=None) birthDate: Optional[str] = Field(default=None) deathDate: Optional[str] = Field(default=None) fairscapeVersion: str = __version__ diff --git a/fairscape_models/rocrate.py b/fairscape_models/rocrate.py index b2d3128..57a3190 100644 --- a/fairscape_models/rocrate.py +++ b/fairscape_models/rocrate.py @@ -131,6 +131,13 @@ class ROCrateMetadataElem(BaseModel): keywords: List[str] = Field(description="Keywords or tags describing the dataset, used for discovery and search.") version: str = Field(description="Version string for this release of the dataset (e.g. '1.0', '2.3.1').") datePublished: Optional[str] = Field(default=None, description="Date the dataset was published or made publicly available (ISO 8601).") + dateCreated: Optional[str] = Field(default=None, description="Date the dataset was created (ISO 8601). schema.org/dateCreated; maps from D4D created_on.") + dateModified: Optional[str] = Field(default=None, description="Date the dataset was most recently modified or updated (ISO 8601). schema.org/dateModified; maps from D4D last_updated_on.") + url: Optional[str] = Field(default=None, description="Landing page or canonical URL for the dataset. schema.org/url; maps from D4D page.") + language: Optional[Union[str, List[str]]] = Field(default=None, description="Language(s) of the dataset content, ideally as BCP-47 codes (e.g. 'en'). schema.org/inLanguage; maps from D4D language.") + creativeWorkStatus: Optional[str] = Field(default=None, description="Publication/lifecycle status of the dataset (e.g. 'Draft', 'Published', 'Deprecated'). schema.org/creativeWorkStatus; maps from D4D status.") + about: Optional[str] = Field(default=None, description="The subject matter of an object.") + correction: Optional[Union[str, List[str]]] = Field(default=None, description="Corrections, errata, or amendments to the dataset. schema.org/correction; maps from D4D errata.") # Relationships isPartOf: Optional[List[IdentifierValue]] = Field(default=[], description="Parent organization(s) or project(s) this crate belongs to, referenced by identifier.") @@ -140,9 +147,9 @@ class ROCrateMetadataElem(BaseModel): author: Union[str, IdentifierValue, Person, List[Union[str, IdentifierValue, Person]]] = Field(description="Who created the dataset. Accepts a plain name string, a Person object (with optional ORCID identifier), a {\"@id\": \"...\"} reference stub to a Person in @graph, or a list of any of those. Plain strings remain valid for backwards compatibility.") publisher: Optional[str] = Field(default=None, description="Organization or person responsible for publishing or distributing the dataset.") principalInvestigator: Optional[Union[str, IdentifierValue, Person]] = Field(default=None, description="A key individual (Principal Investigator) responsible for or overseeing dataset creation. Accepts a plain name string, a reference stub ({\"@id\": \"...\"}) to a Person in @graph, or an inline Person object.") - funder: Optional[Union[str, IdentifierValue, Person]] = Field(default=None, description="Who funded the creation of the dataset? Include grant names and numbers where applicable. Accepts a plain name string, a reference stub, or an inline Person/Organization object.") + funder: Optional[Union[str, IdentifierValue, Person, List[Union[str, IdentifierValue, Person]]]] = Field(default=None, description="Who funded the creation of the dataset? Include grant names and numbers where applicable. Accepts a plain name string, a reference stub, an inline Person/Organization object, or a list of any of those.") contactEmail: Optional[str] = Field(default=None, description="Email address for questions or correspondence about the dataset.") - citation: Optional[str] = Field(default=None, description="Preferred citation string for this dataset.") + citation: Optional[Union[str, List[str]]] = Field(default=None, description="Preferred citation(s) for this dataset. Accepts a single string or a list of citation strings.") associatedPublication: Optional[Union[str, List[str]]] = Field(default=None, description="Publication(s) associated with or describing this dataset.") identifier: Optional[str] = Field(default=None, description="DOI or other external persistent identifier for the dataset (used for Findability and Sustainability scoring).") @@ -158,7 +165,7 @@ class ROCrateMetadataElem(BaseModel): additionalProperty: Optional[List[Dict[str, Any]]] = Field(default=None, description="Additional schema.org PropertyValue entries for metadata not covered by other fields (e.g. [{\"name\": \"Human Subject\", \"value\": \"Yes\"}]).") # Compliance / ethics — D4D_Ethics, D4D_Human, D4D_Data_Governance - ethicalReview: Optional[str] = Field(default=None, description="Were any ethical or compliance review processes conducted (e.g. by an Institutional Review Board)? If so, describe the process, frequency of review, and outcomes. Or provide a contact for ethical review information.") + ethicalReview: Optional[Union[str, List[str]]] = Field(default=None, description="Were any ethical or compliance review processes conducted (e.g. by an Institutional Review Board)? If so, describe the process, frequency of review, and outcomes. Or provide a contact for ethical review information.") confidentialityLevel: Optional[str] = Field(default=None, description="HL7 Confidentiality code indicating the level of confidentiality or sensitivity of the dataset (e.g. 'normal', 'restricted', 'very restricted').") irb: Optional[Union[str, IRB]] = Field(default=None, description="Institutional Review Board (IRB) information — approval status, approving institution, and contact details.") irbProtocolId: Optional[str] = Field(default=None, description="IRB protocol identifier number assigned by the reviewing institution.") @@ -167,7 +174,7 @@ class ROCrateMetadataElem(BaseModel): deidentified: Optional[bool] = Field(default=None, description="Whether the dataset has been de-identified to remove or obscure personally identifiable information.") humanSubjectResearch: Optional[str] = Field(default=None, description="Does this dataset involve human subjects? Indicate Yes/No and describe the nature of human subjects involvement.") dataGovernanceCommittee: Optional[Union[str, IdentifierValue, Person]] = Field(default=None, description="Name or contact for the data governance committee responsible for oversight, access control, and policy enforcement for this dataset. Accepts a plain name string, a reference stub, or an inline Person.") - about: Optional[List[Union[IdentifierValue, DefinedTerm, str]]] = Field(default=None, description="Subjects this dataset is about, ideally as ontology-grounded DefinedTerm entries (MeSH, EDAM, Cellosaurus, etc.) referenced from @graph. Supports AI-Ready Rubric 2.a (Semantics).") + about: Optional[Union[str, List[Union[IdentifierValue, DefinedTerm, str]]]] = Field(default=None, description="Subjects this dataset is about — a summary string, or ontology-grounded DefinedTerm entries (MeSH, EDAM, Cellosaurus, etc.) referenced from @graph. Supports AI-Ready Rubric 2.a (Semantics).") # Checksums md5: Optional[str] = Field(default=None, description="MD5 checksum of the digital object content") @@ -193,7 +200,7 @@ class ROCrateMetadataElem(BaseModel): alias="rai:dataReleaseMaintenancePlan", default=None, description="Will the dataset be updated (e.g. to correct labeling errors, add new instances, delete instances)? If so, how often, by whom, and how will updates be communicated? Covers versioning timeframe, maintainers, and deprecation policies. (rai:dataReleaseMaintenancePlan)" ) - rai_data_collection: Optional[str] = Field( + rai_data_collection: Optional[Union[str, List[str]]] = Field( alias="rai:dataCollection", default=None, description="What mechanisms or procedures were used to collect the data (e.g. hardware sensors, manual curation, software APIs)? Also covers how these mechanisms were validated. (rai:dataCollection)" ) @@ -201,11 +208,11 @@ class ROCrateMetadataElem(BaseModel): alias="rai:dataCollectionType", default=None, description="Data collection type(s). Recommended values: Surveys, Secondary Data Analysis, Physical Data Collection, Direct Measurement, Document Analysis, Manual Human Curator, Software Collection, Experiments, Web Scraping, Web API, Focus Groups, Self-Reporting, Customer Feedback Data, User-Generated Content Data, Passive Data Collection, Others. (rai:dataCollectionType)" ) - rai_data_collection_missing_data: Optional[str] = Field( + rai_data_collection_missing_data: Optional[Union[str, List[str]]] = Field( alias="rai:dataCollectionMissingData", default=None, description="Documentation of missing data in the dataset, including patterns (e.g. MCAR, MAR, MNAR), known or suspected causes (e.g. sensor failures, participant dropout, privacy constraints), and strategies used to handle missing values. (rai:dataCollectionMissingData)" ) - rai_data_collection_raw_data: Optional[str] = Field( + rai_data_collection_raw_data: Optional[Union[str, List[str]]] = Field( alias="rai:dataCollectionRawData", default=None, description="Description of raw data sources before preprocessing, cleaning, or labeling. Documents where the original data comes from and how it can be accessed. (rai:dataCollectionRawData)" ) @@ -213,7 +220,7 @@ class ROCrateMetadataElem(BaseModel): alias="rai:dataCollectionTimeframe", default=None, description="Over what timeframe was the data collected, and does this timeframe match the creation timeframe of the underlying data? Provide start and end dates where possible. (rai:dataCollectionTimeframe)" ) - rai_data_imputation_protocol: Optional[str] = Field( + rai_data_imputation_protocol: Optional[Union[str, List[str]]] = Field( alias="rai:dataImputationProtocol", default=None, description="Description of data imputation methodology, including techniques used to handle missing values (e.g. mean/median imputation, forward fill, model-based imputation) and rationale for chosen approaches. (rai:dataImputationProtocol)" ) @@ -225,7 +232,7 @@ class ROCrateMetadataElem(BaseModel): alias="rai:dataPreprocessingProtocol", default=None, description="Was any preprocessing of the data done (e.g. discretization or bucketing, tokenization, feature extraction, normalization)? Describe the steps required to bring collected data to a state that can be processed by an ML model or algorithm. (rai:dataPreprocessingProtocol)" ) - rai_data_annotation_protocol: Optional[str] = Field( + rai_data_annotation_protocol: Optional[Union[str, List[str]]] = Field( alias="rai:dataAnnotationProtocol", default=None, description="Annotation methodology, tasks, and protocols followed during labeling. Includes annotation guidelines, quality control procedures, task definitions, workforce type, annotation characteristics, and label distributions. (rai:dataAnnotationProtocol)" ) @@ -241,11 +248,11 @@ class ROCrateMetadataElem(BaseModel): alias="rai:personalSensitiveInformation", default=None, description="Does the dataset contain data that might be considered sensitive (e.g. race, sexual orientation, religion, biometrics)? List sensitive attribute types present: Gender, Socio-economic status, Geography, Language, Age, Culture, Experience or Seniority, others. (rai:personalSensitiveInformation)" ) - rai_data_social_impact: Optional[str] = Field( + rai_data_social_impact: Optional[Union[str, List[str]]] = Field( alias="rai:dataSocialImpact", default=None, description="Is there anything about the dataset's composition or collection that might impact future uses or create risks/harm (e.g. unfair treatment, legal or financial risks)? Describe potential impacts and any mitigation strategies. (rai:dataSocialImpact)" ) - rai_annotations_per_item: Optional[str] = Field( + rai_annotations_per_item: Optional[Union[str, List[str]]] = Field( alias="rai:annotationsPerItem", default=None, description="Number of annotations collected per data item. Multiple annotations per item enable calculation of inter-annotator agreement. (rai:annotationsPerItem)" ) @@ -253,8 +260,8 @@ class ROCrateMetadataElem(BaseModel): alias="rai:machineAnnotationTools", default=None, description="Automated or machine-learning-based annotation tools used in dataset creation, including NLP pipelines, computer vision models, or other automated labeling systems. Format each entry as 'ToolName version' (e.g. 'spaCy 3.5.0'). (rai:machineAnnotationTools)" ) - completeness: Optional[str] = Field(alias="completeness", default=None, description="Assessment of how complete the dataset is relative to its intended scope (e.g. percentage of expected records present, known gaps).") - prohibitedUses: Optional[str] = Field(alias="prohibitedUses", default=None, description="Explicit statement of prohibited or forbidden uses for this dataset — uses that are not permitted by license, ethics, or policy. Stronger than discouraged uses.") + completeness: Optional[Union[str, List[str]]] = Field(alias="completeness", default=None, description="Assessment of how complete the dataset is relative to its intended scope (e.g. percentage of expected records present, known gaps).") + prohibitedUses: Optional[Union[str, List[str]]] = Field(alias="prohibitedUses", default=None, description="Explicit statement of prohibited or forbidden uses for this dataset — uses that are not permitted by license, ethics, or policy. Stronger than discouraged uses.") # Aggregated metrics for AI-Ready scoring (roll-up properties from release-level sub-crates) evi_dataset_count: Optional[int] = Field(alias="evi:datasetCount", default=None, description="Pre-aggregated count of Dataset entities across all sub-crates. Used in AI-Ready Provenance scoring in place of counting entities at query time.") @@ -269,11 +276,16 @@ class ROCrateMetadataElem(BaseModel): evi_proccesed: Optional[bool] = Field(alias="evi:processed", default=None, description="Flag indicating whether this release-level RO-Crate has been processed and aggregated metrics computed.") # D4D Placeholders — flat string versions of D4D_Motivation / D4D_Composition / D4D_Human classes - addressingGaps: Optional[str] = Field(alias="d4d:addressingGaps", default=None, description="Was there a specific knowledge or resource gap that needed to be filled by creation of this dataset? (D4D_Motivation: AddressingGap)") - dataAnomalies: Optional[str] = Field(alias="d4d:dataAnomalies", default=None, description="Are there any errors, sources of noise, or redundancies in the dataset? (D4D_Composition: DataAnomaly)") - contentWarning: Optional[str] = Field(alias="d4d:contentWarning", default=None, description="Does the dataset contain any data that might be offensive, insulting, threatening, or otherwise anxiety-provoking if viewed directly? (D4D_Composition: ContentWarning)") - informedConsent: Optional[str] = Field(alias="d4d:informedConsent", default=None, description="Details about informed consent procedures used in human subjects research — consent type, documentation, withdrawal mechanisms, and scope. (D4D_Human: InformedConsent)") - atRiskPopulations: Optional[str] = Field(alias="d4d:atRiskPopulations", default=None, description="Information about protections for at-risk populations (e.g. children, pregnant women, prisoners, cognitively impaired individuals) included in human subjects research. (D4D_Human: AtRiskPopulations)") + addressingGaps: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:addressingGaps", default=None, description="Was there a specific knowledge or resource gap that needed to be filled by creation of this dataset? Accepts a summary string or a list of {\"@id\": ...} references to AddressingGap DatasetProperty nodes. (D4D_Motivation: AddressingGap)") + dataAnomalies: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:dataAnomalies", default=None, description="Are there any errors, sources of noise, or redundancies in the dataset? Accepts a summary string or a list of {\"@id\": ...} references to DataAnomaly DatasetProperty nodes. (D4D_Composition: DataAnomaly)") + contentWarning: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:contentWarning", default=None, description="Does the dataset contain any data that might be offensive, insulting, threatening, or otherwise anxiety-provoking if viewed directly? (D4D_Composition: ContentWarning)") + informedConsent: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:informedConsent", default=None, description="Details about informed consent procedures used in human subjects research — consent type, documentation, withdrawal mechanisms, and scope. (D4D_Human: InformedConsent)") + atRiskPopulations: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:atRiskPopulations", default=None, description="Information about protections for at-risk populations (e.g. children, pregnant women, prisoners, cognitively impaired individuals) included in human subjects research. Accepts a summary string or {\"@id\": ...} reference(s) to an AtRiskPopulations DatasetProperty node. (D4D_Human: AtRiskPopulations)") + participantPrivacy: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:participantPrivacy", default=None, description="How is the privacy of human subjects protected — anonymization, access controls, third-party sharing limits, and compensation arrangements? Accepts a summary string or {\"@id\": ...} reference(s) to a ParticipantPrivacy DatasetProperty node. (D4D_Human: ParticipantPrivacy)") + subpopulations: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:subpopulations", default=None, description="Does the dataset identify any subpopulations (e.g. by age, gender)? Accepts a summary string or a list of {\"@id\": ...} references to Subpopulation DatasetProperty nodes. (D4D_Composition: Subpopulation)") + subsets: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:subsets", default=None, description="Is the dataset partitioned into named subsets (e.g. train/validation/test splits)? Accepts a summary string or a list of {\"@id\": ...} references to Subset DatasetProperty nodes. (D4D_Composition: Subsets)") + samplingStrategies: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:samplingStrategies", default=None, description="If the dataset is a sample from a larger set, what was the sampling strategy (e.g. deterministic, probabilistic, representativeness)? Accepts a summary string or a list of {\"@id\": ...} references to SamplingStrategy DatasetProperty nodes. (D4D_Composition: SamplingStrategy)") + distributionDates: Optional[Union[str, List[IdentifierValue], List[str]]] = Field(alias="d4d:distributionDates", default=None, description="When was/will the dataset be distributed, and over what dates were its distributions or versions released? Accepts a summary string or a list of {\"@id\": ...} references to DistributionDate DatasetProperty nodes. (D4D_Distribution: DistributionDate)") def generateFileElem(self) -> ROCrateMetadataFileElem: """ Given an ROCrate Element create an appropriate ROCrateMetadataFileElem @@ -377,6 +389,9 @@ def validate_metadata_graph(cls, values: Dict[str, Any]) -> Dict[str, Any]: "Computation": Computation, "Annotation": Annotation, "Experiment": Experiment, + "Sample": Sample, + "Instrument": Instrument, + "Patient": Patient, "Activity": Activity, "CreativeWork": ROCrateMetadataFileElem, "Schema": Schema, diff --git a/fairscape_models/sample.py b/fairscape_models/sample.py index c8586e3..9583890 100644 --- a/fairscape_models/sample.py +++ b/fairscape_models/sample.py @@ -11,7 +11,16 @@ class Sample(Identifier): description: str = Field(min_length=1) keywords: List[str] = Field(...) contentUrl: Optional[Union[str, List[str]]] = Field(default=None) - cellLineReference: Optional[IdentifierValue] = Field(default=None) + + #Made up properties to support our projects + cellLineReference: Optional[Union[IdentifierValue, List[IdentifierValue]]] = Field(default=None) + samplePrepMethod: Optional[Union[IdentifierValue, List[IdentifierValue]]] = Field(default=None) + + #BioSchemas Terms for C2M2 + anatomicalStructure: Optional[Union[IdentifierValue, List[IdentifierValue]]] = Field(default=None) + associatedDisease: Optional[Union[IdentifierValue, List[IdentifierValue]]] = Field(default=None) + hasBioChemEntityPart: Optional[Union[IdentifierValue, List[IdentifierValue]]] = Field(default=None) + isPartOf: Optional[List[IdentifierValue]] = Field(default=[]) fairscapeVersion: str = __version__ diff --git a/fairscape_models/schema.py b/fairscape_models/schema.py deleted file mode 100644 index 7b5b821..0000000 --- a/fairscape_models/schema.py +++ /dev/null @@ -1,84 +0,0 @@ -from typing import ( - Optional, - Dict, - List, - Union -) -from pydantic import ( - BaseModel, - Field, - field_validator, - ConfigDict - ) -import re -from enum import Enum -from fairscape_models.fairscape_base import Identifier, IdentifierValue -from fairscape_models._version import __version__ - -# TODO switch to ENUM for better clarification -class ItemTypeEnum(Enum): - integer='integer' - number='number' - string='string' - array='array' - boolean='boolean' - object='object' - -class Property(BaseModel): - description: str = Field(...) - index: Union[str, int] = Field(...) - type: str = Field(...) - value_url: Optional[str] = Field(default = None, alias = 'value-url') - pattern: Optional[str] = Field(default = None) - min_items: Optional[int] = Field(default = None, alias = 'min-items') - max_items: Optional[int] = Field(default = None, alias = 'max-items') - unique_items: Optional[bool] = Field(default = None, alias = 'unique-items') - properties: Optional[Dict[str, 'Property']] = Field(default=None) - - model_config = ConfigDict(extra='allow') - - @field_validator('index', mode='before') - def validate_index(cls, value): - if isinstance(value, str): - # Allow something like int::int for index. Raise error if else - pattern = r'^\d+$|^-?\d+::|^-?\d+::-?\d+$|^::-?\d+' - if not re.match(pattern, value): - raise ValueError("Index must match the pattern 'int::int'") - return value - - @field_validator('pattern', mode='before') - def validate_pattern(cls, value): - if value is not None: - try: - re.compile(value) - except re.error: - raise ValueError("Pattern must be a valid regular expression") - return value - - @field_validator('type', mode='before') - def validate_property_type(cls, value): - valid_types = {'integer', 'number', 'string', 'array','boolean', 'object'} - if value is not None: - if value not in valid_types: - raise ValueError(f"Type must be one of {valid_types}") - return value - -class Schema(Identifier): - context: Dict[str, str] = Field( - default= {"@vocab": "https://schema.org/", "evi": "https://w3id.org/EVI#"}, - alias="@context" - ) - metadataType: str = Field(alias="@type", default= "evi:Schema") - conformsTo: Optional[Union[List[IdentifierValue],IdentifierValue]] = Field(default={"@id": "https://json-schema.org/draft/2020-12/schema"}) - properties: Dict[str, Property] - schemaType: Optional[str] = Field(default="object", alias="type") - additionalProperties: Optional[bool] = Field(default=True) - required: Optional[List[str]] = [] - separator: Optional[str] = Field(default=",") - header: Optional[bool] = Field(default=True) - examples: Optional[List[Dict]] = [] - isPartOf: Optional[List[IdentifierValue]] = Field(default=[]) - fairscapeVersion: str = __version__ - - - model_config = ConfigDict(extra='allow') \ No newline at end of file diff --git a/fairscape_models/schema/__init__.py b/fairscape_models/schema/__init__.py new file mode 100644 index 0000000..debf8f3 --- /dev/null +++ b/fairscape_models/schema/__init__.py @@ -0,0 +1,78 @@ +""" +fairscape_models.schema — the EVI Schema family. + +The import path `fairscape_models.schema` is preserved from the days this was a +single module: `Schema`, `Property`, and `ItemTypeEnum` re-export from here, +alongside the typed schema variants, the file-dispatch entry points, and the +legacy-document loader. + +No heavy dependency (frictionless / h5py / wfdb / pydicom) is imported at module +load time — each is imported lazily inside the infer/validate method that needs +it, so `import fairscape_models.schema` stays light for the API server. +""" + +from fairscape_models.schema.base import ( + AxisProperty, + ItemTypeEnum, + NonTabularSchema, + Property, + Schema, + ValidationErrorRecord, + compare_scalar, + frictionless_error_to_record, + generate_schema_guid, +) +from fairscape_models.schema.tabular import ( + SOURCE_TYPE_KEY, + TabularSchema, + build_frictionless_schema, + frictionless_type_to_json_schema, +) +from fairscape_models.schema.hdf5 import DatasetProperty, HDF5Schema +from fairscape_models.schema.signal import ChannelProperty, SignalSchema +from fairscape_models.schema.image import ImageSchema +from fairscape_models.schema.ndarray import NDArraySchema +from fairscape_models.schema.registry import ( + EXTENSION_MAP, + SCHEMA_TYPE_MODELS, + infer_schema, + load_schema, + normalize_schema_document, + parse_schema, + schema_class_for_file, + validate_schema, +) + +__all__ = [ + # canonical models + 'Schema', + 'Property', + 'ItemTypeEnum', + 'NonTabularSchema', + 'AxisProperty', + # typed variants + 'TabularSchema', + 'HDF5Schema', + 'DatasetProperty', + 'SignalSchema', + 'ChannelProperty', + 'ImageSchema', + 'NDArraySchema', + # validation + helpers + 'ValidationErrorRecord', + 'frictionless_error_to_record', + 'compare_scalar', + 'generate_schema_guid', + 'SOURCE_TYPE_KEY', + 'frictionless_type_to_json_schema', + 'build_frictionless_schema', + # dispatch + loading + 'SCHEMA_TYPE_MODELS', + 'EXTENSION_MAP', + 'schema_class_for_file', + 'parse_schema', + 'load_schema', + 'normalize_schema_document', + 'infer_schema', + 'validate_schema', +] diff --git a/fairscape_models/schema/base.py b/fairscape_models/schema/base.py new file mode 100644 index 0000000..1be8a58 --- /dev/null +++ b/fairscape_models/schema/base.py @@ -0,0 +1,253 @@ +""" +base.py — the canonical Schema/Property models and the shared pieces of the +schema-type family. + +The class hierarchy: + + Property ── AxisProperty (labeled axis: image / ndarray / hdf5) + Schema ──── TabularSchema (csv / tsv / parquet, tabular.py) + └─── NonTabularSchema ────── SignalSchema (signal.py) + ├──── ImageSchema (image.py) + ├──── NDArraySchema (ndarray.py) + └──── HDF5Schema (hdf5.py) + +Every concrete schema type overrides `infer` (build a schema from a data file) +and `validate` (check a data file against this schema). Heavy per-format +dependencies (frictionless, h5py, wfdb, pydicom) are imported lazily inside +those methods so importing fairscape_models stays light. +""" + +import math +import re +import uuid +from enum import Enum +from typing import Dict, List, Literal, Optional, Union + +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + field_validator, +) + +from fairscape_models.fairscape_base import ( + DEFAULT_ARK_NAAN, + Identifier, + IdentifierValue, +) +from fairscape_models._version import __version__ + + +class ItemTypeEnum(Enum): + integer = 'integer' + number = 'number' + string = 'string' + array = 'array' + boolean = 'boolean' + object = 'object' + + +class Property(BaseModel): + description: str = Field(...) + index: Union[str, int] = Field(...) + type: str = Field(...) + value_url: Optional[str] = Field( + default=None, + validation_alias=AliasChoices('valueURL', 'value-url', 'value_url'), + serialization_alias='valueURL', + ) + pattern: Optional[str] = Field(default=None) + min_items: Optional[int] = Field(default=None, alias='min-items') + max_items: Optional[int] = Field(default=None, alias='max-items') + unique_items: Optional[bool] = Field(default=None, alias='unique-items') + properties: Optional[Dict[str, 'Property']] = Field(default=None) + + model_config = ConfigDict(extra='allow', populate_by_name=True) + + @field_validator('index', mode='before') + def validate_index(cls, value): + if isinstance(value, str): + # Allow something like int::int for index. Raise error if else + pattern = r'^\d+$|^-?\d+::|^-?\d+::-?\d+$|^::-?\d+' + if not re.match(pattern, value): + raise ValueError("Index must match the pattern 'int::int'") + return value + + @field_validator('pattern', mode='before') + def validate_pattern(cls, value): + if value is not None: + try: + re.compile(value) + except re.error: + raise ValueError("Pattern must be a valid regular expression") + return value + + @field_validator('type', mode='before') + def validate_property_type(cls, value): + valid_types = {'integer', 'number', 'string', 'array', 'boolean', 'object'} + if value is not None: + if value not in valid_types: + raise ValueError(f"Type must be one of {valid_types}") + return value + + +class AxisProperty(Property): + """ + One axis of an `image`, `ndarray`, or `hdf5` schema — the OME-NGFF / CF + "labeled axis" primitive: a named, typed axis with a size, a unit, and a + physical scale. + + An axis index is discrete so `type` is "integer". `axisType` follows OME-NGFF + (space | time | channel); `standardName` follows CF (a Standard Name Table + term) for the generic-N-D case. + """ + axisType: Optional[Literal["space", "time", "channel"]] = Field( + default=None, description="OME-NGFF axis type" + ) + size: Optional[int] = Field(default=None, description="Number of elements along this axis") + spacing: Optional[float] = Field( + default=None, description="Physical spacing per element (voxel/pixel size, or time step)" + ) + unit: Optional[str] = Field( + default=None, + description="Axis unit. UCUM for DICOM/NIfTI; UDUNITS-2 for OME-NGFF/CF (e.g. 'micrometer', 'degrees_north')", + ) + unitURL: Optional[str] = Field(default=None, description="Resolvable unit URI (QUDT), where a verified term exists") + standardName: Optional[str] = Field( + default=None, description="CF Standard Name for this coordinate (ndarray/netCDF), e.g. 'latitude'" + ) + + +class ValidationErrorRecord(BaseModel): + message: str + failed_keyword: str = "error" + error_type: str = "ValidationError" + field: Optional[str] = None + row: Optional[int] = None + path: Optional[str] = None + + @property + def location(self) -> str: + parts = [ + part for part in ( + self.path, + f"row {self.row}" if self.row is not None else None, + self.field, + ) if part + ] + return " / ".join(parts) or "-" + + +def frictionless_error_to_record(error, path: Optional[str] = None) -> ValidationErrorRecord: + """Convert a frictionless error object to a ValidationErrorRecord. + + Frictionless 5.x exposes row/field attributes under both snake_case and + camelCase depending on error class and minor version, so check both. + """ + row = getattr(error, 'row_number', None) + if row is None: + row = getattr(error, 'rowNumber', None) + field = getattr(error, 'field_name', None) + if field is None: + field = getattr(error, 'fieldName', None) + return ValidationErrorRecord( + message=error.message, + row=row, + field=field, + failed_keyword=getattr(error, 'type', 'error') or 'error', + path=path, + ) + + +def generate_schema_guid(name: str, naan: str = DEFAULT_ARK_NAAN) -> str: + """Generate a unique identifier for a schema.""" + slug = re.sub(r'[^a-zA-Z0-9_-]+', '-', name.lower()).strip('-') + return f"ark:{naan}/schema-{slug}-{uuid.uuid4().hex[:10]}" + + +_REL_TOL = 1e-6 + + +def compare_scalar(errors: List[ValidationErrorRecord], declared, observed, + path: Optional[str], field: str) -> None: + """ + Append a `const`/`type` mismatch when a declared (non-None) value disagrees + with what was observed in the file. A None `declared` means "unconstrained". + Shared by the non-tabular validate() implementations. + """ + if declared is None: + return + if isinstance(declared, float) and isinstance(observed, (int, float)) and not isinstance(observed, bool): + if not math.isclose(declared, observed, rel_tol=_REL_TOL): + errors.append(ValidationErrorRecord( + message=f"{field}: expected {declared}, file has {observed}", + failed_keyword="const", field=field, path=path, + )) + elif declared != observed: + errors.append(ValidationErrorRecord( + message=f"{field}: expected {declared!r}, file has {observed!r}", + failed_keyword="const", field=field, path=path, + )) + + +class Schema(Identifier): + context: Dict[str, str] = Field( + default={"@vocab": "https://schema.org/", "evi": "https://w3id.org/EVI#"}, + alias="@context" + ) + metadataType: str = Field(alias="@type", default="evi:Schema") + conformsTo: Optional[Union[List[IdentifierValue], IdentifierValue]] = Field(default={"@id": "https://json-schema.org/draft/2020-12/schema"}) + properties: Dict[str, Property] + schemaType: Optional[str] = Field(default="object", alias="type") + additionalProperties: Optional[bool] = Field(default=True) + required: Optional[List[str]] = [] + separator: Optional[str] = Field(default=",") + header: Optional[bool] = Field(default=True) + examples: Optional[List[Dict]] = [] + isPartOf: Optional[List[IdentifierValue]] = Field(default=[]) + fairscapeVersion: str = __version__ + + model_config = ConfigDict(extra='allow', populate_by_name=True) + + @classmethod + def infer(cls, filepath: str, name: str, description: str, + guid: Optional[str] = None) -> "Schema": + """Build a schema of this type by inspecting a data file.""" + raise NotImplementedError(f"{cls.__name__} does not support inference") + + def validate(self, filepath: str) -> List[ValidationErrorRecord]: + """Validate a data file against this schema.""" + raise NotImplementedError(f"{type(self).__name__} does not support file validation") + + +class NonTabularSchema(Schema): + """ + Shared base for the non-tabular schema types. Subclasses `Schema` so it + keeps `conformsTo`, `required`, `examples`, `isPartOf`, and + `fairscapeVersion` for free, and keeps `@type == "EVI:Schema"`. + + Differences from the tabular default: + * `@context` and `@type` default to the upper-case `EVI` casing (both + casings resolve to the same IRI). + * The tabular-only fields `separator`, `header`, `type` (schemaType), and + `additionalProperties` default to None so they drop out under + `exclude_none=True` — they are meaningless for signal/image/ndarray/hdf5. + * Adds the `EVI:schemaType` discriminator (concrete Literal per subclass). + """ + context: Dict[str, str] = Field( + default={"@vocab": "https://schema.org/", "EVI": "https://w3id.org/EVI#"}, + alias="@context", + ) + metadataType: str = Field(default="EVI:Schema", alias="@type") + evi_schema_type: str = Field(alias="EVI:schemaType") + + # Neutralize the tabular-only defaults inherited from Schema. + schemaType: Optional[str] = Field(default=None, alias="type") + additionalProperties: Optional[bool] = Field(default=None) + separator: Optional[str] = Field(default=None) + header: Optional[bool] = Field(default=None) + + unitSystem: Optional[str] = Field( + default=None, description="Unit system IRI for UCUM codes, e.g. 'http://unitsofmeasure.org'" + ) diff --git a/fairscape_models/schema/hdf5.py b/fairscape_models/schema/hdf5.py new file mode 100644 index 0000000..3c02441 --- /dev/null +++ b/fairscape_models/schema/hdf5.py @@ -0,0 +1,171 @@ +""" +hdf5.py — the `hdf5` schema type: a whole HDF5 file as a tree of datasets. + +`properties` map each dataset's HDF5 path to a `DatasetProperty` carrying its +array structure (shape / dtype / chunks / compression) — the same NDArray-style +description used by `NDArraySchema`, one per dataset. This replaces the old +"flatten every dataset into a pandas DataFrame and validate column_0..n as a +table" approach: inference reads structure only (attrs / shape / dtype), never +the array data, and validation compares that structure back against the file. + +Reader lib: h5py. +""" + +from typing import Dict, List, Literal, Optional + +from pydantic import Field + +from fairscape_models.schema.base import ( + NonTabularSchema, + Property, + ValidationErrorRecord, + generate_schema_guid, +) + +# numpy dtype kind -> canonical JSON-Schema type, for compound-dtype fields. +_NUMPY_KIND_TO_JSON = { + 'i': 'integer', 'u': 'integer', + 'f': 'number', 'c': 'number', + 'b': 'boolean', + 'S': 'string', 'U': 'string', 'O': 'string', + 'V': 'object', +} + + +def _json_type_for_kind(kind: str) -> str: + return _NUMPY_KIND_TO_JSON.get(kind, 'string') + + +class DatasetProperty(Property): + """ + One HDF5 dataset described structurally. `type` is "array" for a plain + dataset and "object" for a compound (structured) dtype, whose fields become + nested `properties`. + """ + hdf5Path: Optional[str] = Field(default=None, alias="hdf5-path") + shape: Optional[List[int]] = Field(default=None) + dtype: Optional[str] = Field(default=None, description="numpy dtype string, e.g. 'float64', or the compound descr") + chunks: Optional[List[int]] = Field(default=None) + compression: Optional[str] = Field(default=None) + + +class HDF5Schema(NonTabularSchema): + """`hdf5` — a whole HDF5 file. `properties` map dataset path -> structure.""" + + evi_schema_type: Literal["hdf5"] = Field(default="hdf5", alias="EVI:schemaType") + properties: Dict[str, DatasetProperty] = Field(default_factory=dict) + + @classmethod + def infer(cls, filepath: str, name: str, description: str, + guid: Optional[str] = None) -> "HDF5Schema": + import h5py + + props: Dict[str, DatasetProperty] = {} + + def visit(path, obj): + if not isinstance(obj, h5py.Dataset): + return # groups are structure, not leaves; skip + dt = obj.dtype + nested = None + if dt.names: # compound / structured dtype -> field columns + nested = { + fname: Property( + description=f"Field '{fname}' of dataset {path}", + index=i, + type=_json_type_for_kind(dt.fields[fname][0].kind), + ) + for i, fname in enumerate(dt.names) + } + props[path] = DatasetProperty( + description=f"Dataset at {path}", + index=len(props), + type="object" if dt.names else "array", + **{"hdf5-path": path}, + shape=list(obj.shape), + dtype=str(dt), + chunks=list(obj.chunks) if obj.chunks else None, + compression=obj.compression, + properties=nested, + ) + + with h5py.File(filepath, "r") as fh: + fh.visititems(visit) + + return cls.model_validate({ + "@id": guid or generate_schema_guid(name), + "name": name, + "description": description, + "properties": props, + "required": list(props), + }) + + def validate(self, filepath: str) -> List[ValidationErrorRecord]: + """ + Structural validation: re-open the file and compare each declared + dataset's path / shape / dtype / chunks against the file. Reads structure + only (never array data). + """ + import h5py + + errors: List[ValidationErrorRecord] = [] + with h5py.File(filepath, "r") as fh: + for path, declared in self.properties.items(): + obj = fh.get(path) + if obj is None: + errors.append(ValidationErrorRecord( + message=f"Declared dataset '{path}' is absent from the file", + failed_keyword="required", field=path, path=path, + )) + continue + if not isinstance(obj, h5py.Dataset): + errors.append(ValidationErrorRecord( + message=f"'{path}' is a group in the file but declared as a dataset", + failed_keyword="type", field=path, path=path, + )) + continue + + if declared.shape is not None and list(obj.shape) != list(declared.shape): + errors.append(ValidationErrorRecord( + message=f"{path}: expected shape {declared.shape}, file has {list(obj.shape)}", + failed_keyword="const", field=f"{path}.shape", path=path, + )) + if declared.dtype is not None and str(obj.dtype) != declared.dtype: + errors.append(ValidationErrorRecord( + message=f"{path}: expected dtype {declared.dtype!r}, file has {str(obj.dtype)!r}", + failed_keyword="type", field=f"{path}.dtype", path=path, + )) + if declared.chunks is not None: + file_chunks = list(obj.chunks) if obj.chunks else None + if file_chunks != list(declared.chunks): + errors.append(ValidationErrorRecord( + message=f"{path}: expected chunks {declared.chunks}, file has {file_chunks}", + failed_keyword="const", field=f"{path}.chunks", path=path, + )) + # Compound-dtype fields: every declared field must be present. + # Only enforced when the file dataset is actually compound — a + # plain array declared with nested columns (legacy flattened + # form) has no dtype fields to check against. + file_names = set(obj.dtype.names or ()) + if declared.properties and file_names: + for fname in declared.properties: + if fname not in file_names: + errors.append(ValidationErrorRecord( + message=f"{path}: declared field '{fname}' absent from compound dtype", + failed_keyword="required", field=fname, path=path, + )) + + if self.additionalProperties is False: + declared_paths = set(self.properties) + seen = [] + + def collect(path, obj): + if isinstance(obj, h5py.Dataset) and path not in declared_paths: + seen.append(path) + + fh.visititems(collect) + for path in seen: + errors.append(ValidationErrorRecord( + message=f"File has undeclared dataset '{path}'", + failed_keyword="additionalProperties", field=path, path=path, + )) + return errors diff --git a/fairscape_models/schema/image.py b/fairscape_models/schema/image.py new file mode 100644 index 0000000..6b05cc9 --- /dev/null +++ b/fairscape_models/schema/image.py @@ -0,0 +1,199 @@ +""" +image.py — the `image` schema type: DICOM (and, by extension, NIfTI / OME-Zarr). + +`properties` are named, typed axes (`AxisProperty`, shared from base.py). Modality +and anatomy carry real coded values (SNOMED CT / RadLex). `nativeMetadata` passes +a whitelist of technical/acquisition tags through verbatim as the PS3.18 Annex F +DICOM JSON Model. Patient/identity tags are excluded by design — the schema +describes structure, not the subject. + +Reader lib: pydicom (headers only — pixels are never read). +""" + +import os +from typing import Any, Dict, List, Literal, Optional + +from pydantic import Field + +from fairscape_models.schema.base import ( + AxisProperty, + NonTabularSchema, + ValidationErrorRecord, + compare_scalar, + generate_schema_guid, +) +from fairscape_models.schema.ontology import MODALITY_MAP, anatomy_terms + +# (group, element, keyword, VR) whitelist of technical/acquisition tags to pass +# through as nativeMetadata. Patient/identity tags are deliberately excluded. +_DICOM_PASSTHROUGH = [ + (0x0008, 0x0060, "Modality", "CS"), + (0x0008, 0x0070, "Manufacturer", "LO"), + (0x0018, 0x0050, "SliceThickness", "DS"), + (0x0028, 0x0002, "SamplesPerPixel", "US"), + (0x0028, 0x0004, "PhotometricInterpretation", "CS"), + (0x0028, 0x0010, "Rows", "US"), + (0x0028, 0x0011, "Columns", "US"), + (0x0028, 0x0030, "PixelSpacing", "DS"), + (0x0028, 0x0100, "BitsAllocated", "US"), + (0x0028, 0x0101, "BitsStored", "US"), + (0x0028, 0x0103, "PixelRepresentation", "US"), + (0x0028, 0x1052, "RescaleIntercept", "DS"), + (0x0028, 0x1053, "RescaleSlope", "DS"), +] +_NUMERIC_VR = {"US", "SS", "UL", "SL", "FL", "FD", "DS", "IS"} +_MM_URL = "http://qudt.org/vocab/unit/MilliM" + + +def _anatomy(ds): + """ + (anatomy_label, snomed_url) for a DICOM image. Prefer the coded + AnatomicRegionSequence (0008,2218); fall back to the free-text + BodyPartExamined (0018,0015) resolved through the Annex L crosswalk. + (None, None) if neither is present. + """ + seq = ds.get("AnatomicRegionSequence") + if seq: + item = seq[0] + code = str(getattr(item, "CodeValue", "") or "") + scheme = str(getattr(item, "CodingSchemeDesignator", "") or "").upper() + meaning = str(getattr(item, "CodeMeaning", "") or "") or None + if code and scheme in ("SCT", "SRT", "SNOMED"): # SNOMED CT (current/retired designators) + return meaning, f"http://snomed.info/id/{code}" + return anatomy_terms(str(ds.get("BodyPartExamined", "")) or None) + + +def _dicom_json_value(elem): + """Render a pydicom element as a DICOM JSON Model Value array (PS3.18 Annex F).""" + v = elem.value + seq = list(v) if isinstance(v, (list, tuple)) or elem.VM > 1 else [v] + if elem.VR in _NUMERIC_VR: + out = [] + for x in seq: + fx = float(x) + out.append(int(fx) if fx.is_integer() else fx) + return out + return [str(x) for x in seq] + + +class ImageSchema(NonTabularSchema): + """`image` — DICOM. `properties` are named spatial axes.""" + + evi_schema_type: Literal["image"] = Field(default="image", alias="EVI:schemaType") + properties: Dict[str, AxisProperty] + + modality: Optional[str] = Field(default=None, description="Acquisition modality, e.g. 'CT', 'MR'") + modalityURL: Optional[str] = Field(default=None, description="SNOMED CT modality URI") + modalityRadLexURL: Optional[str] = Field(default=None) + dicomModalityCode: Optional[str] = Field(default=None, description="DICOM (0008,0060) modality code — the canonical modality identity") + dicomModalityCodeSystem: Optional[str] = Field(default=None, description="Coding-system URI for dicomModalityCode (the DICOM 'DCM' scheme)") + anatomy: Optional[str] = Field(default=None) + anatomyURL: Optional[str] = Field(default=None, description="Uberon / FMA anatomy URI") + anatomyRadLexURL: Optional[str] = Field(default=None) + + dtype: Optional[str] = Field(default=None) + bitsAllocated: Optional[int] = Field(default=None) + + nativeMetadata: Optional[Dict[str, Any]] = Field(default=None) + + @classmethod + def infer(cls, filepath: str, name: str, description: str, + guid: Optional[str] = None) -> "ImageSchema": + import pydicom + + ds = pydicom.dcmread(filepath, stop_before_pixels=True) + + rows = int(ds.Rows) + cols = int(ds.Columns) + ps = list(ds.get("PixelSpacing", [None, None])) # DICOM order: [row(dy), col(dx)] + dy, dx = (float(ps[0]), float(ps[1])) if ps[0] is not None else (None, None) + + props: Dict[str, AxisProperty] = {} + props["x"] = AxisProperty(description="Column axis (in-plane)", index=0, type="integer", + axisType="space", size=cols, spacing=dx, + unit="mm" if dx is not None else None, + unitURL=_MM_URL if dx is not None else None) + props["y"] = AxisProperty(description="Row axis (in-plane)", index=1, type="integer", + axisType="space", size=rows, spacing=dy, + unit="mm" if dy is not None else None, + unitURL=_MM_URL if dy is not None else None) + n_frames = int(ds.get("NumberOfFrames", 1) or 1) + if n_frames > 1: + thick = ds.get("SliceThickness") + props["z"] = AxisProperty(description="Slice axis (through-plane)", index=2, type="integer", + axisType="space", size=n_frames, + spacing=float(thick) if thick is not None else None, + unit="mm" if thick is not None else None, + unitURL=_MM_URL if thick is not None else None) + + modality = str(ds.get("Modality", "")) or None + mt = MODALITY_MAP.get(modality) + # The DCM code (dicomModalityCode) is the canonical modality identity; + # dcm_system_uri is its coding system. SNOMED/RadLex are enrichment. + dcm_system = mt.dcm_system_uri if mt else None + snomed = mt.snomed_uri if mt else None + radlex = mt.radlex_uri if mt else None + anatomy, anatomy_url = _anatomy(ds) + bits = int(ds.get("BitsAllocated", 0)) or None + signed = int(ds.get("PixelRepresentation", 0)) == 1 + dtype = (f"int{bits}" if signed else f"uint{bits}") if bits else None + + native = {} + for g, e, _kw, _vr in _DICOM_PASSTHROUGH: + if (g, e) in ds: + elem = ds[g, e] + native[f"{g:04X}{e:04X}"] = {"vr": elem.VR, "Value": _dicom_json_value(elem)} + + return cls.model_validate({ + "@id": guid or generate_schema_guid(name), + "name": name, + "description": description, + "conformsTo": {"@id": "https://dicom.nema.org/medical/dicom/current/output/chtml/part18/chapter_f.html"}, + "modality": modality, + "modalityURL": snomed, + "modalityRadLexURL": radlex, + "dicomModalityCode": modality, + "dicomModalityCodeSystem": dcm_system, + "anatomy": anatomy, + "anatomyURL": anatomy_url, + "dtype": dtype, + "bitsAllocated": bits, + "unitSystem": "http://unitsofmeasure.org", + "nativeMetadata": native or None, + "properties": props, + "required": list(props), + }) + + def validate(self, filepath: str) -> List[ValidationErrorRecord]: + """ + Structural validation: re-infer from the DICOM header and compare the + fields this schema declares. Reports missing axes and geometry/modality + drift (a None on either side means "unconstrained"). + """ + observed = type(self).infer(filepath, name=self.name, description="revalidation probe") + errors: List[ValidationErrorRecord] = [] + + for attr in ("dicomModalityCode", "dtype", "bitsAllocated"): + compare_scalar(errors, getattr(self, attr), getattr(observed, attr), + path=None, field=attr) + + for axis, declared in self.properties.items(): + found = observed.properties.get(axis) + if found is None: + errors.append(ValidationErrorRecord( + message=f"Declared axis '{axis}' is absent from the file", + failed_keyword="required", field=axis, path=axis, + )) + continue + for attr in ("size", "spacing"): + compare_scalar(errors, getattr(declared, attr), getattr(found, attr), + path=axis, field=f"{axis}.{attr}") + + if self.additionalProperties is False: + for axis in observed.properties: + if axis not in self.properties: + errors.append(ValidationErrorRecord( + message=f"File has undeclared axis '{axis}'", + failed_keyword="additionalProperties", field=axis, path=axis, + )) + return errors diff --git a/fairscape_models/schema/ndarray.py b/fairscape_models/schema/ndarray.py new file mode 100644 index 0000000..6fb60cd --- /dev/null +++ b/fairscape_models/schema/ndarray.py @@ -0,0 +1,37 @@ +""" +ndarray.py — the `ndarray` schema type: generic HDF5 / netCDF / Zarr tensors. + +Described the CF way: named coordinate axes with a controlled `standardName` and +a UDUNITS unit (shared `AxisProperty` from base.py), plus the variable's own +`measures` (standard_name) and `unit`. A native N-D descriptor that replaces the +lossy HDF5-as-tabular adapter (which flattened axes into column_i). + +Model-only in this version: there is no single-file format that maps one file to +one ndarray, so `infer`/`validate` are not overridden yet. `HDF5Schema` +(hdf5.py) is the concrete container that describes real .h5 files; a single +ndarray uses these fields as an `HDF5Schema` member. Zarr/netCDF inference can +add a classmethod here later. +""" + +from typing import Dict, List, Literal, Optional + +from pydantic import Field + +from fairscape_models.schema.base import AxisProperty, NonTabularSchema + + +class NDArraySchema(NonTabularSchema): + """ + `ndarray` — generic HDF5 / netCDF / Zarr tensors. `properties` are named + coordinate axes. + """ + evi_schema_type: Literal["ndarray"] = Field(default="ndarray", alias="EVI:schemaType") + properties: Dict[str, AxisProperty] = Field(default_factory=dict) + + shape: Optional[List[int]] = Field(default=None) + dtype: Optional[str] = Field(default=None) + chunks: Optional[List[int]] = Field(default=None) + compression: Optional[str] = Field(default=None) + standardNameVocabulary: Optional[str] = Field(default=None, description="CF Standard Name Table URL") + measures: Optional[str] = Field(default=None, description="The array variable's CF standard_name") + unit: Optional[str] = Field(default=None, description="The array variable's unit") diff --git a/fairscape_models/schema/ontology.py b/fairscape_models/schema/ontology.py new file mode 100644 index 0000000..c993db1 --- /dev/null +++ b/fairscape_models/schema/ontology.py @@ -0,0 +1,115 @@ +""" +ontology.py — verified ontology look-ups + value coercion for schema inference. + +The lookup tables are LOADED FROM DATA, not hand-typed, so the codes can be +audited against their published source and grown without editing Python. The +CSVs ship as package data in `reference_tables/`: + + dicom_modality.csv DICOM CID 29 "Acquisition Modality" — the closed + modality code set (PS3.16). `snomed_uri`/`radlex_uri` + are enrichment columns filled in by review. + dicom_bodypart_snomed.csv DICOM PS3.16 Annex L — the standard correspondence + from the dirty `BodyPartExamined` (0018,0015) string + to a SNOMED CT Anatomic Region code. Parsed verbatim + from the spec; nothing invented. + units_qudt.csv UCUM code + QUDT unit URI, keyed on the native unit + string (hand-verified set). + +A token that is NOT in these tables gets a null URL (never a guessed one) — the +raw string is passed through for the human `schema add-*` annotation step. +""" + +import csv +from collections import namedtuple +try: + # importlib.resources.files() is Python 3.9+ + from importlib.resources import files as _resource_files +except ImportError: + # Python 3.8 falls back to the importlib_resources backport + from importlib_resources import files as _resource_files + +__all__ = [ + "UNIT_MAP", + "MODALITY_MAP", + "ModalityTerms", + "BODYPART_MAP", + "STD_12_LEADS", + "LOINC_12LEAD", + "unit_terms", + "anatomy_terms", + "as_num", +] + + +def _read_table(name): + path = _resource_files("fairscape_models.schema") / "reference_tables" / name + with path.open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def _blank_to_none(s): + s = (s or "").strip() + return s or None + + +# UCUM unit code + QUDT unit URI, keyed on the lower-cased native unit string. +UNIT_MAP = { + row["raw_unit"].strip().lower(): (row["ucum_code"].strip(), _blank_to_none(row["qudt_uri"])) + for row in _read_table("units_qudt.csv") +} + +# DICOM modality code -> coding-system URI (DICOM DCM scheme; the canonical home +# of the modality code, filled for every row) + optional SNOMED/RadLex enrichment +# (blank -> None so it drops out under exclude_none). +ModalityTerms = namedtuple("ModalityTerms", ["dcm_system_uri", "snomed_uri", "radlex_uri"]) +MODALITY_MAP = { + row["modality_code"].strip(): ModalityTerms( + _blank_to_none(row["dcm_system_uri"]), + _blank_to_none(row["snomed_uri"]), + _blank_to_none(row["radlex_uri"]), + ) + for row in _read_table("dicom_modality.csv") +} + +# DICOM BodyPartExamined defined term (upper-case) -> (code meaning, SNOMED URI). +# The URI is built from the Annex L SNOMED code; the meaning is the spec's label. +BODYPART_MAP = { + row["body_part_examined"].strip().upper(): ( + row["code_meaning"].strip(), + f"http://snomed.info/id/{row['snomed_code'].strip()}", + ) + for row in _read_table("dicom_bodypart_snomed.csv") +} + +# The 12 standard leads of a diagnostic ECG; if a signal record carries exactly +# these, we attach the verified LOINC 12-lead EKG panel code at record level. +STD_12_LEADS = {"I", "II", "III", "AVR", "AVL", "AVF", "V1", "V2", "V3", "V4", "V5", "V6"} +LOINC_12LEAD = "https://loinc.org/34534-8/" + + +def unit_terms(raw): + """(ucum_code, qudt_url) for a native unit token; passthrough + null URL if unknown.""" + if not raw: + return None, None + hit = UNIT_MAP.get(str(raw).strip().lower()) + return hit if hit else (str(raw).strip(), None) + + +def anatomy_terms(raw): + """ + (anatomy_label, snomed_url) for a DICOM BodyPartExamined term via the Annex L + crosswalk; passthrough label + null URL if the term is unknown. (None, None) + for an empty/absent value — anatomy is then left for the human step. + """ + if not raw: + return None, None + hit = BODYPART_MAP.get(str(raw).strip().upper()) + return hit if hit else (str(raw).strip(), None) + + +def as_num(x): + """int if integral else float — keeps 100 not 100.0 in the JSON.""" + if x is None: + return None + f = float(x) + return int(f) if f.is_integer() else f diff --git a/fairscape_models/schema/reference_tables/dicom_bodypart_snomed.csv b/fairscape_models/schema/reference_tables/dicom_bodypart_snomed.csv new file mode 100644 index 0000000..a3bf6fa --- /dev/null +++ b/fairscape_models/schema/reference_tables/dicom_bodypart_snomed.csv @@ -0,0 +1,341 @@ +body_part_examined,snomed_code,code_meaning,fma_code,umls_cui +3RDVENTRICLE,49841001,Third ventricle,,C0149555 +4THVENTRICLE,35918002,Fourth ventricle,,C0149556 +ABDOMEN,818981001,Abdomen,,C5231290 +ABDOMENPELVIS,818982008,Abdomen and Pelvis,,C5231291 +ABDOMINALAORTA,7832008,Abdominal aorta,,C0003484 +ACA,60176003,Anterior cerebral artery,50028,C0149561 +ACJOINT,85856004,Acromioclavicular joint,,C0001208 +ADRENAL,23451007,Adrenal gland,,C0001625 +AMNIOTICFLUID,77012006,Amniotic fluid,, +ANKLE,70258002,Ankle joint,,C0003087 +ANTCARDIACV,194996006,Anterior cardiac vein,,C0226662 +ANTCOMMA,8012006,Anterior communicating artery,,C0149562 +ANTECUBITALV,128553008,Antecubital vein,,C1276271 +ANTSPINALA,17388009,Anterior spinal artery,,C0149603 +ANTTIBIALA,68053000,Anterior tibial artery,,C0085816 +ANUS,53505006,Anus,,C0003461 +ANUSRECTUMSIGMD,110612005,"Anus, rectum and sigmoid colon",,C1267595 +AORTA,15825003,Aorta,,C0003483 +AORTICARCH,57034009,Aortic arch,,C0003489 +APPENDIX,66754008,Appendix,,C0003617 +ARTERY,51114001,Artery,,C0003842 +ASCAORTA,54247002,Ascending aorta,,C0003956 +ASCENDINGCOLON,9040008,Ascending colon,,C0227375 +ATLANTOAXIAL,62555009,Atlantal-axial joint,,C0224585 +ATLANTOOCCIPITAL,20292002,Atlanto-occipital joint,,C0004169 +AXILLA,91470000,Axilla,,C0004454 +AXILLARYA,67937003,Axillary Artery,,C0004455 +AXILLARYV,68705008,Axillary vein,,C0004456 +AZYGOSVEIN,72107004,Azygos vein,,C0004526 +BACK,77568009,Back,,C0004600 +BASILARA,59011009,Basilar artery,,C0004811 +BILEDUCT,28273000,Bile duct,,C0005400 +BILIARYTRACT,34707002,Biliary tract,,C0005423 +BLADDER,89837001,Bladder,,C0005682 +BLADDERURETHRA,110837003,Bladder and urethra,,C1268386 +BRACHIALA,17137000,Brachial artery,,C0006087 +BRACHIALV,20115005,Brachial vein,,C0226812 +BRAIN,12738006,Brain,,C0006104 +BREAST,76752008,Breast,,C0006141 +BRONCHUS,955009,Bronchus,,C0006255 +BULB,21479005,Carotid bulb,50094,C0007281 +BUTTOCK,46862004,Buttock,,C0006497 +CALCANEUS,80144004,Calcaneus,,C0006655 +CALF,53840002,Calf of leg,,C0230445 +CARDIOVASCSYS,113257007,Cardiovascular system,,C0007226 +CAROTID,69105007,Carotid Artery,,C0007272 +CARPUS,8205005,Carpus,,C0043262 +CCA,32062004,Common carotid artery,,C0162859 +CELIACA,57850000,Celiac artery,50737,C0007569 +CEPHALICV,20699002,Cephalic vein,,C0226802 +CEREBELLUM,113305005,Cerebellum,,C0007765 +CEREBHEMISPHERE,372073000,Cerebral hemisphere,,C0228174 +CEREBRALA,88556005,Cerebral artery,,C0007770 +CERVIX,71252005,Cervix,,C0007874 +CFA,181347005,Common femoral artery,323778,C0447105 +CFV,397363009,Common femoral vein,323829,C1275667 +CHEEK,60819002,Cheek,,C0007966 +CHEST,816094009,Chest,,C5230958 +CHESTABDOMEN,416550000,Chest and Abdomen,,C1442171 +CHESTABDPELVIS,416775004,"Chest, Abdomen and Pelvis",,C1562547 +CHOROIDPLEXUS,80621003,Choroid plexus,61934,C0008524 +CIRCLEOFWILLIS,11279006,Circle of Willis,,C0008812 +CLAVICLE,51299004,Clavicle,,C0008913 +COCCYX,64688005,Coccyx,,C0009194 +COLON,71854001,Colon,,C0009368 +COMILIACA,73634005,Common iliac artery,,C1261084 +COMILIACV,46027005,Common iliac vein,,C0226758 +COMMONBILEDUCT,79741001,Common bile duct,,C0009437 +CORNEA,28726007,Cornea,,C0010031 +CORONARYARTERY,41801008,Coronary artery,,C0205042 +CORONARYSINUS,90219004,Coronary sinus,,C0456944 +CSPINE,122494005,Cervical spine,,C0728985 +CTSPINE,1217257000,Cervico-thoracic spine,,C5687879 +CULDESAC,53843000,Rectouterine pouch,,C0013075 +DESCAORTA,281130003,Descending aorta,,C0011666 +DESCENDINGCOLON,32622004,Descending colon,,C0227389 +DIGIT,82680008,Digit,,C0582802 +DUODENUM,38848004,Duodenum,,C0013303 +EAC,84301002,External auditory canal,,C0013444 +EAR,117590005,Ear,,C0013443 +ECA,22286001,External carotid artery,,C0007275 +ELBOW,16953009,Elbow joint,,C0013770 +ENDOARTERIAL,51114001,Endo-arterial,,C0003842 +ENDOCARDIAC,80891009,Endo-cardiac,,C0018787 +ENDOESOPHAGEAL,32849002,Endo-esophageal,,C0014876 +ENDOMETRIUM,2739003,Endometrium,,C0014180 +ENDONASAL,53342003,Endo-nasal,,C0225425 +ENDONASOPHARYNYX,18962004,Endo-nasopharyngeal,,C0225497 +ENDORECTAL,34402009,Endo-rectal,,C0034896 +ENDORENAL,64033007,Endo-renal,,C0022646 +ENDOURETERIC,87953007,Endo-ureteric,,C0041951 +ENDOURETHRAL,13648007,Endo-urethral,,C0041967 +ENDOVAGINAL,76784001,Endo-vaginal,,C0042232 +ENDOVASCULAR,59820001,Endo-vascular,,C0005847 +ENDOVENOUS,29092000,Endo-venous,,C0042449 +ENDOVESICAL,48367006,Endo-vesical,,C0227710 +EPIDIDYMIS,87644002,Epididymis,18255,C0014533 +EPIGASTRIC,27947004,Epigastric region,,C0230185 +ESOPHAGUS,32849002,Esophagus,,C0014876 +EXTILIACA,113269004,External iliac artery,,C0226398 +EXTILIACV,63507001,External iliac vein,,C0226761 +EXTJUGV,71585003,External jugular vein,13110,C0226543 +EXTREMITY,66019005,Extremity,,C0015385 +EYE,81745001,Eye,,C0015392 +EYELID,80243003,Eyelid,,C0015426 +FACE,89545001,Face,,C0015450 +FACIALA,23074001,Facial artery,,C0226109 +FEMORALA,7657000,Femoral artery,,C0015801 +FEMORALV,83419000,Femoral vein,,C0015809 +FEMUR,71341001,Femur,,C0015811 +FIBULA,87342007,Fibula,,C0016068 +FINGER,7569003,Finger,,C0016129 +FLANK,58602004,Flank,,C0230171 +FONTANEL,79361005,Fontanel of skull,,C0224548 +FOOT,56459004,Foot,,C0016504 +FOREARM,14975008,Forearm,,C0016536 +FOREFETLOCK,13190002,Fetlock of forelimb,,C0521445 +FOREFOOT,419176008,Forefoot,,C1630649 +FORENAVICULAR,30518006,Navicular of forefoot,,C0223724 +FOREPASTERN,31329001,Pastern of forefoot,,C0230368 +FRONTALSINUS,55060009,Frontal sinus,,C0016734 +GALLBLADDER,28231008,Gallbladder,,C0016976 +GASTRICV,110568007,Gastric vein,,C0750610 +GENICULARA,128559007,Genicular artery,,C0447108 +GESTSAC,300571009,Gestational sac,,C0577295 +GLUTEAL,46862004,Gluteal region,,C0006497 +GSV,60734001,Great saphenous vein,21376,C0392907 +HAND,85562004,Hand,,C0018563 +HEAD,69536005,Head,,C0018670 +HEADNECK,774007,Head and Neck,,C0460004 +HEART,80891009,Heart,,C0018787 +HEPATICA,76015000,Hepatic artery,,C0019145 +HEPATICV,8993003,Hepatic vein,,C0019155 +HINDFETLOCK,113351006,Fetlock of hindlimb,,C0521446 +HINDFOOT,416804009,Hindfoot,,C0230459 +HINDNAVICULAR,75772009,Navicular of hindfoot,,C0223947 +HINDPASTERN,18525008,Pastern of hindfoot,,C0230455 +HIP,24136001,Hip joint,,C0019558 +HUMERUS,85050009,Humerus,,C0020164 +HYPOGASTRIC,11708003,Hypogastric region,,C0230189 +HYPOPHARYNX,81502006,Hypopharynx,,C0020629 +IAC,361078006,Internal Auditory Canal,,C1283773 +ICA,86117002,Internal carotid artery,,C0007276 +ILEUM,34516001,Ileum,,C0020885 +ILIACA,10293006,Iliac artery,,C0020887 +ILIACV,244411005,Iliac vein,,C0020888 +ILIUM,22356005,Ilium,,C0020889 +INFMESA,33795007,Inferior mesenteric artery,,C0162860 +INFVENACAVA,64131007,Inferior vena cava,,C0042458 +INGUINAL,26893007,Inguinal region,,C0018246 +INNOMINATEA,12691009,Innominate artery,,C0006094 +INNOMINATEV,8887007,Innominate vein,,C0006095 +INTILIACA,90024005,Internal iliac artery,,C0226364 +INTJUGULARV,12123001,Internal jugular vein,,C0226550 +INTMAMMARYA,69327007,Internal mammary artery,,C0226276 +INTRACRANIAL,1101003,Intracranial,,C0230041 +JAW,661005,Jaw region,,C3887617 +JEJUNUM,21306003,Jejunum,,C0022378 +JOINT,39352004,Joint,,C0022417 +KIDNEY,64033007,Kidney,,C0022646 +KNEE,72696002,Knee,,C0022742 +LACRIMALA,59749000,Lacrimal artery,,C0226171 +LARGEINTESTINE,14742008,Large intestine,,C0021851 +LARYNX,4596009,Larynx,,C0023078 +LATRIUM,82471001,Left atrium,,C0225860 +LATVENTRICLE,66720007,Lateral Ventricle,78448,C0152279 +LEGS,42694008,All legs,,C0230331 +LFEMORALA,113270003,Left femoral artery,,C0226448 +LHEPATICV,273202007,Left hepatic vein,14339,C0226708 +LHYPOCHONDRIAC,133945003,Left hypochondriac region,,C0738591 +LINGUALA,113264009,Lingual artery,,C0226104 +LINGUINAL,85119005,Left inguinal region,,C0230321 +LIVER,10200004,Liver,,C0023884 +LLQ,68505006,Left lower quadrant of abdomen,,C0230180 +LLUMBAR,1017210004,Left lumbar region,,C5439491 +LOWERLEG,30021000,Lower leg,,C1140621 +LOWERLIMB,61685007,Lower limb,,C0023216 +LOWERTRUNK,63337009,Lower trunk,,C0230094 +LPORTALV,70253006,Left portal vein,15415,C0933785 +LPULMONARYA,50408007,Left pulmonary artery,,C0226069 +LSPINE,122496007,Lumbar spine,,C0024091 +LSSPINE,1217253001,Lumbo-sacral spine,,C5687876 +LSUPPULMONARYV,43863001,Superior left pulmonary vein,,C0226682 +LUMBAR,52612000,Lumbar region,,C0024090 +LUMBARA,34635009,Lumbar artery,,C0226408 +LUMEN,91747007,Lumen of blood vessel,,C0524424 +LUNG,39607008,Lung,,C0024109 +LUQ,86367003,Left upper quadrant of abdomen,,C0230179 +LVENTRICLE,87878005,Left ventricle,,C0225897 +MANDIBLE,91609006,Mandible,,C0024687 +MASTOID,59066005,Mastoid bone,,C0446908 +MAXILLA,70925003,Maxilla,,C0024947 +MCA,17232002,Middle cerebral artery,50079,C0149566 +MEDIASTINUM,72410000,Mediastinum,,C0025066 +MESENTRICA,86570000,Mesenteric artery,,C0025465 +MESENTRICV,128583004,Mesenteric vein,,C0025473 +METACARPUS,36455000,Metacarpus,,C0025526 +METATARSUS,280711000,Metatarsus,,C0025590 +MIDHEPATICV,273099000,Middle hepatic vein,14340,C0226707 +MORISONSPOUCH,243977002,Morisons pouch,,C0446609 +MOUTH,123851003,Mouth,,C0230028 +NASOPHARYNX,360955006,Nasopharynx,,C1283682 +NECK,45048000,Neck,,C0027530 +NECKCHEST,417437006,Neck and Chest,,C1562459 +NECKCHESTABDOMEN,416152001,"Neck, Chest and Abdomen",,C1562378 +NECKCHESTABDPELV,416319003,"Neck, Chest, Abdomen and Pelvis",,C1562776 +NOSE,45206002,Nose,,C0028429 +OCCIPTALV,32114007,Occipital vein,,C0226579 +OCCPITALA,31145008,Occipital artery,,C0226117 +OPHTHALMICA,53549008,Ophthalmic artery,,C0029078 +OPTICCANAL,55024004,Optic canal,,C0450102 +ORBIT,363654007,Orbital structure,,C0029180 +OVARY,15497006,Ovary,,C0029939 +PANCBILEDUCT,110621006,Pancreatic duct and bile duct systems,,C1267614 +PANCREAS,15776009,Pancreas,,C0030274 +PANCREATICDUCT,69930009,Pancreatic duct,,C0030288 +PARASTERNAL,91691001,Parasternal,,C0458345 +PARATHYROID,111002,Parathyroid,,C0030518 +PAROTID,45289007,Parotid gland,,C0030580 +PATELLA,64234005,Patella,,C0030647 +PCA,70382005,Posterior cerebral artery,50583,C0149576 +PELVIS,816092008,Pelvis,,C5230955 +PELVISLOWEXTREMT,1231522001,Pelvis and lower extremities,,C5688969 +PENILEA,282044005,Penile artery,66318,C0559907 +PENIS,18911002,Penis,,C0030851 +PERINEUM,38864007,Perineum,,C0031066 +PERONEALA,8821006,Peroneal artery,,C0226476 +PHANTOM,706342009,Phantom,,C0282611 +PHARYNX,54066008,Pharynx,,C0031354 +PHARYNXLARYNX,312535008,Pharynx and larynx,,C0729889 +PLACENTA,78067005,Placenta,,C0032043 +POPLITEALA,43899006,Popliteal artery,,C0032649 +POPLITEALFOSSA,32361000,Popliteal fossa,,C0230436 +POPLITEALV,56849005,Popliteal vein,44327,C0032652 +PORTALV,32764006,Portal vein,66645,C0032718 +POSCOMMA,43119007,Posterior communicating artery,,C0149559 +POSTIBIALA,13363002,Posterior tibial artery,,C0086835 +PROFFEMA,31677005,Profunda femoris artery,20741,C0226455 +PROFFEMV,23438002,Profunda femoris vein,51041,C0226841 +PROSTATE,41216001,Prostate,,C0033572 +PULMONARYA,81040000,Pulmonary artery,,C0034052 +PULMONARYV,122972007,Pulmonary vein,,C0034090 +RADIALA,45631007,Radial artery,,C0162857 +RADIUS,62413002,Radius,,C0034627 +RADIUSULNA,110535000,Radius and ulna,,C1267080 +RATRIUM,73829009,Right atrium,,C0225844 +RECTUM,34402009,Rectum,,C0034896 +RENALA,2841007,Renal artery,,C0035065 +RENALV,56400007,Renal vein,,C0035092 +RETROPERITONEUM,82849001,Retroperitoneum,,C0035359 +RFEMORALA,69833005,Right femoral artery,,C0226447 +RHEPATICV,272998002,Right hepatic vein,14338,C0226706 +RHYPOCHONDRIAC,133946002,Right hypochondriac region,,C0738590 +RIB,113197003,Rib,,C0035561 +RINGUINAL,37117007,Right inguinal region,,C0230318 +RLQ,48544008,Right lower quadrant of abdomen,,C0230178 +RLUMBAR,1017211000,Right lumbar region,,C5439490 +RPORTALV,73931004,Right portal vein,15414,C0226730 +RPULMONARYA,78480002,Right pulmonary artery,,C0226054 +RSUPPULMONARYV,8629005,Superior right pulmonary vein,,C0226671 +RUQ,50519007,Right upper quadrant of abdomen,,C0230177 +RVENTRICLE,53085002,Right ventricle,,C0225883 +SAPHENOUSV,362072009,Saphenous vein,,C0036186 +SCALP,41695006,Scalp,,C0036270 +SCAPULA,79601000,Scapula,,C0036277 +SCJOINT,7844006,Sternoclavicular joint,,C0038291 +SCLERA,18619003,Sclera,,C0036410 +SCROTUM,20233005,Scrotum,,C0036471 +SELLA,42575006,Sella turcica,,C0036609 +SEMVESICLE,64739004,Seminal vesicle,19386,C0036628 +SESAMOID,58742003,Sesamoid bones of foot,,C0278418 +SFA,181349008,Superficial femoral artery,323777,C0447106 +SFJ,128587003,Saphenofemoral junction,,C0447132 +SFV,397364003,Superficial femoral vein,,C1301369 +SHOULDER,16982005,Shoulder,,C0037004 +SIGMOID,60184004,Sigmoid colon,,C0227391 +SIJOINT,39723000,Sacroiliac joint,,C0036036 +SKULL,89546000,Skull,,C0037303 +SMA,42258001,Superior mesenteric artery,,C0162861 +SMALLINTESTINE,30315005,Small intestine,,C0021852 +SPINALCORD,2748008,Spinal cord,,C0037925 +SPINE,421060004,Spine,,C0037949 +SPLEEN,78961009,Spleen,,C0037993 +SPLENICA,22083002,Splenic artery,,C0037996 +SPLENICV,35819009,Splenic vein,,C0038001 +SSPINE,54735007,Sacrum,,C0036037 +STERNUM,56873002,Sternum,,C0038293 +STIFLE,116010006,Stiffle,,C1456798 +STOMACH,69695003,Stomach,,C0038351 +SUBCLAVIANA,36765005,Subclavian artery,,C0038530 +SUBCLAVIANV,9454009,Subclavian vein,,C0038532 +SUBCOSTAL,19695001,Subcostal,,C0442184 +SUBMANDIBULAR,54019009,Submandibular gland,,C0038556 +SUPRACLAVICULAR,77621008,Supraclavicular region of neck,,C0230078 +SUPRAPUBIC,11708003,Suprapubic region,,C0230189 +SUPTHYROIDA,72021004,Superior thyroid artery,,C0226093 +SVC,48345005,Superior vena cava,,C0042459 +TAIL,18149002,Coccygeal vertrebrae,,C0223616 +TARSUS,108371006,Tarsus,,C0039316 +TESTIS,40689003,Testis,,C0039597 +THALAMUS,42695009,Thalamus,62007,C0039729 +THIGH,68367000,Thigh,,C0039866 +THORACICAORTA,113262008,Thoracic aorta,,C1522460 +THORAX,816094009,Chest,,C5230958 +THUMB,76505004,Thumb,,C0040067 +THYMUS,9875009,Thymus,,C0040113 +THYROID,69748006,Thyroid,,C0040132 +TIBIA,12611008,Tibia,,C0040184 +TIBIAFIBULA,110536004,Tibia and fibula,,C0224692 +TLSPINE,1217256009,Thoraco-lumbar spine,,C5687878 +TMJ,53620006,Temporomandibular joint,,C0039493 +TOE,29707007,Toe,,C0040357 +TONGUE,21974007,Tongue,,C0040408 +TRACHEA,44567001,Trachea,,C0040578 +TRACHEABRONCHUS,110726009,Trachea and bronchus,,C1268276 +TRANSVERSECOLON,485005,Transverse colon,,C0227386 +TRUNK,22943007,Trunk,,C0460005 +TSPINE,122495006,Thoracic spine,,C0581269 +UGITRACT,62834003,Upper gastro-intestinal tract,,C3203348 +ULNA,23416004,Ulna,,C0041600 +ULNARA,44984001,Ulnar artery,,C0162858 +UMBILICAL,90290004,Umbilical region,,C0041638 +UMBILICALA,50536004,Umbilical artery,,C0041632 +UMBILICALV,284639000,Umbilical vein,,C0226734 +UPPERARM,40983000,Upper arm,,C0446516 +UPPERLIMB,53120007,Upper limb,,C1140618 +UPPERTRUNK,67734004,Upper trunk,,C0230093 +UPRURINARYTRACT,431491007,Upper urinary tract,,C2317509 +URETER,87953007,Ureter,,C0041951 +URETHRA,13648007,Urethra,,C0041967 +URINARYTRACT,431938005,Urinary tract,,C2316969 +UTERUS,35039007,Uterus,,C0042149 +VAGINA,76784001,Vagina,,C0042232 +VEIN,29092000,Vein,,C0042449 +VERTEBRALA,85234005,Vertebral artery,,C0042559 +VULVA,45292006,Vulva,,C0042993 +WHOLEBODY,38266002,Entire body,,C0229960 +WING,53036007,Wing,,C0043189 +WRIST,74670003,Wrist joint,,C1322271 +ZYGOMA,13881006,Zygoma,,C0043539 diff --git a/fairscape_models/schema/reference_tables/dicom_modality.csv b/fairscape_models/schema/reference_tables/dicom_modality.csv new file mode 100644 index 0000000..1c99daf --- /dev/null +++ b/fairscape_models/schema/reference_tables/dicom_modality.csv @@ -0,0 +1,44 @@ +modality_code,description,dcm_system_uri,snomed_uri,radlex_uri +AR,Autorefraction,http://dicom.nema.org/resources/ontology/DCM,, +BDUS,Ultrasound Bone Densitometry,http://dicom.nema.org/resources/ontology/DCM,, +BI,Biomagnetic Imaging,http://dicom.nema.org/resources/ontology/DCM,, +BMD,Bone Mineral Densitometry,http://dicom.nema.org/resources/ontology/DCM,, +CFM,Confocal Microscopy,http://dicom.nema.org/resources/ontology/DCM,, +CR,Computed Radiography,http://dicom.nema.org/resources/ontology/DCM,, +CT,Computed Tomography,http://dicom.nema.org/resources/ontology/DCM,http://snomed.info/id/77477000,http://radlex.org/RID/RID10321 +DG,Diaphanography,http://dicom.nema.org/resources/ontology/DCM,, +DMS,Dermoscopy,http://dicom.nema.org/resources/ontology/DCM,, +DX,Digital Radiography,http://dicom.nema.org/resources/ontology/DCM,, +ES,Endoscopy,http://dicom.nema.org/resources/ontology/DCM,, +GM,General Microscopy,http://dicom.nema.org/resources/ontology/DCM,, +IO,Intra-oral Radiography,http://dicom.nema.org/resources/ontology/DCM,, +IVOCT,Intravascular Optical Coherence Tomography,http://dicom.nema.org/resources/ontology/DCM,, +IVUS,Intravascular Ultrasound,http://dicom.nema.org/resources/ontology/DCM,, +KER,Keratometry,http://dicom.nema.org/resources/ontology/DCM,, +LEN,Lensometry,http://dicom.nema.org/resources/ontology/DCM,, +LS,Laser surface scan,http://dicom.nema.org/resources/ontology/DCM,, +MG,Mammography,http://dicom.nema.org/resources/ontology/DCM,, +MR,Magnetic Resonance,http://dicom.nema.org/resources/ontology/DCM,http://snomed.info/id/113091000, +NM,Nuclear Medicine,http://dicom.nema.org/resources/ontology/DCM,, +OAM,Ophthalmic Axial Measurements,http://dicom.nema.org/resources/ontology/DCM,, +OCT,Optical Coherence Tomography,http://dicom.nema.org/resources/ontology/DCM,, +OP,Ophthalmic Photography,http://dicom.nema.org/resources/ontology/DCM,, +OPM,Ophthalmic Mapping,http://dicom.nema.org/resources/ontology/DCM,, +OPT,Ophthalmic Tomography,http://dicom.nema.org/resources/ontology/DCM,, +OPTBSV,Ophthalmic Tomography B-scan Volume Analysis,http://dicom.nema.org/resources/ontology/DCM,, +OPTENF,Ophthalmic Tomography En Face,http://dicom.nema.org/resources/ontology/DCM,, +OPV,Ophthalmic Visual Field,http://dicom.nema.org/resources/ontology/DCM,, +OSS,Optical Surface Scanner,http://dicom.nema.org/resources/ontology/DCM,, +PA,Photoacoustic,http://dicom.nema.org/resources/ontology/DCM,, +PT,Positron emission tomography,http://dicom.nema.org/resources/ontology/DCM,, +PX,Panoramic X-Ray,http://dicom.nema.org/resources/ontology/DCM,, +RF,Radiofluoroscopy,http://dicom.nema.org/resources/ontology/DCM,, +RG,Radiographic imaging,http://dicom.nema.org/resources/ontology/DCM,, +RTIMAGE,RT Image,http://dicom.nema.org/resources/ontology/DCM,, +SM,Slide Microscopy,http://dicom.nema.org/resources/ontology/DCM,, +SRF,Subjective Refraction,http://dicom.nema.org/resources/ontology/DCM,, +TG,Thermography,http://dicom.nema.org/resources/ontology/DCM,, +US,Ultrasound,http://dicom.nema.org/resources/ontology/DCM,, +VA,Visual Acuity,http://dicom.nema.org/resources/ontology/DCM,, +XA,X-Ray Angiography,http://dicom.nema.org/resources/ontology/DCM,, +XC,External-camera Photography,http://dicom.nema.org/resources/ontology/DCM,, diff --git a/fairscape_models/schema/reference_tables/units_qudt.csv b/fairscape_models/schema/reference_tables/units_qudt.csv new file mode 100644 index 0000000..24901c5 --- /dev/null +++ b/fairscape_models/schema/reference_tables/units_qudt.csv @@ -0,0 +1,7 @@ +raw_unit,ucum_code,qudt_uri +mv,mV,http://qudt.org/vocab/unit/MilliV +mm,mm,http://qudt.org/vocab/unit/MilliM +um,um,http://qudt.org/vocab/unit/MicroM +s,s,http://qudt.org/vocab/unit/SEC +sec,s,http://qudt.org/vocab/unit/SEC +hz,Hz,http://qudt.org/vocab/unit/HZ diff --git a/fairscape_models/schema/registry.py b/fairscape_models/schema/registry.py new file mode 100644 index 0000000..2ca808b --- /dev/null +++ b/fairscape_models/schema/registry.py @@ -0,0 +1,177 @@ +""" +registry.py — schema-type dispatch: file extension -> class, and schema document +-> the right typed model. + +Dispatch is a plain dict lookup keyed on the `EVI:schemaType` discriminator, not +a pydantic discriminated union — a legacy document has no discriminator, and an +alias-keyed union over `extra='allow'` models is fragile. `parse_schema` decides: + + 1. known `EVI:schemaType` tag -> that class + 2. no tag + legacy HDF5 dataset shape -> HDF5Schema (normalized) + 3. otherwise -> TabularSchema (the legacy default) + +An unknown tag falls back to the base `Schema` rather than erroring. +""" + +import json +import pathlib +import re +from typing import Dict, List, Optional, Type, Union + +from fairscape_models.schema.base import Schema, ValidationErrorRecord +from fairscape_models.schema.tabular import ( + CANONICAL_TYPES, + SOURCE_TYPE_KEY, + TabularSchema, + frictionless_type_to_json_schema, +) +from fairscape_models.schema.hdf5 import HDF5Schema +from fairscape_models.schema.signal import SignalSchema +from fairscape_models.schema.image import ImageSchema +from fairscape_models.schema.ndarray import NDArraySchema + +SCHEMA_TYPE_MODELS: Dict[str, Type[Schema]] = { + "tabular": TabularSchema, + "hdf5": HDF5Schema, + "signal": SignalSchema, + "image": ImageSchema, + "ndarray": NDArraySchema, +} + +EXTENSION_MAP: Dict[str, Type[Schema]] = { + "csv": TabularSchema, + "tsv": TabularSchema, + "parquet": TabularSchema, + "h5": HDF5Schema, + "hdf5": HDF5Schema, + "hea": SignalSchema, + "dcm": ImageSchema, +} + + +def schema_class_for_file(filepath: str) -> Type[Schema]: + ext = pathlib.Path(filepath).suffix.lower().lstrip('.') + try: + return EXTENSION_MAP[ext] + except KeyError: + raise ValueError( + f"Unsupported file extension '{ext}'. " + f"Supported extensions: {', '.join(sorted(EXTENSION_MAP))}" + ) + + +def infer_schema(filepath: str, name: str, description: str, + guid: Optional[str] = None) -> Schema: + """Infer a typed schema from a data file, dispatching by extension.""" + return schema_class_for_file(filepath).infer(filepath, name, description, guid=guid) + + +def validate_schema(schema: Schema, filepath: str) -> List[ValidationErrorRecord]: + """Validate a data file against a schema, dispatching on the schema's type.""" + return schema.validate(filepath) + + +# --------------------------------------------------------------------------- # +# Legacy normalization (tabular + legacy-HDF5 documents without a discriminator) +# --------------------------------------------------------------------------- # + +_INDEX_PATTERN = re.compile(r'^\d+$|^-?\d+::|^-?\d+::-?\d+$|^::-?\d+') + + +def _valid_index(value) -> bool: + if isinstance(value, int): + return True + if isinstance(value, str): + return bool(_INDEX_PATTERN.match(value)) + return False + + +def _is_legacy_hdf5(data: dict) -> bool: + """ + A pre-discriminator HDF5 schema is a dict of per-dataset entries, each a full + sub-schema (nested 'properties' but no valid scalar 'index'), or carrying an + 'hdf5-path'. + """ + for prop in (data.get('properties') or {}).values(): + if not isinstance(prop, dict): + continue + if 'hdf5-path' in prop: + return True + if isinstance(prop.get('properties'), dict) and not _valid_index(prop.get('index')): + return True + return False + + +def _normalize_column(name: str, prop: dict, index_fallback: int) -> dict: + """Normalize a leaf/column property to canonical Property shape (legacy types).""" + normalized = dict(prop) + prop_type = normalized.get('type') + if prop_type is None: + normalized['type'] = 'string' + elif prop_type not in CANONICAL_TYPES: + normalized.setdefault(SOURCE_TYPE_KEY, prop_type) + normalized['type'] = frictionless_type_to_json_schema(prop_type) + if not _valid_index(normalized.get('index')): + normalized['index'] = index_fallback + if not normalized.get('description'): + normalized['description'] = f"Column {name}" + return normalized + + +def _normalize_hdf5_dataset(name: str, prop: dict, index_fallback: int) -> dict: + """Rebuild a legacy per-dataset entry as a DatasetProperty-shaped dict.""" + nested = prop.get('properties') + out = { + 'type': 'object' if isinstance(nested, dict) else 'array', + 'index': index_fallback, + 'description': prop.get('description') or f"Dataset at {name}", + 'hdf5-path': prop.get('hdf5-path', name), + } + if isinstance(nested, dict): + out['properties'] = { + child: _normalize_column(child, child_prop if isinstance(child_prop, dict) else {}, i) + for i, (child, child_prop) in enumerate(nested.items()) + } + return out + + +def normalize_schema_document(data: dict) -> dict: + """Normalize a legacy tabular schema document so it validates canonically.""" + data = dict(data) + data.setdefault('@type', 'evi:Schema') + data.setdefault('separator', ',') + data.setdefault('header', True) + data['properties'] = { + name: _normalize_column(name, prop, i) if isinstance(prop, dict) else prop + for i, (name, prop) in enumerate((data.get('properties') or {}).items()) + } + return data + + +def _normalize_legacy_hdf5_document(data: dict) -> dict: + data = dict(data) + data['properties'] = { + name: _normalize_hdf5_dataset(name, prop, i) if isinstance(prop, dict) else prop + for i, (name, prop) in enumerate((data.get('properties') or {}).items()) + } + return data + + +def parse_schema(data: dict) -> Schema: + """Parse a schema JSON document into the correct typed model.""" + tag = data.get("EVI:schemaType") + if tag in SCHEMA_TYPE_MODELS: + return SCHEMA_TYPE_MODELS[tag].model_validate(data) + if tag is not None: + # Unknown discriminator — keep it as an opaque base Schema, never error. + return Schema.model_validate(data) + if _is_legacy_hdf5(data): + return HDF5Schema.model_validate(_normalize_legacy_hdf5_document(data)) + return TabularSchema.model_validate(normalize_schema_document(data)) + + +def load_schema(path: Union[str, pathlib.Path]) -> Schema: + """Read a schema JSON file into the correct typed model.""" + with open(path) as f: + data = json.load(f) + return parse_schema(data) diff --git a/fairscape_models/schema/signal.py b/fairscape_models/schema/signal.py new file mode 100644 index 0000000..e17c522 --- /dev/null +++ b/fairscape_models/schema/signal.py @@ -0,0 +1,160 @@ +""" +signal.py — the `signal` schema type: WFDB / EDF+ waveforms & biosignals. + +`properties` are channels (`ChannelProperty`). Inference reads a WFDB `.hea` +header (signal-specification lines: name / units / gain / baseline / ADC +resolution) via the `wfdb` library. Nothing is invented — units link to +QUDT/UCUM only where verified, and the record-level LOINC panel code is attached +only when the channel set is exactly the 12 standard ECG leads. +""" + +import os +import re +from typing import Dict, List, Literal, Optional + +from pydantic import Field + +from fairscape_models.schema.base import ( + NonTabularSchema, + Property, + ValidationErrorRecord, + compare_scalar, + generate_schema_guid, +) +from fairscape_models.schema.ontology import ( + LOINC_12LEAD, + STD_12_LEADS, + as_num, + unit_terms, +) + + +class ChannelProperty(Property): + """ + One channel of a `signal` schema (a WFDB signal-spec line or an EDF+ signal). + + A channel's samples are continuous so `type` is "number". Carries both + WFDB-style calibration (gain / baseline / adcResolution) and EDF-style + calibration (physical/digital min-max) — all optional, since a given record + uses one convention or the other. + """ + unit: Optional[str] = Field(default=None, description="UCUM unit code, e.g. 'mV', 'uV'") + unitURL: Optional[str] = Field(default=None, description="Resolvable unit URI (QUDT), where a verified term exists") + + # WFDB calibration (GAIN(BASELINE)/UNITS + ADC resolution) + gain: Optional[float] = Field(default=None, description="WFDB ADC gain (ADC units per physical unit)") + baseline: Optional[int] = Field(default=None, description="WFDB ADC value corresponding to 0 physical units") + adcResolution: Optional[int] = Field(default=None, description="WFDB ADC resolution in bits") + + # EDF+ calibration (physical<->digital linear map) + signal provenance + physicalMinimum: Optional[float] = Field(default=None) + physicalMaximum: Optional[float] = Field(default=None) + digitalMinimum: Optional[int] = Field(default=None) + digitalMaximum: Optional[int] = Field(default=None) + transducer: Optional[str] = Field(default=None, description="EDF transducer type, e.g. 'AgAgCl cup electrode'") + prefiltering: Optional[str] = Field(default=None, description="EDF prefiltering, e.g. 'HP:0.1Hz LP:75Hz N:60Hz'") + + +class SignalSchema(NonTabularSchema): + """`signal` — WFDB / EDF+ waveforms & biosignals. `properties` are channels.""" + + evi_schema_type: Literal["signal"] = Field(default="signal", alias="EVI:schemaType") + properties: Dict[str, ChannelProperty] + + samplingFrequency: Optional[float] = Field(default=None, description="Record sampling frequency") + samplingFrequencyUnit: Optional[str] = Field(default="Hz") + samplingFrequencyUnitURL: Optional[str] = Field(default=None, description="QUDT hertz is upper-case 'HZ'") + numberOfSamples: Optional[int] = Field(default=None, description="Samples per channel") + duration: Optional[float] = Field(default=None) + durationUnit: Optional[str] = Field(default=None) + durationUnitURL: Optional[str] = Field(default=None) + startTime: Optional[str] = Field(default=None, description="Record base time/date (ISO-8601 where known)") + + @classmethod + def infer(cls, filepath: str, name: str, description: str, + guid: Optional[str] = None) -> "SignalSchema": + import wfdb + + record_dir = os.path.dirname(filepath) or "." + record_name = os.path.splitext(os.path.basename(filepath))[0] + rec = wfdb.rdheader(os.path.join(record_dir, record_name)) + + props: Dict[str, ChannelProperty] = {} + for i in range(rec.n_sig): + # Header description fields can carry trailing punctuation (BIDMC: 'RESP,'). + channel = re.sub(r"[^\w+-]+$", "", (rec.sig_name[i] or f"sig_{i}").strip()) + ucum, qudt = unit_terms(rec.units[i] if rec.units else None) + props[channel] = ChannelProperty( + description=f"Signal channel '{channel}'", + index=i, + type="number", + unit=ucum, + unitURL=qudt, + gain=as_num(rec.adc_gain[i]) if rec.adc_gain else None, + baseline=rec.baseline[i] if rec.baseline else None, + adcResolution=rec.adc_res[i] if rec.adc_res else None, + ) + + fs = as_num(rec.fs) + n = rec.sig_len + duration = round(n / fs, 6) if (fs and n) else None + names_uc = {k.upper() for k in props} + record_value_url = LOINC_12LEAD if STD_12_LEADS <= names_uc else None + + start = None + if getattr(rec, "base_time", None): + start = str(rec.base_date) + "T" + str(rec.base_time) if getattr(rec, "base_date", None) else str(rec.base_time) + + return cls.model_validate({ + "@id": guid or generate_schema_guid(name), + "name": name, + "description": description, + "conformsTo": {"@id": "https://physionet.org/physiotools/wag/header-5.htm"}, + "samplingFrequency": fs, + "samplingFrequencyUnit": "Hz", + "samplingFrequencyUnitURL": "http://qudt.org/vocab/unit/HZ", + "numberOfSamples": n, + "duration": duration, + "durationUnit": "s", + "durationUnitURL": "http://qudt.org/vocab/unit/SEC", + "unitSystem": "http://unitsofmeasure.org", + "startTime": start, + "valueURL": record_value_url, + "properties": props, + "required": list(props), + }) + + def validate(self, filepath: str) -> List[ValidationErrorRecord]: + """ + Structural validation: re-infer from the file header and compare the + fields this schema actually declares (a None on either side means + "unconstrained"). Reports missing channels and calibration/rate drift. + """ + observed = type(self).infer(filepath, name=self.name, description="revalidation probe") + errors: List[ValidationErrorRecord] = [] + + # Record-level scalar comparisons. + for attr in ("samplingFrequency", "numberOfSamples"): + compare_scalar(errors, getattr(self, attr), getattr(observed, attr), + path=None, field=attr) + + for channel, declared in self.properties.items(): + found = observed.properties.get(channel) + if found is None: + errors.append(ValidationErrorRecord( + message=f"Declared channel '{channel}' is absent from the file", + failed_keyword="required", field=channel, path=channel, + )) + continue + for attr in ("unit", "gain", "baseline", "adcResolution"): + compare_scalar(errors, getattr(declared, attr), getattr(found, attr), + path=channel, field=f"{channel}.{attr}") + + if self.additionalProperties is False: + for channel in observed.properties: + if channel not in self.properties: + errors.append(ValidationErrorRecord( + message=f"File has undeclared channel '{channel}'", + failed_keyword="additionalProperties", field=channel, path=channel, + )) + return errors diff --git a/fairscape_models/schema/tabular.py b/fairscape_models/schema/tabular.py new file mode 100644 index 0000000..4053ccd --- /dev/null +++ b/fairscape_models/schema/tabular.py @@ -0,0 +1,224 @@ +""" +tabular.py — the `tabular` schema type: csv / tsv / parquet. + +Inference and row-level validation both run through frictionless for all three +formats (frictionless's parquet parser reads via pyarrow + pandas). Field types +are stored canonically (the six JSON-Schema types); when frictionless infers a +richer type (datetime, year, ...) the original is kept under 'source-type' so +validation can rebuild the native frictionless field. +""" + +import pathlib +from typing import Dict, List, Literal, Optional + +from pydantic import Field + +from fairscape_models.schema.base import ( + Property, + Schema, + ValidationErrorRecord, + frictionless_error_to_record, + generate_schema_guid, +) + +CANONICAL_TYPES = {'integer', 'number', 'string', 'array', 'boolean', 'object'} +SOURCE_TYPE_KEY = 'source-type' + +_CSV_EXTENSIONS = {'csv', 'tsv'} + + +def frictionless_type_to_json_schema(field_type: str) -> str: + """Convert Frictionless types to JSON Schema types""" + type_mapping = { + 'string': 'string', + 'integer': 'integer', + 'number': 'number', + 'boolean': 'boolean', + 'date': 'string', + 'datetime': 'string', + 'year': 'integer', + 'yearmonth': 'string', + 'duration': 'string', + 'geopoint': 'array', + 'geojson': 'object', + 'array': 'array', + 'object': 'object', + 'time': 'string' + } + return type_mapping.get(field_type, 'string') + + +def _get_either(prop_details: dict, *keys): + for key in keys: + if prop_details.get(key) is not None: + return prop_details[key] + return None + + +def build_frictionless_schema(properties: Dict[str, Property]): + """Rebuild a frictionless Schema from canonical Properties for row validation.""" + from frictionless import Schema as FrictionlessSchema, fields + + type_to_field = { + 'string': fields.StringField, + 'integer': fields.IntegerField, + 'number': fields.NumberField, + 'boolean': fields.BooleanField, + } + # A 'source-type' stamped at infer time wins over the canonical type, so + # e.g. a datetime column validates as datetime cells (parquet yields real + # datetime objects, which would fail a plain StringField). + source_type_to_field = { + 'date': fields.DateField, + 'datetime': fields.DatetimeField, + 'time': fields.TimeField, + 'year': fields.YearField, + 'yearmonth': fields.YearmonthField, + 'duration': fields.DurationField, + } + + properties_input = { + name: prop.model_dump(by_alias=True, exclude_none=True) + for name, prop in properties.items() + } + + frictionless_schema_obj = FrictionlessSchema() + + sorted_prop_items = [] + spanning_array_prop_name = None + spanning_array_prop_details = None + + for name, prop_details in properties_input.items(): + index_val = prop_details.get("index") + if prop_details.get("type") == "array" and isinstance(index_val, str) and "::" in index_val: + if spanning_array_prop_name is not None: + raise ValueError("Multiple spanning array properties (index: 'X::') are not supported.") + spanning_array_prop_name = name + spanning_array_prop_details = prop_details + elif isinstance(index_val, int): + sorted_prop_items.append((name, prop_details, index_val)) + else: + sorted_prop_items.append((name, prop_details, float('inf'))) + + sorted_prop_items.sort(key=lambda x: x[2]) + + for name, prop_details, _ in sorted_prop_items: + field_class = source_type_to_field.get(prop_details.get(SOURCE_TYPE_KEY)) + if field_class is None: + field_class = type_to_field.get(prop_details.get('type', 'string'), fields.StringField) + + constraints = {} + if 'minimum' in prop_details: constraints['minimum'] = prop_details['minimum'] + if 'maximum' in prop_details: constraints['maximum'] = prop_details['maximum'] + if 'pattern' in prop_details: constraints['pattern'] = prop_details['pattern'] + if 'minLength' in prop_details: constraints['minLength'] = prop_details['minLength'] + if 'maxLength' in prop_details: constraints['maxLength'] = prop_details['maxLength'] + + if prop_details.get('type') == 'array': + field = fields.ArrayField(name=name, description=prop_details.get('description', ''), constraints=constraints) + else: + field = field_class(name=name, description=prop_details.get('description', ''), constraints=constraints) + frictionless_schema_obj.add_field(field) + + if spanning_array_prop_name and spanning_array_prop_details: + prop_name_original = spanning_array_prop_name + details = spanning_array_prop_details + item_details = details.get('items', {}) + item_type = item_details.get('type', 'number') + item_field_class = type_to_field.get(item_type, fields.NumberField) + + # min/max item counts appear as 'min-items'/'max-items' (canonical alias) + # or 'minItems'/'maxItems' (CLI property models) depending on origin + num_items = _get_either(details, 'min-items', 'minItems') + max_items = _get_either(details, 'max-items', 'maxItems') + if num_items is None or num_items != max_items: + raise ValueError(f"Spanning array '{prop_name_original}' must have equal and defined minItems and maxItems.") + + for i in range(num_items): + field_name_for_frictionless = f"{prop_name_original}_{i}" # e.g., embed_0, embed_1, ... + field = item_field_class(name=field_name_for_frictionless, description=f"Element {i} of {prop_name_original}") + frictionless_schema_obj.add_field(field) + + return frictionless_schema_obj + + +class TabularSchema(Schema): + """`tabular` — csv / tsv / parquet. `properties` are columns.""" + + # Match the rest of the EVI:Schema family (and the CLI's historical output). + metadataType: str = Field(default="EVI:Schema", alias="@type") + evi_schema_type: Literal["tabular"] = Field(default="tabular", alias="EVI:schemaType") + + @classmethod + def infer(cls, filepath: str, name: str, description: str, + guid: Optional[str] = None) -> "TabularSchema": + from frictionless import describe + + ext = pathlib.Path(filepath).suffix.lower().lstrip('.') + + resource = describe(filepath) + + properties = {} + required_fields = [] + + for i, field in enumerate(resource.schema.fields): + json_schema_type = frictionless_type_to_json_schema(field.type) + extra = {} + if json_schema_type != field.type: + extra[SOURCE_TYPE_KEY] = field.type + + properties[field.name] = Property( + type=json_schema_type, + description=field.description or f"Column {field.name}", + index=i, + **extra, + ) + required_fields.append(field.name) + + document = { + "@id": guid or generate_schema_guid(name), + "name": name, + "description": description, + "properties": properties, + "required": required_fields, + } + if ext in _CSV_EXTENSIONS: + document["separator"] = '\t' if ext == 'tsv' else ',' + document["header"] = True + else: + # separator/header are csv concepts; prune them for parquet + document["separator"] = None + document["header"] = None + return cls.model_validate(document) + + def validate(self, filepath: str) -> List[ValidationErrorRecord]: + from frictionless import Resource, Dialect, formats + + frictionless_schema = build_frictionless_schema(self.properties) + + # frictionless rejects absolute paths as "not safe"; anchor on the file's + # directory as basepath and reference it by name to allow either form. + path = pathlib.Path(filepath) + name = path.name + basepath = str(path.parent) + ext = path.suffix.lower().lstrip('.') + + if ext in _CSV_EXTENSIONS: + # Delimiter and header row are csv dialect settings; parquet has neither. + file_dialect = Dialect() + file_dialect.header = self.header if self.header is not None else True + csv_control = formats.csv.CsvControl(delimiter=self.separator or ',') + file_dialect.add_control(csv_control) + resource = Resource(path=name, basepath=basepath, + schema=frictionless_schema, dialect=file_dialect) + else: + resource = Resource(path=name, basepath=basepath, schema=frictionless_schema) + + report = resource.validate() + errors_list = [] + if not report.valid: + for task in report.tasks: + for error_detail in task.errors: + errors_list.append(frictionless_error_to_record(error_detail)) + + return errors_list diff --git a/pyproject.toml b/pyproject.toml index 39f54fa..bc18c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,29 +4,53 @@ build-backend = "setuptools.build_meta" [project] name = "fairscape-models" -version = "1.1.7" +version = "1.2.2" description = "Fairscape pydantic models" readme = "README.md" authors = [ {name = "Justin Niestroy", email = "jniestroy@gmail.com"} ] -license = {text = "MIT"} +license = {file = "LICENSE"} classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ] dependencies = [ - "pydantic", + "pydantic>=2.5", "pymongo", "typing", "mongomock", "pyyaml>=6.0.3", + # importlib.resources.files() is stdlib on 3.9+; backport it for 3.8 + "importlib_resources; python_version < '3.9'", ] requires-python = ">=3.8" +[project.optional-dependencies] +# Per-format schema inference/validation deps, imported lazily by the +# fairscape_models.schema.* infer/validate methods. The base package needs none. +schema-tabular = ["frictionless>=5.0,<6.0", "pyarrow>=14", "pandas"] +schema-hdf5 = ["h5py"] +schema-signal = ["wfdb"] +schema-image = ["pydicom"] +schema-infer = [ + "frictionless>=5.0,<6.0", + "pyarrow>=14", + "pandas", + "h5py", + "wfdb", + "pydicom", +] + [project.urls] -"Homepage" = "https://github.com/fairscape/fairscape-models" +"Homepage" = "https://github.com/fairscape/fairscape_models" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.package-data] +"fairscape_models.schema" = ["reference_tables/*.csv"] [tool.coverage.run] omit = [ diff --git a/tests/test_schema.py b/tests/test_schema.py index bf1e096..39aefd0 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -17,6 +17,18 @@ def test_property_instantiation(property_data): assert prop.index == 1 assert prop.value_url == "http://example.com/value" + +def test_property_value_url_alias_choices(): + # All three spellings parse; serialization always emits camelCase 'valueURL'. + for key in ("valueURL", "value-url", "value_url"): + prop = Property.model_validate({ + "description": "p", "index": 0, "type": "string", key: "http://x/v", + }) + assert prop.value_url == "http://x/v" + dumped = prop.model_dump(by_alias=True, exclude_none=True) + assert dumped["valueURL"] == "http://x/v" + assert "value-url" not in dumped + def test_property_invalid_type(property_data): property_data["type"] = "invalid_type" with pytest.raises(ValueError, match="Type must be one of"): diff --git a/tests/test_schema_infer_validate.py b/tests/test_schema_infer_validate.py new file mode 100644 index 0000000..7778878 --- /dev/null +++ b/tests/test_schema_infer_validate.py @@ -0,0 +1,261 @@ +""" +End-to-end infer/validate tests per schema type. All fixtures are generated or +hand-written in tmp_path — no binary fixtures are read from disk. Each test group +is skipped if its optional dependency is missing. +""" + +import numpy as np +import pytest + +from fairscape_models.schema import ( + HDF5Schema, + ImageSchema, + SignalSchema, + TabularSchema, + infer_schema, + validate_schema, +) +from fairscape_models.schema.ontology import anatomy_terms, unit_terms + + +# --------------------------------------------------------------------------- # +# ontology (stdlib only, always runs) +# --------------------------------------------------------------------------- # + +def test_unit_terms_known_and_unknown(): + ucum, qudt = unit_terms("mV") + assert ucum == "mV" + assert qudt == "http://qudt.org/vocab/unit/MilliV" + assert unit_terms("notaunit") == ("notaunit", None) + assert unit_terms(None) == (None, None) + + +def test_anatomy_terms_crosswalk(): + label, url = anatomy_terms("CHEST") + assert url and url.startswith("http://snomed.info/id/") + assert anatomy_terms("madeuppart") == ("madeuppart", None) + + +# --------------------------------------------------------------------------- # +# tabular (frictionless) +# --------------------------------------------------------------------------- # + +frictionless = pytest.importorskip("frictionless") + + +def test_tabular_csv_infer_validate(tmp_path): + p = tmp_path / "t.csv" + p.write_text("id,score,label\n1,0.5,a\n2,1.5,b\n") + schema = TabularSchema.infer(str(p), name="csv", description="d") + assert schema.properties["id"].type == "integer" + assert schema.properties["score"].type == "number" + assert schema.properties["label"].type == "string" + assert schema.separator == "," + assert validate_schema(schema, str(p)) == [] + + +def test_tabular_tsv_separator(tmp_path): + p = tmp_path / "t.tsv" + p.write_text("id\tscore\n1\t0.5\n") + schema = TabularSchema.infer(str(p), name="tsv", description="d") + assert schema.separator == "\t" + assert validate_schema(schema, str(p)) == [] + + +def test_tabular_type_mismatch_reports_error(tmp_path): + p = tmp_path / "t.csv" + p.write_text("id,label\n1,a\n2,b\n") + schema = TabularSchema.infer(str(p), name="csv", description="d") + schema.properties["label"].type = "integer" + errors = validate_schema(schema, str(p)) + assert errors + assert any(e.field == "label" for e in errors) + + +def test_tabular_parquet_infer_validate(tmp_path): + pytest.importorskip("pyarrow") + import pandas as pd + import pyarrow as pa + import pyarrow.parquet as pq + + p = tmp_path / "t.parquet" + df = pd.DataFrame({ + "id": [1, 2], + "score": [0.5, 1.5], + "when": pd.to_datetime(["2024-01-01", "2024-02-01"]), + }) + pq.write_table(pa.Table.from_pandas(df), str(p)) + schema = TabularSchema.infer(str(p), name="pq", description="d") + assert schema.properties["id"].type == "integer" + # datetime column keeps a source-type so validation rebuilds a DatetimeField + assert schema.properties["when"].model_extra.get("source-type") == "datetime" + assert schema.separator is None + assert validate_schema(schema, str(p)) == [] + + +def test_tabular_spanning_array(tmp_path): + p = tmp_path / "e.csv" + p.write_text("embed_0,embed_1,embed_2\n0.1,0.2,0.3\n0.4,0.5,0.6\n") + schema = TabularSchema.model_validate({ + "@id": "ark:59853/schema-span", + "name": "span", + "description": "d", + "properties": { + "embed": { + "description": "vector", "index": "0::", "type": "array", + "min-items": 3, "max-items": 3, + "items": {"type": "number"}, + } + }, + }) + assert validate_schema(schema, str(p)) == [] + + +# --------------------------------------------------------------------------- # +# hdf5 (h5py) +# --------------------------------------------------------------------------- # + +h5py = pytest.importorskip("h5py") + + +@pytest.fixture +def h5_file(tmp_path): + p = tmp_path / "d.h5" + with h5py.File(p, "w") as f: + f.create_dataset("a", data=np.zeros((3,), dtype="f8")) + grp = f.create_group("grp") + grp.create_dataset("b", data=np.zeros((2, 4), dtype="i4"), + chunks=(1, 4), compression="gzip") + comp = np.zeros(2, dtype=[("id", "i8"), ("score", "f4")]) + f.create_dataset("c", data=comp) + return str(p) + + +def test_hdf5_infer_structure(h5_file): + schema = HDF5Schema.infer(h5_file, name="h5", description="d") + assert schema.properties["a"].shape == [3] + assert schema.properties["a"].dtype == "float64" + b = schema.properties["grp/b"] + assert b.shape == [2, 4] + assert b.chunks == [1, 4] + assert b.compression == "gzip" + c = schema.properties["c"] + assert c.type == "object" + assert set(c.properties) == {"id", "score"} + + +def test_hdf5_validate_clean(h5_file): + schema = HDF5Schema.infer(h5_file, name="h5", description="d") + assert validate_schema(schema, h5_file) == [] + + +def test_hdf5_validate_drift(h5_file): + schema = HDF5Schema.infer(h5_file, name="h5", description="d") + schema.properties["a"].shape = [9] + schema.properties["grp/b"].dtype = "int64" + errors = validate_schema(schema, h5_file) + keywords = {(e.failed_keyword, e.field) for e in errors} + assert ("const", "a.shape") in keywords + assert ("type", "grp/b.dtype") in keywords + + +def test_hdf5_validate_missing_dataset(h5_file): + schema = HDF5Schema.infer(h5_file, name="h5", description="d") + from fairscape_models.schema import DatasetProperty + schema.properties["ghost"] = DatasetProperty( + description="x", index=99, type="array", shape=[1]) + errors = validate_schema(schema, h5_file) + assert any(e.failed_keyword == "required" and e.field == "ghost" for e in errors) + + +# --------------------------------------------------------------------------- # +# signal (wfdb) +# --------------------------------------------------------------------------- # + +wfdb = pytest.importorskip("wfdb") + + +@pytest.fixture +def wfdb_record(tmp_path): + rec = "sig" + header = ( + f"{rec} 2 250 100\n" + f"{rec}.dat 16 200(0)/mV 16 0 0 0 0 MLII\n" + f"{rec}.dat 16 200(0)/mV 16 0 0 0 0 V5\n" + ) + (tmp_path / f"{rec}.hea").write_text(header) + np.zeros((100, 2), dtype=" NonTabularSchema default (EVI:Schema) applies. + doc = { + "@id": "ark:59853/schema-x", + "name": "x", + "description": "d", + "EVI:schemaType": "signal", + "properties": {"MLII": {"description": "ch", "index": 0, "type": "number"}}, + } + dumped = parse_schema(doc).model_dump(by_alias=True, exclude_none=True) + assert dumped["EVI:schemaType"] == "signal" + assert dumped["@type"] == "EVI:Schema" + # tabular-only fields are neutralized to None and drop out + for key in ("separator", "header", "type", "additionalProperties"): + assert key not in dumped