Full ai ready

fairscape_models/conversion/mapping/FairscapeDatasheet.py

@@ -123,10 +123,10 @@ def _extract_id(value: Any) -> Optional[str]:
 
 
 USECASES_MAPPING = {
-    "intended_use":          {"source_key": "rai:dataUseCases"},
-    "limitations":           {"source_key": "rai:dataLimitations"},
+    "intended_use":          {"source_key": "rai:dataUseCases", "parser": _list_to_str},
+    "limitations":           {"source_key": "rai:dataLimitations", "parser": _list_to_str},
     "prohibited_uses":        {"source_key": "prohibitedUses", "fallback_source_key": "additionalProperty", "fallback_parser": from_additional_property("Prohibited Uses")},
-    "potential_sources_of_bias": {"source_key": "rai:dataBiases"},
+    "potential_sources_of_bias": {"source_key": "rai:dataBiases", "parser": _list_to_str},
     "maintenance_plan":      {"source_key": "rai:dataReleaseMaintenancePlan"},
 
     # Additional RAI fields

fairscape_models/conversion/models/FairscapeDatasheet.py

@@ -57,27 +57,27 @@ class OverviewSection(BaseModel):
 class UseCasesSection(BaseModel):
     """Datasheet section for describing dataset use cases, limitations, and biases."""
 
-    intended_use: Optional[str] = Field(
+    intended_use: Optional[Union[str, List[str]]] = Field(
         default=None, description="Recommended dataset uses (e.g., training, validation)"
     )
-    limitations: Optional[str] = Field(
+    limitations: Optional[Union[str, List[str]]] = Field(
         default=None, description="Known limitations and non-recommended uses"
     )
     prohibited_uses: Optional[str] = Field(
         default=None, description="Explicitly prohibited uses (subset of limitations)"
     )
-    potential_sources_of_bias: Optional[str] = Field(
+    potential_sources_of_bias: Optional[Union[str, List[str]]] = Field(
         default=None, description="Description of known biases in the dataset"
     )
-    maintenance_plan: Optional[str] = Field(
+    maintenance_plan: Optional[Union[str, List[str]]] = Field(
         default=None, description="Versioning, maintainers, and deprecation policies"
     )
 
     # Additional RAI fields
     data_collection: Optional[str] = Field(
         default=None, description="Description of data collection methodology"
     )
-    data_collection_type: Optional[str] = Field(
+    data_collection_type: Optional[Union[str, List[str]]] = Field(
         default=None, description="Type of data collection"
     )
     data_collection_missing_data: Optional[str] = Field(
@@ -86,28 +86,28 @@ class UseCasesSection(BaseModel):
     data_collection_raw_data: Optional[str] = Field(
         default=None, description="Description of raw data from collection"
     )
-    data_collection_timeframe: Optional[str] = Field(
+    data_collection_timeframe: Optional[Union[str, List[str]]] = Field(
         default=None, description="Timeframe of data collection"
     )
     data_imputation_protocol: Optional[str] = Field(
         default=None, description="Protocol used for data imputation"
     )
-    data_manipulation_protocol: Optional[str] = Field(
+    data_manipulation_protocol: Optional[Union[str, List[str]]] = Field(
         default=None, description="Protocol used for data manipulation"
     )
-    data_preprocessing_protocol: Optional[str] = Field(
+    data_preprocessing_protocol: Optional[Union[str, List[str]]] = Field(
         default=None, description="Protocol used for data preprocessing"
     )
     data_annotation_protocol: Optional[str] = Field(
         default=None, description="Protocol used for data annotation"
     )
-    data_annotation_platform: Optional[str] = Field(
+    data_annotation_platform: Optional[Union[str, List[str]]] = Field(
         default=None, description="Platform used for data annotation"
     )
-    data_annotation_analysis: Optional[str] = Field(
+    data_annotation_analysis: Optional[Union[str, List[str]]] = Field(
         default=None, description="Analysis of data annotations"
     )
-    personal_sensitive_information: Optional[str] = Field(
+    personal_sensitive_information: Optional[Union[str, List[str]]] = Field(
         default=None, description="Description of personal/sensitive information"
     )
     data_social_impact: Optional[str] = Field(
@@ -116,10 +116,10 @@ class UseCasesSection(BaseModel):
     annotations_per_item: Optional[str] = Field(
         default=None, description="Number of annotations per item"
     )
-    annotator_demographics: Optional[str] = Field(
+    annotator_demographics: Optional[Union[str, List[str]]] = Field(
         default=None, description="Demographics of annotators"
     )
-    machine_annotation_tools: Optional[str] = Field(
+    machine_annotation_tools: Optional[Union[str, List[str]]] = Field(
         default=None, description="Machine tools used for annotation"
     )
 

fairscape_models/dataset.py

@@ -1,11 +1,37 @@
+import csv
+from pathlib import Path
 from pydantic import BaseModel, Field, ConfigDict, AliasChoices, model_validator
-from typing import Optional, List, Union
+from typing import Optional, List, Union, Tuple
 from enum import Enum
 
 from fairscape_models.fairscape_base import IdentifierValue, DATASET_TYPE
 from fairscape_models.digital_object import DigitalObject
 
 
+TABULAR_FORMATS = {"csv", "tsv", "text/csv", "text/tab-separated-values"}
+TABULAR_EXTENSIONS = {".csv", ".tsv"}
+
+
+def _count_csv(path: Path, delimiter: str) -> Tuple[int, int]:
+    """Stream a csv/tsv counting data rows and columns (header excluded from rowCount)."""
+    with path.open("r", newline="", encoding="utf-8", errors="replace") as f:
+        reader = csv.reader(f, delimiter=delimiter)
+        try:
+            header = next(reader)
+        except StopIteration:
+            return 0, 0
+        cols = len(header)
+        rows = sum(1 for _ in reader)
+    return rows, cols
+
+
+def _human_size(n: int) -> str:
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1024 or unit == "TB":
+            return f"{n:.1f} {unit}" if unit != "B" else f"{n} {unit}"
+        n /= 1024
+
+
 class SplitType(str, Enum):
     """Croissant-aligned split type semantics.
 
@@ -61,6 +87,110 @@ class Dataset(DigitalObject):
     derivedFrom: Optional[List[IdentifierValue]] = Field(default=[])
     splits: Optional[List[Split]] = Field(default=None)
 
+    # statistics support AI-Ready rubric 2.b Statistics).
+    contentSize: Optional[str] = Field(default=None, description="Total size of the dataset content (e.g. '2.4 GB', '150 MB').")
+    rowCount: Optional[int] = Field(default=None, description="Number of rows / records for tabular datasets.")
+    columnCount: Optional[int] = Field(default=None, description="Number of columns / fields for tabular datasets.")
+    sampleSize: Optional[int] = Field(default=None, description="Number of samples represented by the dataset (often == rowCount for tabular data, but may differ).")
+    hasSummaryStatistics: Optional[Union[str, IdentifierValue]] = Field(default=None, description="Reference to a summary statistics entity describing distributions, counts, and key statistics for this dataset.")
+
+    def add_summary_stats(
+        self,
+        file_path: Optional[Union[str, Path]] = None,
+        crate_root: Optional[Union[str, Path]] = None,
+    ) -> "Dataset":
+        """Compute row/column counts for a tabular Dataset and produce a linked stats child.
+
+        Reads ``self.contentUrl`` (or an explicit ``file_path`` override) as csv/tsv
+        using the stdlib ``csv`` module — no pandas / no extra deps. Populates
+        ``rowCount``, ``columnCount``, ``contentSize``, ``sampleSize`` on ``self``
+        and on a newly-constructed child ``Dataset``. Sets ``self.hasSummaryStatistics``
+        to the child's ``@id`` and returns the child so the caller can append it
+        to the RO-Crate ``@graph``.
+
+        Remote URLs and non-csv/tsv formats are out of scope here — use the
+        ``fairscape augment summary-stats`` CLI command for parquet / http(s) /
+        per-column statistics.
+        """
+        resolved = self._resolve_tabular_path(file_path, crate_root)
+        delimiter = "\t" if resolved.suffix.lower() == ".tsv" or "tab" in (self.fileFormat or "").lower() else ","
+        rows, cols = _count_csv(resolved, delimiter)
+        size_bytes = resolved.stat().st_size
+        size_str = _human_size(size_bytes)
+
+        self.rowCount = rows
+        self.columnCount = cols
+        self.contentSize = size_str
+        if self.sampleSize is None:
+            self.sampleSize = rows
+
+        stats_guid = f"{self.guid.rstrip('/')}/summary-stats"
+        stats = Dataset(
+            guid=stats_guid,
+            name=f"{self.name} — Summary Statistics",
+            author=self.author,
+            description=f"Row and column counts for {self.name} ({self.guid}), generated from the source tabular file.",
+            datePublished=self.datePublished,
+            keywords=(self.keywords or []) + ["summary-statistics"],
+            fileFormat="application/json",
+            rowCount=rows,
+            columnCount=cols,
+            contentSize=size_str,
+            sampleSize=rows,
+            derivedFrom=[IdentifierValue(**{"@id": self.guid})],
+        )
+        self.hasSummaryStatistics = IdentifierValue(**{"@id": stats_guid})
+        return stats
+
+    def _resolve_tabular_path(
+        self,
+        file_path: Optional[Union[str, Path]],
+        crate_root: Optional[Union[str, Path]],
+    ) -> Path:
+        if file_path is not None:
+            p = Path(file_path)
+            if not p.exists():
+                raise FileNotFoundError(f"file_path does not exist: {p}")
+            self._require_tabular(p)
+            return p
+
+        if not self.contentUrl:
+            raise ValueError(f"Dataset {self.guid} has no contentUrl; pass file_path explicitly.")
+
+        url = self.contentUrl if isinstance(self.contentUrl, str) else self.contentUrl[0]
+        if url.startswith(("http://", "https://")):
+            raise NotImplementedError(
+                "Remote contentUrl is not supported by add_summary_stats; use the "
+                "`fairscape augment summary-stats` CLI command, or pass file_path."
+            )
+
+        if url.startswith("file://"):
+            rel = url[len("file://"):].lstrip("/")
+            if crate_root is None:
+                raise ValueError(
+                    f"contentUrl {url!r} is crate-relative; pass crate_root to resolve."
+                )
+            p = Path(crate_root) / rel
+        else:
+            p = Path(url)
+            if not p.is_absolute() and crate_root is not None:
+                p = Path(crate_root) / p
+
+        if not p.exists():
+            raise FileNotFoundError(f"Resolved contentUrl does not exist: {p}")
+        self._require_tabular(p)
+        return p
+
+    def _require_tabular(self, path: Path) -> None:
+        fmt = (self.fileFormat or "").lower()
+        ext = path.suffix.lower()
+        if fmt in TABULAR_FORMATS or ext in TABULAR_EXTENSIONS:
+            return
+        raise ValueError(
+            f"Dataset {self.guid} is not csv/tsv (fileFormat={self.fileFormat!r}, "
+            f"extension={ext!r}); use the CLI augment summary-stats command for richer formats."
+        )
+
     @model_validator(mode='after')
     def populate_prov_fields(self):
         """Auto-populate PROV-O fields from EVI fields"""

fairscape_models/defined_term.py

@@ -0,0 +1,24 @@
+from pydantic import BaseModel, Field, ConfigDict
+from typing import Any, Dict, Optional, Union
+
+from fairscape_models.fairscape_base import IdentifierValue
+
+
+class DefinedTerm(BaseModel):
+    """Schema.org DefinedTerm. `@id` is the ontology IRI when one exists
+    (MeSH, EDAM, Cellosaurus, etc.); `identifier` is set to the same IRI when
+    `@id` is an ARK fallback so the external identifier is still discoverable.
+    """
+    guid: Optional[str] = Field(default=None, alias="@id")
+    metadataType: str = Field(default="DefinedTerm", alias="@type")
+    name: str
+    termCode: Optional[str] = Field(default=None)
+    inDefinedTermSet: Optional[Union[str, IdentifierValue, Dict[str, Any]]] = Field(
+        default=None,
+        description="The ontology / scheme this term belongs to. Either a reference stub ({\"@id\": \"...\"}) or an inline scheme dict.",
+    )
+    identifier: Optional[str] = Field(
+        default=None,
+        description="External IRI for this term (e.g. MeSH / EDAM / Cellosaurus URI). Set when @id is an ARK fallback.",
+    )
+    model_config = ConfigDict(extra="allow", populate_by_name=True)

fairscape_models/person.py

@@ -0,0 +1,34 @@
+from pydantic import BaseModel, Field, ConfigDict
+from typing import Optional, Union
+
+from fairscape_models.fairscape_base import IdentifierValue
+
+
+class Organization(BaseModel):
+    """Schema.org Organization. `identifier` is the ROR URI when available."""
+    guid: Optional[str] = Field(default=None, alias="@id")
+    metadataType: str = Field(default="Organization", alias="@type")
+    name: str
+    identifier: Optional[str] = Field(
+        default=None,
+        description="Persistent identifier for the organization, typically a ROR URI (https://ror.org/...)."
+    )
+    url: Optional[str] = Field(default=None)
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+
+class Person(BaseModel):
+    """Schema.org Person. `identifier` is the ORCID URI when available."""
+    guid: Optional[str] = Field(default=None, alias="@id")
+    metadataType: str = Field(default="Person", alias="@type")
+    name: str
+    identifier: Optional[str] = Field(
+        default=None,
+        description="Persistent identifier for the person, typically an ORCID URI (https://orcid.org/...)."
+    )
+    email: Optional[str] = Field(default=None)
+    affiliation: Optional[Union[str, IdentifierValue, Organization]] = Field(
+        default=None,
+        description="Affiliation as a plain string, a reference stub to an Organization in @graph, or an inline Organization.",
+    )
+    model_config = ConfigDict(extra="allow", populate_by_name=True)