Many changes

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
+## Unreleased
+
+### Changed
+
+* Datasheet visual redesign: hero header with version/DOI/license/size badges, stat cards, pure-SVG AI-readiness donut, carded sections, improved print/PDF styling. Templates are themed via CSS custom properties in `base.html`.
+* Evidence graph HTML rebuilt from a 1,000-line Python f-string into a Jinja template (`templates/evidence_graph/`) plus a real JavaScript asset. React/ReactDOM/dagre are now vendored and inlined, so the generated file is fully self-contained and works offline (previously required CDN access). Added a node-type legend, hover tooltips, and a reset-view button.
+* Embedded RO-Crate JSON in evidence graph HTML is now `<`-escaped, preventing `</script>` breakout from metadata values.
+* RO-Crate root entities are resolved via the `ro-crate-metadata.json` descriptor's `about` reference instead of assuming `@graph[1]`.
+* Metadata files are rewritten atomically (temp file + rename), so an interrupted run can no longer corrupt `ro-crate-metadata.json`.
+* Subcrate metadata is loaded and validated once per datasheet build instead of three times.
+* Datasheet generation errors now go through `logging` instead of bare prints.
+
+### Fixed
+
+* `build datasheet --template-dir` was accepted but silently ignored; it is now honored.
+
 ## 0.2.0 (2024-03-28)
 
 [GitHub release](https://github.com/fairscape/fairscape-cli/releases/tag/0.2.0)

pyproject.toml

@@ -49,6 +49,9 @@ dependencies = [
         "mongomock",
         "huggingface_hub>=0.20.0",
         "pyarrow>=17.0.0",
+        "fairscape_graph_tools",
+        "pydantic-ai>=0.1.0",
+        "httpx>=0.28.1",
 ]
 
 [tool.setuptools]
@@ -71,6 +74,7 @@ members = [
 
 [tool.uv.sources]
 fairscape-cli = { workspace = true, editable = true }
+fairscape_graph_tools = { path = "../fairscape_graph_tools", editable = true }
 
 [dependency-groups]
 dev = [

src/fairscape_cli/__main__.py

@@ -1,5 +1,9 @@
+import logging
+
 import click
 
+logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(name)s: %(message)s")
+
 # Import command groups from their new locations
 from fairscape_cli.commands.rocrate_commands import rocrate_group
 from fairscape_cli.commands.import_commands import import_group
@@ -8,6 +12,7 @@
 from fairscape_cli.commands.schema_commands import schema
 from fairscape_cli.commands.augment_commands import augment_group
 from fairscape_cli.commands.track import track
+from fairscape_cli.commands.interpret import interpret_group
 
 @click.group(invoke_without_command=True)
 @click.pass_context
@@ -27,6 +32,7 @@ def cli(ctx):
 cli.add_command(schema, name='schema')
 cli.add_command(augment_group, name='augment')
 cli.add_command(track, name='track')
+cli.add_command(interpret_group, name='interpret')
 
 if __name__ == "__main__":
     cli()

src/fairscape_cli/commands/augment_commands.py

@@ -12,6 +12,14 @@
     calculate_inputs_outputs,
     add_inputs_outputs_to_rocrate
 )
+from fairscape_cli.entailments.summary_stats import (
+    build_summary_dataset,
+    compute_stats,
+    human_size,
+    is_dataset,
+    is_tabular_entity,
+    read_table,
+)
 
 @click.group('augment')
 def augment_group():
@@ -202,4 +210,115 @@ def add_io_command(
         ctx.exit(1)
     except Exception as e:
         click.echo(f"An unexpected error occurred: {type(e).__name__} - {e}", err=True)
-        ctx.exit(1)
+        ctx.exit(1)
+
+
+@augment_group.command('summary-stats')
+@click.argument('rocrate-path', type=click.Path(exists=True, path_type=pathlib.Path))
+@click.option('--id', 'entity_id', default=None, help="Compute stats only for this Dataset @id. Default: every tabular Dataset.")
+@click.option('--overwrite/--skip-existing', default=False, help="Recompute stats for Datasets that already have hasSummaryStatistics. Default: skip.")
+@click.option('--http-timeout', default=60, show_default=True, help="HTTP timeout in seconds for remote contentUrls.")
+@click.option('--dry-run', is_flag=True, help="Print what would change without writing ro-crate-metadata.json.")
+@click.pass_context
+def summary_stats_command(
+    ctx,
+    rocrate_path: pathlib.Path,
+    entity_id: str,
+    overwrite: bool,
+    http_timeout: int,
+    dry_run: bool,
+):
+    """
+    Compute row/column counts plus per-column statistics for tabular Datasets
+    in an RO-Crate. For each matched Dataset:
+
+      * read its contentUrl (local path, file://, or http(s)://) as csv/tsv/parquet
+      * populate rowCount / columnCount / contentSize / sampleSize on the source
+      * append a child SummaryStats Dataset (per-column dtype, null counts, and
+        numeric min/max/mean/std) and link it via hasSummaryStatistics
+
+    For a no-extra-deps row+column counter that works on csv/tsv only, use
+    `Dataset.add_summary_stats()` from fairscape_models directly.
+    """
+    crate_root = rocrate_path if rocrate_path.is_dir() else rocrate_path.parent
+    metadata_path = crate_root / "ro-crate-metadata.json"
+    if not metadata_path.exists():
+        click.echo(f"Error: RO-Crate metadata file not found at {metadata_path}", err=True)
+        ctx.exit(1)
+
+    with metadata_path.open("r") as f:
+        metadata = json.load(f)
+
+    graph = metadata.get("@graph") or []
+    if not graph:
+        click.echo("Error: RO-Crate metadata has no @graph", err=True)
+        ctx.exit(1)
+
+    root_dataset = graph[1] if len(graph) > 1 else None
+    new_entities: list = []
+    updated_count = 0
+    skipped: list = []
+
+    for entity in graph:
+        if not is_dataset(entity):
+            continue
+        if entity_id and entity.get("@id") != entity_id:
+            continue
+        if entity is root_dataset:
+            continue
+        if not is_tabular_entity(entity):
+            skipped.append((entity.get("@id"), "not tabular"))
+            continue
+        if entity.get("hasSummaryStatistics") and not overwrite:
+            skipped.append((entity.get("@id"), "already has hasSummaryStatistics (use --overwrite)"))
+            continue
+        content_url = entity.get("contentUrl")
+        if isinstance(content_url, list):
+            content_url = content_url[0] if content_url else None
+        if not content_url:
+            skipped.append((entity.get("@id"), "no contentUrl"))
+            continue
+
+        try:
+            df, size_bytes, source_desc = read_table(content_url, crate_root, http_timeout=http_timeout)
+        except Exception as e:
+            click.echo(f"  ! {entity.get('@id')}: failed to read ({type(e).__name__}: {e})", err=True)
+            skipped.append((entity.get("@id"), f"read failed: {e}"))
+            continue
+
+        rows, cols = int(df.shape[0]), int(df.shape[1])
+        size_str = human_size(size_bytes)
+        per_column = compute_stats(df)
+
+        entity["rowCount"] = rows
+        entity["columnCount"] = cols
+        entity["contentSize"] = size_str
+        entity.setdefault("sampleSize", rows)
+
+        stats_entity = build_summary_dataset(entity, rows, cols, size_str, per_column)
+        entity["hasSummaryStatistics"] = {"@id": stats_entity["@id"]}
+        new_entities.append(stats_entity)
+        updated_count += 1
+        click.echo(f"  ✓ {entity['@id']} ← {rows} rows × {cols} cols, {size_str} (source: {source_desc})")
+
+    if not updated_count:
+        click.echo("No Datasets updated.")
+        for eid, reason in skipped:
+            click.echo(f"  - {eid}: {reason}")
+        return
+
+    if new_entities and root_dataset is not None:
+        root_dataset.setdefault("hasPart", []).extend([{"@id": e["@id"]} for e in new_entities])
+        graph.extend(new_entities)
+
+    if dry_run:
+        click.echo(f"\nDRY RUN — would update {updated_count} dataset(s) and add {len(new_entities)} SummaryStats entit{'y' if len(new_entities) == 1 else 'ies'}.")
+        return
+
+    with metadata_path.open("w") as f:
+        json.dump(metadata, f, indent=2)
+    click.echo(f"\nUpdated {updated_count} dataset(s); appended {len(new_entities)} SummaryStats entit{'y' if len(new_entities) == 1 else 'ies'} to {metadata_path}.")
+    if skipped:
+        click.echo("Skipped:")
+        for eid, reason in skipped:
+            click.echo(f"  - {eid}: {reason}")

src/fairscape_cli/commands/build_commands.py

@@ -8,8 +8,10 @@
 from datetime import datetime
 
 from fairscape_cli.datasheet_builder.rocrate.datasheet_generator import DatasheetGenerator
-from fairscape_cli.datasheet_builder.evidence_graph.graph_builder import generate_evidence_graph_from_rocrate
 from fairscape_cli.datasheet_builder.evidence_graph.html_builder import generate_evidence_graph_html
+from fairscape_cli.interpret.local_graph import LocalGraphSource
+from fairscape_cli.interpret.local_sink import LocalResultSink
+from fairscape_graph_tools.evidence_graph_builder import EvidenceGraphBuilder
 from fairscape_cli.utils.build_utils import (
     process_all_subcrates,
     process_croissant,
@@ -28,9 +30,12 @@
     collect_subcrate_metadata,
     collect_subcrate_aggregated_metrics
 )
+from fairscape_cli.models.rocrate import _extract_content_size_bytes, format_content_size_bytes
 
 from fairscape_models.rocrate import ROCrateV1_2, ROCrateMetadataElem
-from fairscape_cli.utils.serialization import prune_none
+from fairscape_cli.datasheet_builder import get_default_template_dir
+from fairscape_cli.utils.serialization import prune_none, write_json_atomic
+from fairscape_cli.utils.rocrate_helpers import get_root_entity_dict
 from fairscape_models.conversion.converter import ROCToTargetConverter
 from fairscape_models.conversion.mapping.croissant import MAPPING_CONFIGURATION as CROISSANT_MAPPING
 
@@ -236,7 +241,12 @@ def build_release(
     if citation: parent_params["citation"] = citation
     if funder: parent_params["funder"] = funder
     if usage_info: parent_params["usageInfo"] = usage_info
-    if content_size: parent_params["contentSize"] = content_size
+    # An explicitly provided release contentSize is authoritative; otherwise
+    # fill it from the hierarchy-aware sub-crate roll-up.
+    if content_size:
+        parent_params["contentSize"] = content_size
+    elif aggregated_metrics.total_content_size_bytes > 0:
+        parent_params["contentSize"] = format_content_size_bytes(aggregated_metrics.total_content_size_bytes)
     if ethical_review: parent_params["ethicalReview"] = ethical_review
     if has_summary_stats: parent_params["hasSummaryStats"] = has_summary_stats
 
@@ -323,7 +333,8 @@ def build_release(
     parent_params["evi:computationCount"] = aggregated_metrics.computation_count
     parent_params["evi:softwareCount"] = aggregated_metrics.software_count
     parent_params["evi:schemaCount"] = aggregated_metrics.schema_count
-    parent_params["evi:totalContentSizeBytes"] = aggregated_metrics.total_content_size_bytes
+    declared_size_bytes = _extract_content_size_bytes(content_size) if content_size else 0
+    parent_params["evi:totalContentSizeBytes"] = declared_size_bytes or aggregated_metrics.total_content_size_bytes
     parent_params["evi:entitiesWithSummaryStats"] = aggregated_metrics.entities_with_summary_stats
     parent_params["evi:entitiesWithChecksums"] = aggregated_metrics.entities_with_checksums
     parent_params["evi:totalEntities"] = aggregated_metrics.total_entities
@@ -402,8 +413,7 @@ def build_datasheet(ctx, rocrate_path, output, template_dir, published, pdf, ski
 
     output_path = output if output else crate_dir / "ro-crate-datasheet.html"
 
-    package_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    template_dir = Path(os.path.join(package_dir, 'datasheet_builder', 'templates'))
+    template_dir = template_dir if template_dir else get_default_template_dir()
 
     # Link subcrates if needed
     click.echo("Checking subcrate links...")
@@ -512,10 +522,23 @@ def generate_evidence_graph(
     # Generate the evidence graph
     try:
         click.echo(f"Generating evidence graph for {ark_id} from {metadata_file}...")
-        evidence_graph = generate_evidence_graph_from_rocrate(
-            rocrate_path=metadata_file,
-            output_path=output_file,
-            node_id=ark_id
+
+        source = LocalGraphSource(primary_path=metadata_file)
+        resolved = source.find_entity(ark_id)
+        if resolved is None:
+            click.echo(f"ERROR: Could not find entity with ID {ark_id} in RO-Crate")
+            ctx.exit(1)
+            return
+        resolved_id = resolved.get("@id", ark_id)
+        resolved_name = resolved.get("name") or "Unknown"
+
+        sink = LocalResultSink(output_path=output_file)
+        builder = EvidenceGraphBuilder(source, sink)
+        builder.build(
+            resolved_id,
+            owner_email=resolved_id,
+            name=f"Evidence Graph - {resolved_name}",
+            description=f"Evidence graph for {resolved_name}",
         )
         click.echo(f"Evidence graph saved to {output_file}")
 
@@ -532,25 +555,20 @@ def generate_evidence_graph(
             click.echo("WARNING: generate_evidence_graph_html module not found, skipping visualization")
             click.echo("To generate visualizations, please install the visualization module.")
         except Exception as e:
-            click.echo(f"ERROR generating visualization: {str(e)}")\
-                    
+            click.echo(f"ERROR generating visualization: {str(e)}")
+
         try:
             with open(metadata_file, 'r') as f:
                 metadata = json.load(f)
-
-            i = 0
-            for entity in metadata.get('@graph', []):
-                if i == 1:
-                    entity['localEvidenceGraph'] = {
-                        "@id": str(html_output_path)
-                    }
-                    break
-                i += 1
-
-            # Write the updated metadata back to the file
-            with open(metadata_file, 'w') as f:
-                json.dump(prune_none(metadata), f, indent=2)
-
+
+            root_entity = get_root_entity_dict(metadata.get('@graph', []))
+            if root_entity is not None:
+                root_entity['localEvidenceGraph'] = {
+                    "@id": str(html_output_path)
+                }
+
+            write_json_atomic(metadata_file, prune_none(metadata))
+
             click.echo(f"Added hasEvidenceGraph reference to {ark_id} in RO-Crate metadata")
         except Exception as e:
             click.echo(f"WARNING: Failed to add hasEvidenceGraph reference: {str(e)}")