...
has mixed children and BS4's string= predicate never matches. + # Locate the label first and walk up to the enclosing . + def _doi_from_strong(label_prefix: str) -> Optional[str]: + strong_tag = discovery_card_body.find( + 'strong', + string=lambda text: text and text.strip().startswith(label_prefix) + ) + if not strong_tag: + return None + parent_p = strong_tag.find_parent('p') + if not parent_p: + return None + doi_link = parent_p.find('a', href=True) + return doi_link['href'] if doi_link else None + + versioned_doi = _doi_from_strong('DOI (version') + if versioned_doi: + metadata['dataset_identifier'] = versioned_doi + + if 'dataset_identifier' not in metadata: + latest_doi = _doi_from_strong('DOI (latest version') + if latest_doi: + metadata['dataset_identifier'] = latest_doi access_card_body = None access_header = soup.find('h5', class_='card-header', string='Access') if access_header: access_card_body = access_header.find_next_sibling('div', class_='card-body') if access_card_body: - license_p = access_card_body.find('p', string=lambda text: text and text.strip().startswith('License (for files):')) + # Same mixed-children issue as DOI: locate the label and look up to . + strong_license = access_card_body.find( + 'strong', + string=lambda text: text and text.strip().startswith('License (for files):') + ) + license_p = strong_license.find_parent('p') if strong_license else None if license_p: - license_link = license_p.find('a') - if license_link and license_link.get('href'): + license_link = license_p.find('a', href=True) + if license_link: metadata['extracted_license'] = license_link['href'] return metadata + def _fetch_checksums(self) -> Tuple[Dict[str, str], Optional[str]]: + """Fetch and parse PhysioNet's checksum manifest for this dataset. + + Returns a (mapping, algorithm) tuple where mapping is {relative_path: hash} + and algorithm is one of "sha256", "md5", or None when no manifest is found. + Each manifest line is ` ` (single space separator). + """ + candidates = [ + ("SHA256SUMS.txt", "sha256"), + ("SHA256SUMS", "sha256"), + ("MD5SUMS.txt", "md5"), + ("MD5SUMS", "md5"), + ] + for filename, algorithm in candidates: + url = f"{self.base_file_download_url}{filename}" + try: + response = requests.get(url, timeout=30) + except requests.exceptions.RequestException as e: + print(f"Warning: Failed to fetch checksum file {url}: {e}", file=sys.stderr) + continue + if response.status_code != 200: + continue + + mapping: Dict[str, str] = {} + for line in response.text.splitlines(): + line = line.strip() + if not line: + continue + # Format: " " — split on first whitespace only, + # since paths may contain spaces. + parts = line.split(None, 1) + if len(parts) != 2: + continue + digest, rel_path = parts[0].strip(), parts[1].strip() + mapping[rel_path] = digest + if mapping: + print(f"Loaded {len(mapping)} {algorithm} checksums from {filename}", file=sys.stderr) + return mapping, algorithm + return {}, None + def _process_file_table_rows(self, table_body_soup: BeautifulSoup, current_subdir_path: str, all_files_list: List[Dict[str, Any]], subdirs_to_explore: List[str], crate_default_date_published: str): rows_to_process = list(table_body_soup.select('tr')) for row in rows_to_process: + row_classes = row.get('class', []) + + # PhysioNet renders a "Parent Directory" navigation row (class="parentdir") + # on every subdirectory page. It has no class="subdir", so without this skip + # it would be misclassified as a file and added as a Dataset per traversed dir. + if 'parentdir' in row_classes: + continue + name_cell_a_tag = row.select_one('td a') if not name_cell_a_tag: continue name = name_cell_a_tag.get_text(strip=True) - - if name in ['.', '..', '']: + + if name in ['.', '..', '', 'Parent Directory']: continue - is_directory = 'subdir' in row.get('class', []) - + is_directory = 'subdir' in row_classes or name_cell_a_tag.has_attr('data-dfp-dir') + item_relative_path: str = "" if is_directory: item_relative_path = name_cell_a_tag.get('data-dfp-dir') @@ -346,9 +412,9 @@ def _process_file_table_rows(self, table_body_soup: BeautifulSoup, current_subdi if item_relative_path not in subdirs_to_explore and item_relative_path != current_subdir_path : print(f"DEBUG: Found directory: {name} (data-dfp-dir: {item_relative_path}) in {current_subdir_path or ''}. Adding to queue.", file=sys.stderr) - subdirs_to_explore.append(item_relative_path) - else: - item_relative_path = os.path.join(current_subdir_path, name).replace('\\', '/') + subdirs_to_explore.append(item_relative_path) + else: + item_relative_path = os.path.join(current_subdir_path, name).replace('\\', '/') if not is_directory: @@ -532,6 +598,8 @@ def to_rocrate( print(f"Total files found after exploring subdirectories: {len(all_file_entries)}", file=sys.stderr) + checksum_map, checksum_algorithm = self._fetch_checksums() + crate_elements_to_add = [] for item_data in all_file_entries: item_path_guid_segment = item_data['relative_path'].replace('/', '_').replace('.', '_').replace(' ', '_').lower() @@ -554,6 +622,10 @@ def to_rocrate( if 'contentSize' in item_data: dataset_params['contentSize'] = item_data['contentSize'] if 'contentUrl' in item_data: dataset_params['contentUrl'] = item_data['contentUrl'] + digest = checksum_map.get(item_data['relative_path']) + if digest and checksum_algorithm: + dataset_params[checksum_algorithm] = digest + try: dataset_entity = GenerateDataset(**dataset_params) crate_elements_to_add.append(dataset_entity) diff --git a/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py b/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py index 605cd54..8710b54 100644 --- a/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py +++ b/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py @@ -213,7 +213,9 @@ def fetch_data(self, doi: str, include_files: bool = True) -> ResearchData: if is_software: software_data.append({ **item_common_data, - "version": latest_version_info.get("versionNumber", "1.0"), # Use dataset version for software + # Dataverse returns versionNumber as a numeric type; the + # Software pydantic model requires a string. + "version": str(latest_version_info.get("versionNumber", "1.0")), "documentation_url": dataset_landing_url }) else: @@ -249,7 +251,7 @@ def fetch_data(self, doi: str, include_files: bool = True) -> ResearchData: "url": f"{self.server_url}/dataset.xhtml?persistentId=doi:{doi}", "contentUrl": f"{self.server_url}/dataset.xhtml?persistentId=doi:{doi}", "publisher": dataset_metadata_full.get("publisher", ""), - "version": latest_version_info.get("versionNumber"), + "version": str(latest_version_info.get("versionNumber")) if latest_version_info.get("versionNumber") is not None else None, } ) diff --git a/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py b/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py new file mode 100644 index 0000000..0900534 --- /dev/null +++ b/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py @@ -0,0 +1,244 @@ +"""Manifest connector. + +Reads a CSV manifest + JSON sidecar describing a published dataset that +doesn't have (or doesn't need) a dedicated repository connector, and returns a +ResearchData instance for `ResearchData.to_rocrate()` to write out. +""" +import csv +import json +import os +import pathlib +from datetime import datetime +from typing import Any, Dict, List, Optional + +from fairscape_cli.data_fetcher.generic_data.research_data import ResearchData + + +SIDECAR_REQUIRED_FIELDS = ("name", "description", "authors") +MANIFEST_REQUIRED_COLUMNS = ("name", "description", "contentUrl") +SIZE_UNITS = ("B", "KB", "MB", "GB", "TB", "PB") + +# Extensions that should default to Software entities rather than Datasets. +# Mirrors fairscape_cli.data_fetcher.generic_data.connectors.dataverse_connector +# with a few additional script suffixes. +SOFTWARE_EXTENSIONS = { + ".py", ".r", ".sh", ".bash", ".ipynb", ".jl", ".m", + ".exe", ".java", ".cpp", ".js", ".jsx", ".ts", ".tsx", ".css", +} + + +def _auto_detect_type(name: str) -> str: + """Return 'software' or 'dataset' based on file extension.""" + ext = os.path.splitext(name)[1].lower() + return "software" if ext in SOFTWARE_EXTENSIONS else "dataset" + + +def _human_size(n: int) -> str: + """Format byte count as a short human-readable string matching the other connectors.""" + if n is None: + return "" + size = float(n) + i = 0 + while size >= 1024 and i < len(SIZE_UNITS) - 1: + size /= 1024.0 + i += 1 + if i == 0: + return f"{int(size)} {SIZE_UNITS[i]}" + return f"{size:.1f} {SIZE_UNITS[i]}" + + +def _ext_format(name: str) -> str: + """Derive a format token from a filename extension. Returns 'unknown' if there's nothing.""" + ext = os.path.splitext(name)[1].lstrip(".") + return ext.lower() if ext else "unknown" + + +def _split_pipe(value: Optional[str]) -> List[str]: + if not value: + return [] + return [piece.strip() for piece in value.split("|") if piece.strip()] + + +def _resolve_sidecar(manifest_path: pathlib.Path, explicit: Optional[pathlib.Path]) -> pathlib.Path: + if explicit is not None: + if not explicit.exists(): + raise ValueError(f"Sidecar not found: {explicit}") + return explicit + + siblings = [ + manifest_path.with_suffix(".json"), + manifest_path.parent / "crate.json", + ] + for candidate in siblings: + if candidate.exists(): + return candidate + + raise ValueError( + "No sidecar found. Looked for " + + ", ".join(str(s) for s in siblings) + + ". Pass --sidecar to point at one explicitly." + ) + + +def _load_sidecar(path: pathlib.Path) -> Dict[str, Any]: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + missing = [field for field in SIDECAR_REQUIRED_FIELDS if not data.get(field)] + if missing: + raise ValueError( + f"Sidecar {path} is missing required field(s): {', '.join(missing)}" + ) + if not isinstance(data.get("authors"), list): + raise ValueError(f"Sidecar 'authors' must be a list (got {type(data.get('authors')).__name__}).") + return data + + +def _row_to_file_dict(row: Dict[str, str], sidecar: Dict[str, Any]) -> Dict[str, Any]: + """Translate one manifest row into a dict ready for ResearchData. + + The dict carries an extra '_type' key (`'dataset'` or `'software'`) so the + caller can route it into `ResearchData.files` or `ResearchData.software`. + The `_type` is sourced from the row's `type` column (case-insensitive), + falling back to extension-based auto-detection. + """ + name = (row.get("name") or "").strip() + if not name: + raise ValueError(f"Manifest row missing 'name': {row}") + + content_url = (row.get("contentUrl") or "").strip() + if not content_url: + raise ValueError(f"Manifest row '{name}' missing 'contentUrl'.") + + declared_type = (row.get("type") or "").strip().lower() + if declared_type in ("software", "dataset"): + row_type = declared_type + elif declared_type == "": + row_type = _auto_detect_type(name) + else: + raise ValueError( + f"Manifest row '{name}' has unknown type {declared_type!r}; " + f"expected 'dataset', 'software', or blank." + ) + + description = (row.get("description") or "").strip() or sidecar.get("description") or "" + fmt = (row.get("format") or "").strip() or _ext_format(name) + + # Size: prefer explicit byte count (we format it); else accept pre-formatted string; else nothing. + content_size: Optional[str] = None + size_bytes_raw = (row.get("size_bytes") or "").strip() + if size_bytes_raw: + try: + content_size = _human_size(int(size_bytes_raw)) + except ValueError: + raise ValueError(f"Manifest row '{name}' has non-integer size_bytes: {size_bytes_raw!r}") + elif (row.get("contentSize") or "").strip(): + content_size = row["contentSize"].strip() + + file_dict: Dict[str, Any] = { + "name": name, + "description": description, + "contentUrl": content_url, + "url": content_url, + "format": fmt, + "datePublished": (row.get("datePublished") or "").strip() or sidecar.get("publication_date"), + "version": (row.get("version") or "").strip() or "1.0", + } + + md5 = (row.get("md5") or "").strip() + if md5: + file_dict["md5"] = md5 + sha256 = (row.get("sha256") or "").strip() + if sha256: + file_dict["sha256"] = sha256 + if content_size: + file_dict["contentSize"] = content_size + # The Dataverse connector also stuffs raw bytes under 'size'; mirror that + # so anything downstream that reads 'size' keeps working. + if size_bytes_raw: + file_dict["size"] = int(size_bytes_raw) + + per_row_keywords = _split_pipe(row.get("keywords")) + if per_row_keywords: + file_dict["keywords"] = per_row_keywords + + group = (row.get("group") or "").strip() + if group: + # Reserved for future schema-inference grouping; stash it where it'll + # round-trip without breaking anything. + file_dict["group"] = group + + file_dict["_type"] = row_type + return file_dict + + +def _to_software_dict(record: Dict[str, Any]) -> Dict[str, Any]: + """Translate a file-dict (built for ResearchData.files) into the shape + ResearchData.software expects, which `to_rocrate` passes to GenerateSoftware. + + The only meaningful difference is `datePublished` → `dateModified`. + """ + software = dict(record) + if "datePublished" in software: + software["dateModified"] = software.pop("datePublished") + return software + + +class ManifestConnector: + """Reads a CSV manifest + JSON sidecar and produces ResearchData.""" + + def __init__(self, sidecar_path: Optional[pathlib.Path] = None): + self.sidecar_path = pathlib.Path(sidecar_path) if sidecar_path else None + + def fetch_data(self, manifest_path: str, include_files: bool = True) -> ResearchData: + manifest_p = pathlib.Path(manifest_path) + if not manifest_p.exists(): + raise ValueError(f"Manifest not found: {manifest_p}") + + sidecar_p = _resolve_sidecar(manifest_p, self.sidecar_path) + sidecar = _load_sidecar(sidecar_p) + + files_data: List[Dict[str, Any]] = [] + software_data: List[Dict[str, Any]] = [] + if include_files: + with manifest_p.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + missing_cols = [c for c in MANIFEST_REQUIRED_COLUMNS if c not in (reader.fieldnames or [])] + if missing_cols: + raise ValueError( + f"Manifest {manifest_p} is missing required column(s): {', '.join(missing_cols)}. " + f"Found columns: {reader.fieldnames}" + ) + for row in reader: + if not any((row.get(c) or "").strip() for c in row): + continue # skip blank lines + record = _row_to_file_dict(row, sidecar) + row_type = record.pop("_type") + if row_type == "software": + software_data.append(_to_software_dict(record)) + else: + files_data.append(record) + + publication_date = sidecar.get("publication_date") or datetime.utcnow().date().isoformat() + + metadata: Dict[str, Any] = {} + if sidecar.get("url"): + metadata["url"] = sidecar["url"] + if sidecar.get("associated_publication"): + metadata["contentUrl"] = sidecar["associated_publication"] + if sidecar.get("version"): + metadata["version"] = sidecar["version"] + + return ResearchData( + repository_name=sidecar.get("repository_name") or "Manifest", + project_id=sidecar.get("project_id") or sidecar.get("doi") or manifest_p.stem, + title=sidecar["name"], + description=sidecar["description"], + authors=list(sidecar["authors"]), + license=sidecar.get("license"), + keywords=list(sidecar.get("keywords") or []), + publication_date=publication_date, + doi=sidecar.get("doi"), + files=files_data, + software=software_data, + metadata=metadata, + ) diff --git a/src/fairscape_cli/data_fetcher/generic_data/research_data.py b/src/fairscape_cli/data_fetcher/generic_data/research_data.py index fc29888..6934c12 100644 --- a/src/fairscape_cli/data_fetcher/generic_data/research_data.py +++ b/src/fairscape_cli/data_fetcher/generic_data/research_data.py @@ -90,12 +90,14 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: } if file_info.get("md5"): dataset_params["md5"] = file_info["md5"] + if file_info.get("sha256"): + dataset_params["sha256"] = file_info["sha256"] if file_info.get("contentSize"): dataset_params["contentSize"] = file_info["contentSize"] - + dataset = GenerateDataset(**dataset_params) elements_to_append.append(dataset) - + for software_info in self.software: software_version = software_info.get("version", "0.1.0") @@ -105,8 +107,8 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: software_author = self.authors[0] if self.authors else "Unknown" - software_item = GenerateSoftware( - guid=None, + software_kwargs = dict( + guid=None, name=software_info.get("name", "Unnamed software"), author=software_author, dateModified=software_info.get("dateModified", datetime.now().isoformat()), @@ -114,10 +116,18 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: description=software_description, associatedPublication=self.doi or None, additionalDocumentation=software_info.get("documentation_url", None), - format=software_info.get("format", "application/zip"), + format=software_info.get("format", "application/zip"), usedByComputation=[], - contentUrl=software_info.get("contentUrl", software_info.get("url", "")) + contentUrl=software_info.get("contentUrl", software_info.get("url", "")), ) + if software_info.get("md5"): + software_kwargs["md5"] = software_info["md5"] + if software_info.get("sha256"): + software_kwargs["sha256"] = software_info["sha256"] + if software_info.get("contentSize"): + software_kwargs["contentSize"] = software_info["contentSize"] + + software_item = GenerateSoftware(**software_kwargs) elements_to_append.append(software_item) if elements_to_append: @@ -138,6 +148,10 @@ def from_repository(cls, repository_type: str, identifier: str, **kwargs) -> 'Re api_token=kwargs.get('token') ) return connector.fetch_data(identifier, include_files=kwargs.get('include_files', True)) + elif repository_type.lower() == 'manifest': + from fairscape_cli.data_fetcher.generic_data.connectors.manifest_connector import ManifestConnector + connector = ManifestConnector(sidecar_path=kwargs.get('sidecar_path')) + return connector.fetch_data(identifier, include_files=kwargs.get('include_files', True)) else: raise ValueError(f"Unsupported repository type: {repository_type}") diff --git a/src/fairscape_cli/datasheet_builder/__init__.py b/src/fairscape_cli/datasheet_builder/__init__.py index 97e0706..56635fb 100644 --- a/src/fairscape_cli/datasheet_builder/__init__.py +++ b/src/fairscape_cli/datasheet_builder/__init__.py @@ -2,6 +2,14 @@ Datasheet Builder module for RO-Crate metadata visualization and documentation. """ +from pathlib import Path + from fairscape_cli.datasheet_builder.rocrate.datasheet_generator import DatasheetGenerator -__all__ = ['DatasheetGenerator'] \ No newline at end of file + +def get_default_template_dir() -> Path: + """Return the Jinja template directory shipped with the package.""" + return Path(__file__).parent / 'templates' + + +__all__ = ['DatasheetGenerator', 'get_default_template_dir'] diff --git a/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py b/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py deleted file mode 100644 index 62842c8..0000000 --- a/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py +++ /dev/null @@ -1,242 +0,0 @@ -from typing import Dict, List, Optional, Any, Union, Set -import json -import pathlib -from pathlib import Path -import click - -class EvidenceNode: - def __init__(self, id: str, type: str): - self.id = id - self.type = type - # For Computation nodes - self.usedSoftware: Optional[List[str]] = None - self.usedDataset: Optional[List[str]] = None - self.usedSample: Optional[List[str]] = None - self.usedInstrument: Optional[List[str]] = None - # For Dataset/Sample/Instrument nodes - self.generatedBy: Optional[str] = None - -class EvidenceGraphJSON: - def __init__(self, guid: str, owner: str, description: str, name: str = "Evidence Graph"): - self.metadataType = "evi:EvidenceGraph" - self.guid = guid - self.owner = owner - self.description = description - self.name = name - self.graph = None - - def build_graph(self, node_id: str, json_data: Dict[str, Any]): - processed = set() - self.graph = self._build_graph_recursive(node_id, json_data, processed) - - def _build_graph_recursive(self, node_id: str, json_data: Dict[str, Any], processed: Set[str]) -> Dict: - if node_id in processed: - return {"@id": node_id} - - # Find node in json data graph - node = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == node_id: - node = entity - break - - if not node: - return {"@id": node_id} - - processed.add(node_id) - result = self._build_base_node(node) - - # Determine node type based on @type - node_type = None - if isinstance(node.get("@type"), list): - for type_entry in node["@type"]: - if "Dataset" in type_entry: - node_type = "Dataset" - break - elif "Computation" in type_entry: - node_type = "Computation" - break - elif "Sample" in type_entry: - node_type = "Sample" - break - elif "Instrument" in type_entry: - node_type = "Instrument" - break - elif "Experiment" in type_entry: - node_type = "Experiment" - break - elif isinstance(node.get("@type"), str): - type_str = node.get("@type") - if "Dataset" in type_str: - node_type = "Dataset" - elif "Computation" in type_str: - node_type = "Computation" - elif "Sample" in type_str: - node_type = "Sample" - elif "Instrument" in type_str: - node_type = "Instrument" - elif "Experiment" in type_str: - node_type = "Experiment" - - if node_type in ["Dataset", "Sample", "Instrument"]: - if "generatedBy" in node: - result["generatedBy"] = self._build_computation_node(node, json_data, processed) - elif node_type in ["Computation", "Experiment"]: - if "usedDataset" in node: - result["usedDataset"] = self._build_used_resources(node["usedDataset"], json_data, processed) - if "usedSoftware" in node: - result["usedSoftware"] = self._build_software_reference(node["usedSoftware"], json_data) - if "usedSample" in node: - result["usedSample"] = self._build_used_resources(node["usedSample"], json_data, processed) - if "usedInstrument" in node: - result["usedInstrument"] = self._build_used_resources(node["usedInstrument"], json_data, processed) - - return result - - def _build_base_node(self, node: Dict) -> Dict: - return { - "@id": node["@id"], - "@type": node.get("@type"), - "name": node.get("name"), - "description": node.get("description") - } - - def _build_computation_node(self, parent_node: Dict, json_data: Dict[str, Any], processed: Set[str]) -> Dict: - # If generatedBy is an empty list, don't add anything - if isinstance(parent_node["generatedBy"], list) and len(parent_node["generatedBy"]) == 0: - return {} - - comp_id = None - if isinstance(parent_node["generatedBy"], list): - if parent_node["generatedBy"] and isinstance(parent_node["generatedBy"][0], dict): - comp_id = parent_node["generatedBy"][0].get("@id") - elif isinstance(parent_node["generatedBy"], dict): - comp_id = parent_node["generatedBy"].get("@id") - - if not comp_id: - return {"@id": "unknown-computation"} - - comp = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == comp_id: - comp = entity - break - - if not comp: - return {"@id": comp_id} - - return self._build_graph_recursive(comp_id, json_data, processed) - - def _build_used_resources(self, used_resources: Union[Dict, List], json_data: Dict[str, Any], processed: Set[str]) -> List: - if isinstance(used_resources, dict): - resource_id = used_resources.get("@id") - if resource_id: - return [self._build_graph_recursive(resource_id, json_data, processed)] - return [] - - if isinstance(used_resources, list): - resources = [] - for resource in used_resources: - if isinstance(resource, dict) and resource.get("@id"): - resources.append(self._build_graph_recursive(resource.get("@id"), json_data, processed)) - return resources - - return [] - - def _build_software_reference(self, used_software: Union[Dict, List], json_data: Dict[str, Any]) -> Union[Dict, List[Dict]]: - if isinstance(used_software, list): - if not used_software: - return [] - - software_refs = [] - for sw in used_software: - if isinstance(sw, dict) and sw.get("@id"): - software_id = sw.get("@id") - software = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == software_id: - software = entity - break - - if software: - software_refs.append(self._build_base_node(software)) - else: - software_refs.append({"@id": software_id}) - - return software_refs - - elif isinstance(used_software, dict) and used_software.get("@id"): - software_id = used_software.get("@id") - software = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == software_id: - software = entity - break - - if software: - return self._build_base_node(software) - return {"@id": software_id} - - return {"@id": "unknown-software"} - - def to_dict(self): - return { - "@type": self.metadataType, - "@id": self.guid, - "owner": self.owner, - "description": self.description, - "name": self.name, - "@graph": self.graph - } - - def save_to_file(self, file_path: Union[str, pathlib.Path]): - with open(file_path, 'w') as f: - json.dump(self.to_dict(), f, indent=2) - -def generate_evidence_graph_from_rocrate( - rocrate_path: Union[str, pathlib.Path], - output_path: Optional[Union[str, pathlib.Path]] = None, - node_id: str = "" -) -> Dict[str, Any]: - """ - Generate an evidence graph from an RO-Crate JSON file for a specific node ID - - Args: - rocrate_path: Path to the RO-Crate metadata JSON file - output_path: Optional path to save the evidence graph JSON - node_id: ID of the node to start building the graph from - - Returns: - The generated evidence graph as a dictionary - """ - # Load RO-Crate data - with open(rocrate_path, 'r') as f: - rocrate_data = json.load(f) - - # Find the root entity - root_entity = None - for entity in rocrate_data.get("@graph", []): - if entity.get("@id") == node_id: - root_entity = entity - break - - if not root_entity: - raise ValueError(f"Could not find entity with ID {node_id} in RO-Crate") - - # Create evidence graph - graph_id = f"{node_id}-evidence-graph" - graph = EvidenceGraphJSON( - guid=graph_id, - owner=node_id, - description=f"Evidence graph for {root_entity.get('name', 'Unknown')}", - name=f"Evidence Graph - {root_entity.get('name', 'Unknown')}" - ) - - # Build the graph - graph.build_graph(node_id, rocrate_data) - - # Save to file if output path provided - if output_path: - graph.save_to_file(output_path) - - return graph.to_dict() diff --git a/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py b/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py index fd272db..1635ab9 100644 --- a/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py +++ b/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py @@ -1,33 +1,57 @@ -import json -import os +"""Generate a standalone, self-contained HTML visualization of an evidence graph. + +The HTML shell lives in templates/evidence_graph/evidence_graph.html.j2 and the +application JavaScript in evidence_graph.js next to it. React, ReactDOM and +dagre are vendored under templates/evidence_graph/vendor/ and inlined into the +output, so the generated file is a single offline-shareable HTML document with +no CDN dependency. +""" import argparse +import json +import logging from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +logger = logging.getLogger(__name__) + +TEMPLATE_DIR = Path(__file__).parent.parent / 'templates' / 'evidence_graph' +VENDOR_FILES = ( + 'vendor/react.production.min.js', + 'vendor/react-dom.production.min.js', + 'vendor/dagre.min.js', +) + + +def _inline_script_safe(js: str) -> str: + """Make JS safe to inline in a ' breakout).""" + return js.replace(' - - - - - Evidence Graph Visualization - - - - - - - - - - - - """ + vendor_js = ';\n'.join( + (TEMPLATE_DIR / name).read_text(encoding='utf-8') for name in VENDOR_FILES + ) + app_js = (TEMPLATE_DIR / 'evidence_graph.js').read_text(encoding='utf-8') + + # Escaping '<' keeps the JSON valid JS while preventing a '' (or + # ' ☰ - + ↑ Navigation Datasheet Summary - Dataset Details + Release Overview AI Ready Information Composition @@ -717,24 +1071,36 @@ {% endif %} - {{ title }} - Version: {{ version }} + + {% if version %} + Version {{ version }} + {% endif %} + {% if doi %} + DOI ↗ + {% endif %} + {% if license_value %} + License ↗ + {% endif %} + {% if content_size %} + {{ content_size }} + {% endif %} + {% if release_date %} + Released {{ release_date }} + {% endif %} + {{ summary_section | safe }} - {{ overview_section | safe }} - {{ use_cases_section | safe }} - Composition (Datasets {{ subcrate_count }}) @@ -750,17 +1116,168 @@ Composition (Datasets {{ subcrate_count }}) >). -
. + def _doi_from_strong(label_prefix: str) -> Optional[str]: + strong_tag = discovery_card_body.find( + 'strong', + string=lambda text: text and text.strip().startswith(label_prefix) + ) + if not strong_tag: + return None + parent_p = strong_tag.find_parent('p') + if not parent_p: + return None + doi_link = parent_p.find('a', href=True) + return doi_link['href'] if doi_link else None + + versioned_doi = _doi_from_strong('DOI (version') + if versioned_doi: + metadata['dataset_identifier'] = versioned_doi + + if 'dataset_identifier' not in metadata: + latest_doi = _doi_from_strong('DOI (latest version') + if latest_doi: + metadata['dataset_identifier'] = latest_doi access_card_body = None access_header = soup.find('h5', class_='card-header', string='Access') if access_header: access_card_body = access_header.find_next_sibling('div', class_='card-body') if access_card_body: - license_p = access_card_body.find('p', string=lambda text: text and text.strip().startswith('License (for files):')) + # Same mixed-children issue as DOI: locate the label and look up to . + strong_license = access_card_body.find( + 'strong', + string=lambda text: text and text.strip().startswith('License (for files):') + ) + license_p = strong_license.find_parent('p') if strong_license else None if license_p: - license_link = license_p.find('a') - if license_link and license_link.get('href'): + license_link = license_p.find('a', href=True) + if license_link: metadata['extracted_license'] = license_link['href'] return metadata + def _fetch_checksums(self) -> Tuple[Dict[str, str], Optional[str]]: + """Fetch and parse PhysioNet's checksum manifest for this dataset. + + Returns a (mapping, algorithm) tuple where mapping is {relative_path: hash} + and algorithm is one of "sha256", "md5", or None when no manifest is found. + Each manifest line is ` ` (single space separator). + """ + candidates = [ + ("SHA256SUMS.txt", "sha256"), + ("SHA256SUMS", "sha256"), + ("MD5SUMS.txt", "md5"), + ("MD5SUMS", "md5"), + ] + for filename, algorithm in candidates: + url = f"{self.base_file_download_url}{filename}" + try: + response = requests.get(url, timeout=30) + except requests.exceptions.RequestException as e: + print(f"Warning: Failed to fetch checksum file {url}: {e}", file=sys.stderr) + continue + if response.status_code != 200: + continue + + mapping: Dict[str, str] = {} + for line in response.text.splitlines(): + line = line.strip() + if not line: + continue + # Format: " " — split on first whitespace only, + # since paths may contain spaces. + parts = line.split(None, 1) + if len(parts) != 2: + continue + digest, rel_path = parts[0].strip(), parts[1].strip() + mapping[rel_path] = digest + if mapping: + print(f"Loaded {len(mapping)} {algorithm} checksums from {filename}", file=sys.stderr) + return mapping, algorithm + return {}, None + def _process_file_table_rows(self, table_body_soup: BeautifulSoup, current_subdir_path: str, all_files_list: List[Dict[str, Any]], subdirs_to_explore: List[str], crate_default_date_published: str): rows_to_process = list(table_body_soup.select('tr')) for row in rows_to_process: + row_classes = row.get('class', []) + + # PhysioNet renders a "Parent Directory" navigation row (class="parentdir") + # on every subdirectory page. It has no class="subdir", so without this skip + # it would be misclassified as a file and added as a Dataset per traversed dir. + if 'parentdir' in row_classes: + continue + name_cell_a_tag = row.select_one('td a') if not name_cell_a_tag: continue name = name_cell_a_tag.get_text(strip=True) - - if name in ['.', '..', '']: + + if name in ['.', '..', '', 'Parent Directory']: continue - is_directory = 'subdir' in row.get('class', []) - + is_directory = 'subdir' in row_classes or name_cell_a_tag.has_attr('data-dfp-dir') + item_relative_path: str = "" if is_directory: item_relative_path = name_cell_a_tag.get('data-dfp-dir') @@ -346,9 +412,9 @@ def _process_file_table_rows(self, table_body_soup: BeautifulSoup, current_subdi if item_relative_path not in subdirs_to_explore and item_relative_path != current_subdir_path : print(f"DEBUG: Found directory: {name} (data-dfp-dir: {item_relative_path}) in {current_subdir_path or ''}. Adding to queue.", file=sys.stderr) - subdirs_to_explore.append(item_relative_path) - else: - item_relative_path = os.path.join(current_subdir_path, name).replace('\\', '/') + subdirs_to_explore.append(item_relative_path) + else: + item_relative_path = os.path.join(current_subdir_path, name).replace('\\', '/') if not is_directory: @@ -532,6 +598,8 @@ def to_rocrate( print(f"Total files found after exploring subdirectories: {len(all_file_entries)}", file=sys.stderr) + checksum_map, checksum_algorithm = self._fetch_checksums() + crate_elements_to_add = [] for item_data in all_file_entries: item_path_guid_segment = item_data['relative_path'].replace('/', '_').replace('.', '_').replace(' ', '_').lower() @@ -554,6 +622,10 @@ def to_rocrate( if 'contentSize' in item_data: dataset_params['contentSize'] = item_data['contentSize'] if 'contentUrl' in item_data: dataset_params['contentUrl'] = item_data['contentUrl'] + digest = checksum_map.get(item_data['relative_path']) + if digest and checksum_algorithm: + dataset_params[checksum_algorithm] = digest + try: dataset_entity = GenerateDataset(**dataset_params) crate_elements_to_add.append(dataset_entity) diff --git a/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py b/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py index 605cd54..8710b54 100644 --- a/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py +++ b/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py @@ -213,7 +213,9 @@ def fetch_data(self, doi: str, include_files: bool = True) -> ResearchData: if is_software: software_data.append({ **item_common_data, - "version": latest_version_info.get("versionNumber", "1.0"), # Use dataset version for software + # Dataverse returns versionNumber as a numeric type; the + # Software pydantic model requires a string. + "version": str(latest_version_info.get("versionNumber", "1.0")), "documentation_url": dataset_landing_url }) else: @@ -249,7 +251,7 @@ def fetch_data(self, doi: str, include_files: bool = True) -> ResearchData: "url": f"{self.server_url}/dataset.xhtml?persistentId=doi:{doi}", "contentUrl": f"{self.server_url}/dataset.xhtml?persistentId=doi:{doi}", "publisher": dataset_metadata_full.get("publisher", ""), - "version": latest_version_info.get("versionNumber"), + "version": str(latest_version_info.get("versionNumber")) if latest_version_info.get("versionNumber") is not None else None, } ) diff --git a/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py b/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py new file mode 100644 index 0000000..0900534 --- /dev/null +++ b/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py @@ -0,0 +1,244 @@ +"""Manifest connector. + +Reads a CSV manifest + JSON sidecar describing a published dataset that +doesn't have (or doesn't need) a dedicated repository connector, and returns a +ResearchData instance for `ResearchData.to_rocrate()` to write out. +""" +import csv +import json +import os +import pathlib +from datetime import datetime +from typing import Any, Dict, List, Optional + +from fairscape_cli.data_fetcher.generic_data.research_data import ResearchData + + +SIDECAR_REQUIRED_FIELDS = ("name", "description", "authors") +MANIFEST_REQUIRED_COLUMNS = ("name", "description", "contentUrl") +SIZE_UNITS = ("B", "KB", "MB", "GB", "TB", "PB") + +# Extensions that should default to Software entities rather than Datasets. +# Mirrors fairscape_cli.data_fetcher.generic_data.connectors.dataverse_connector +# with a few additional script suffixes. +SOFTWARE_EXTENSIONS = { + ".py", ".r", ".sh", ".bash", ".ipynb", ".jl", ".m", + ".exe", ".java", ".cpp", ".js", ".jsx", ".ts", ".tsx", ".css", +} + + +def _auto_detect_type(name: str) -> str: + """Return 'software' or 'dataset' based on file extension.""" + ext = os.path.splitext(name)[1].lower() + return "software" if ext in SOFTWARE_EXTENSIONS else "dataset" + + +def _human_size(n: int) -> str: + """Format byte count as a short human-readable string matching the other connectors.""" + if n is None: + return "" + size = float(n) + i = 0 + while size >= 1024 and i < len(SIZE_UNITS) - 1: + size /= 1024.0 + i += 1 + if i == 0: + return f"{int(size)} {SIZE_UNITS[i]}" + return f"{size:.1f} {SIZE_UNITS[i]}" + + +def _ext_format(name: str) -> str: + """Derive a format token from a filename extension. Returns 'unknown' if there's nothing.""" + ext = os.path.splitext(name)[1].lstrip(".") + return ext.lower() if ext else "unknown" + + +def _split_pipe(value: Optional[str]) -> List[str]: + if not value: + return [] + return [piece.strip() for piece in value.split("|") if piece.strip()] + + +def _resolve_sidecar(manifest_path: pathlib.Path, explicit: Optional[pathlib.Path]) -> pathlib.Path: + if explicit is not None: + if not explicit.exists(): + raise ValueError(f"Sidecar not found: {explicit}") + return explicit + + siblings = [ + manifest_path.with_suffix(".json"), + manifest_path.parent / "crate.json", + ] + for candidate in siblings: + if candidate.exists(): + return candidate + + raise ValueError( + "No sidecar found. Looked for " + + ", ".join(str(s) for s in siblings) + + ". Pass --sidecar to point at one explicitly." + ) + + +def _load_sidecar(path: pathlib.Path) -> Dict[str, Any]: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + missing = [field for field in SIDECAR_REQUIRED_FIELDS if not data.get(field)] + if missing: + raise ValueError( + f"Sidecar {path} is missing required field(s): {', '.join(missing)}" + ) + if not isinstance(data.get("authors"), list): + raise ValueError(f"Sidecar 'authors' must be a list (got {type(data.get('authors')).__name__}).") + return data + + +def _row_to_file_dict(row: Dict[str, str], sidecar: Dict[str, Any]) -> Dict[str, Any]: + """Translate one manifest row into a dict ready for ResearchData. + + The dict carries an extra '_type' key (`'dataset'` or `'software'`) so the + caller can route it into `ResearchData.files` or `ResearchData.software`. + The `_type` is sourced from the row's `type` column (case-insensitive), + falling back to extension-based auto-detection. + """ + name = (row.get("name") or "").strip() + if not name: + raise ValueError(f"Manifest row missing 'name': {row}") + + content_url = (row.get("contentUrl") or "").strip() + if not content_url: + raise ValueError(f"Manifest row '{name}' missing 'contentUrl'.") + + declared_type = (row.get("type") or "").strip().lower() + if declared_type in ("software", "dataset"): + row_type = declared_type + elif declared_type == "": + row_type = _auto_detect_type(name) + else: + raise ValueError( + f"Manifest row '{name}' has unknown type {declared_type!r}; " + f"expected 'dataset', 'software', or blank." + ) + + description = (row.get("description") or "").strip() or sidecar.get("description") or "" + fmt = (row.get("format") or "").strip() or _ext_format(name) + + # Size: prefer explicit byte count (we format it); else accept pre-formatted string; else nothing. + content_size: Optional[str] = None + size_bytes_raw = (row.get("size_bytes") or "").strip() + if size_bytes_raw: + try: + content_size = _human_size(int(size_bytes_raw)) + except ValueError: + raise ValueError(f"Manifest row '{name}' has non-integer size_bytes: {size_bytes_raw!r}") + elif (row.get("contentSize") or "").strip(): + content_size = row["contentSize"].strip() + + file_dict: Dict[str, Any] = { + "name": name, + "description": description, + "contentUrl": content_url, + "url": content_url, + "format": fmt, + "datePublished": (row.get("datePublished") or "").strip() or sidecar.get("publication_date"), + "version": (row.get("version") or "").strip() or "1.0", + } + + md5 = (row.get("md5") or "").strip() + if md5: + file_dict["md5"] = md5 + sha256 = (row.get("sha256") or "").strip() + if sha256: + file_dict["sha256"] = sha256 + if content_size: + file_dict["contentSize"] = content_size + # The Dataverse connector also stuffs raw bytes under 'size'; mirror that + # so anything downstream that reads 'size' keeps working. + if size_bytes_raw: + file_dict["size"] = int(size_bytes_raw) + + per_row_keywords = _split_pipe(row.get("keywords")) + if per_row_keywords: + file_dict["keywords"] = per_row_keywords + + group = (row.get("group") or "").strip() + if group: + # Reserved for future schema-inference grouping; stash it where it'll + # round-trip without breaking anything. + file_dict["group"] = group + + file_dict["_type"] = row_type + return file_dict + + +def _to_software_dict(record: Dict[str, Any]) -> Dict[str, Any]: + """Translate a file-dict (built for ResearchData.files) into the shape + ResearchData.software expects, which `to_rocrate` passes to GenerateSoftware. + + The only meaningful difference is `datePublished` → `dateModified`. + """ + software = dict(record) + if "datePublished" in software: + software["dateModified"] = software.pop("datePublished") + return software + + +class ManifestConnector: + """Reads a CSV manifest + JSON sidecar and produces ResearchData.""" + + def __init__(self, sidecar_path: Optional[pathlib.Path] = None): + self.sidecar_path = pathlib.Path(sidecar_path) if sidecar_path else None + + def fetch_data(self, manifest_path: str, include_files: bool = True) -> ResearchData: + manifest_p = pathlib.Path(manifest_path) + if not manifest_p.exists(): + raise ValueError(f"Manifest not found: {manifest_p}") + + sidecar_p = _resolve_sidecar(manifest_p, self.sidecar_path) + sidecar = _load_sidecar(sidecar_p) + + files_data: List[Dict[str, Any]] = [] + software_data: List[Dict[str, Any]] = [] + if include_files: + with manifest_p.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + missing_cols = [c for c in MANIFEST_REQUIRED_COLUMNS if c not in (reader.fieldnames or [])] + if missing_cols: + raise ValueError( + f"Manifest {manifest_p} is missing required column(s): {', '.join(missing_cols)}. " + f"Found columns: {reader.fieldnames}" + ) + for row in reader: + if not any((row.get(c) or "").strip() for c in row): + continue # skip blank lines + record = _row_to_file_dict(row, sidecar) + row_type = record.pop("_type") + if row_type == "software": + software_data.append(_to_software_dict(record)) + else: + files_data.append(record) + + publication_date = sidecar.get("publication_date") or datetime.utcnow().date().isoformat() + + metadata: Dict[str, Any] = {} + if sidecar.get("url"): + metadata["url"] = sidecar["url"] + if sidecar.get("associated_publication"): + metadata["contentUrl"] = sidecar["associated_publication"] + if sidecar.get("version"): + metadata["version"] = sidecar["version"] + + return ResearchData( + repository_name=sidecar.get("repository_name") or "Manifest", + project_id=sidecar.get("project_id") or sidecar.get("doi") or manifest_p.stem, + title=sidecar["name"], + description=sidecar["description"], + authors=list(sidecar["authors"]), + license=sidecar.get("license"), + keywords=list(sidecar.get("keywords") or []), + publication_date=publication_date, + doi=sidecar.get("doi"), + files=files_data, + software=software_data, + metadata=metadata, + ) diff --git a/src/fairscape_cli/data_fetcher/generic_data/research_data.py b/src/fairscape_cli/data_fetcher/generic_data/research_data.py index fc29888..6934c12 100644 --- a/src/fairscape_cli/data_fetcher/generic_data/research_data.py +++ b/src/fairscape_cli/data_fetcher/generic_data/research_data.py @@ -90,12 +90,14 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: } if file_info.get("md5"): dataset_params["md5"] = file_info["md5"] + if file_info.get("sha256"): + dataset_params["sha256"] = file_info["sha256"] if file_info.get("contentSize"): dataset_params["contentSize"] = file_info["contentSize"] - + dataset = GenerateDataset(**dataset_params) elements_to_append.append(dataset) - + for software_info in self.software: software_version = software_info.get("version", "0.1.0") @@ -105,8 +107,8 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: software_author = self.authors[0] if self.authors else "Unknown" - software_item = GenerateSoftware( - guid=None, + software_kwargs = dict( + guid=None, name=software_info.get("name", "Unnamed software"), author=software_author, dateModified=software_info.get("dateModified", datetime.now().isoformat()), @@ -114,10 +116,18 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: description=software_description, associatedPublication=self.doi or None, additionalDocumentation=software_info.get("documentation_url", None), - format=software_info.get("format", "application/zip"), + format=software_info.get("format", "application/zip"), usedByComputation=[], - contentUrl=software_info.get("contentUrl", software_info.get("url", "")) + contentUrl=software_info.get("contentUrl", software_info.get("url", "")), ) + if software_info.get("md5"): + software_kwargs["md5"] = software_info["md5"] + if software_info.get("sha256"): + software_kwargs["sha256"] = software_info["sha256"] + if software_info.get("contentSize"): + software_kwargs["contentSize"] = software_info["contentSize"] + + software_item = GenerateSoftware(**software_kwargs) elements_to_append.append(software_item) if elements_to_append: @@ -138,6 +148,10 @@ def from_repository(cls, repository_type: str, identifier: str, **kwargs) -> 'Re api_token=kwargs.get('token') ) return connector.fetch_data(identifier, include_files=kwargs.get('include_files', True)) + elif repository_type.lower() == 'manifest': + from fairscape_cli.data_fetcher.generic_data.connectors.manifest_connector import ManifestConnector + connector = ManifestConnector(sidecar_path=kwargs.get('sidecar_path')) + return connector.fetch_data(identifier, include_files=kwargs.get('include_files', True)) else: raise ValueError(f"Unsupported repository type: {repository_type}") diff --git a/src/fairscape_cli/datasheet_builder/__init__.py b/src/fairscape_cli/datasheet_builder/__init__.py index 97e0706..56635fb 100644 --- a/src/fairscape_cli/datasheet_builder/__init__.py +++ b/src/fairscape_cli/datasheet_builder/__init__.py @@ -2,6 +2,14 @@ Datasheet Builder module for RO-Crate metadata visualization and documentation. """ +from pathlib import Path + from fairscape_cli.datasheet_builder.rocrate.datasheet_generator import DatasheetGenerator -__all__ = ['DatasheetGenerator'] \ No newline at end of file + +def get_default_template_dir() -> Path: + """Return the Jinja template directory shipped with the package.""" + return Path(__file__).parent / 'templates' + + +__all__ = ['DatasheetGenerator', 'get_default_template_dir'] diff --git a/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py b/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py deleted file mode 100644 index 62842c8..0000000 --- a/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py +++ /dev/null @@ -1,242 +0,0 @@ -from typing import Dict, List, Optional, Any, Union, Set -import json -import pathlib -from pathlib import Path -import click - -class EvidenceNode: - def __init__(self, id: str, type: str): - self.id = id - self.type = type - # For Computation nodes - self.usedSoftware: Optional[List[str]] = None - self.usedDataset: Optional[List[str]] = None - self.usedSample: Optional[List[str]] = None - self.usedInstrument: Optional[List[str]] = None - # For Dataset/Sample/Instrument nodes - self.generatedBy: Optional[str] = None - -class EvidenceGraphJSON: - def __init__(self, guid: str, owner: str, description: str, name: str = "Evidence Graph"): - self.metadataType = "evi:EvidenceGraph" - self.guid = guid - self.owner = owner - self.description = description - self.name = name - self.graph = None - - def build_graph(self, node_id: str, json_data: Dict[str, Any]): - processed = set() - self.graph = self._build_graph_recursive(node_id, json_data, processed) - - def _build_graph_recursive(self, node_id: str, json_data: Dict[str, Any], processed: Set[str]) -> Dict: - if node_id in processed: - return {"@id": node_id} - - # Find node in json data graph - node = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == node_id: - node = entity - break - - if not node: - return {"@id": node_id} - - processed.add(node_id) - result = self._build_base_node(node) - - # Determine node type based on @type - node_type = None - if isinstance(node.get("@type"), list): - for type_entry in node["@type"]: - if "Dataset" in type_entry: - node_type = "Dataset" - break - elif "Computation" in type_entry: - node_type = "Computation" - break - elif "Sample" in type_entry: - node_type = "Sample" - break - elif "Instrument" in type_entry: - node_type = "Instrument" - break - elif "Experiment" in type_entry: - node_type = "Experiment" - break - elif isinstance(node.get("@type"), str): - type_str = node.get("@type") - if "Dataset" in type_str: - node_type = "Dataset" - elif "Computation" in type_str: - node_type = "Computation" - elif "Sample" in type_str: - node_type = "Sample" - elif "Instrument" in type_str: - node_type = "Instrument" - elif "Experiment" in type_str: - node_type = "Experiment" - - if node_type in ["Dataset", "Sample", "Instrument"]: - if "generatedBy" in node: - result["generatedBy"] = self._build_computation_node(node, json_data, processed) - elif node_type in ["Computation", "Experiment"]: - if "usedDataset" in node: - result["usedDataset"] = self._build_used_resources(node["usedDataset"], json_data, processed) - if "usedSoftware" in node: - result["usedSoftware"] = self._build_software_reference(node["usedSoftware"], json_data) - if "usedSample" in node: - result["usedSample"] = self._build_used_resources(node["usedSample"], json_data, processed) - if "usedInstrument" in node: - result["usedInstrument"] = self._build_used_resources(node["usedInstrument"], json_data, processed) - - return result - - def _build_base_node(self, node: Dict) -> Dict: - return { - "@id": node["@id"], - "@type": node.get("@type"), - "name": node.get("name"), - "description": node.get("description") - } - - def _build_computation_node(self, parent_node: Dict, json_data: Dict[str, Any], processed: Set[str]) -> Dict: - # If generatedBy is an empty list, don't add anything - if isinstance(parent_node["generatedBy"], list) and len(parent_node["generatedBy"]) == 0: - return {} - - comp_id = None - if isinstance(parent_node["generatedBy"], list): - if parent_node["generatedBy"] and isinstance(parent_node["generatedBy"][0], dict): - comp_id = parent_node["generatedBy"][0].get("@id") - elif isinstance(parent_node["generatedBy"], dict): - comp_id = parent_node["generatedBy"].get("@id") - - if not comp_id: - return {"@id": "unknown-computation"} - - comp = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == comp_id: - comp = entity - break - - if not comp: - return {"@id": comp_id} - - return self._build_graph_recursive(comp_id, json_data, processed) - - def _build_used_resources(self, used_resources: Union[Dict, List], json_data: Dict[str, Any], processed: Set[str]) -> List: - if isinstance(used_resources, dict): - resource_id = used_resources.get("@id") - if resource_id: - return [self._build_graph_recursive(resource_id, json_data, processed)] - return [] - - if isinstance(used_resources, list): - resources = [] - for resource in used_resources: - if isinstance(resource, dict) and resource.get("@id"): - resources.append(self._build_graph_recursive(resource.get("@id"), json_data, processed)) - return resources - - return [] - - def _build_software_reference(self, used_software: Union[Dict, List], json_data: Dict[str, Any]) -> Union[Dict, List[Dict]]: - if isinstance(used_software, list): - if not used_software: - return [] - - software_refs = [] - for sw in used_software: - if isinstance(sw, dict) and sw.get("@id"): - software_id = sw.get("@id") - software = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == software_id: - software = entity - break - - if software: - software_refs.append(self._build_base_node(software)) - else: - software_refs.append({"@id": software_id}) - - return software_refs - - elif isinstance(used_software, dict) and used_software.get("@id"): - software_id = used_software.get("@id") - software = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == software_id: - software = entity - break - - if software: - return self._build_base_node(software) - return {"@id": software_id} - - return {"@id": "unknown-software"} - - def to_dict(self): - return { - "@type": self.metadataType, - "@id": self.guid, - "owner": self.owner, - "description": self.description, - "name": self.name, - "@graph": self.graph - } - - def save_to_file(self, file_path: Union[str, pathlib.Path]): - with open(file_path, 'w') as f: - json.dump(self.to_dict(), f, indent=2) - -def generate_evidence_graph_from_rocrate( - rocrate_path: Union[str, pathlib.Path], - output_path: Optional[Union[str, pathlib.Path]] = None, - node_id: str = "" -) -> Dict[str, Any]: - """ - Generate an evidence graph from an RO-Crate JSON file for a specific node ID - - Args: - rocrate_path: Path to the RO-Crate metadata JSON file - output_path: Optional path to save the evidence graph JSON - node_id: ID of the node to start building the graph from - - Returns: - The generated evidence graph as a dictionary - """ - # Load RO-Crate data - with open(rocrate_path, 'r') as f: - rocrate_data = json.load(f) - - # Find the root entity - root_entity = None - for entity in rocrate_data.get("@graph", []): - if entity.get("@id") == node_id: - root_entity = entity - break - - if not root_entity: - raise ValueError(f"Could not find entity with ID {node_id} in RO-Crate") - - # Create evidence graph - graph_id = f"{node_id}-evidence-graph" - graph = EvidenceGraphJSON( - guid=graph_id, - owner=node_id, - description=f"Evidence graph for {root_entity.get('name', 'Unknown')}", - name=f"Evidence Graph - {root_entity.get('name', 'Unknown')}" - ) - - # Build the graph - graph.build_graph(node_id, rocrate_data) - - # Save to file if output path provided - if output_path: - graph.save_to_file(output_path) - - return graph.to_dict() diff --git a/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py b/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py index fd272db..1635ab9 100644 --- a/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py +++ b/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py @@ -1,33 +1,57 @@ -import json -import os +"""Generate a standalone, self-contained HTML visualization of an evidence graph. + +The HTML shell lives in templates/evidence_graph/evidence_graph.html.j2 and the +application JavaScript in evidence_graph.js next to it. React, ReactDOM and +dagre are vendored under templates/evidence_graph/vendor/ and inlined into the +output, so the generated file is a single offline-shareable HTML document with +no CDN dependency. +""" import argparse +import json +import logging from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +logger = logging.getLogger(__name__) + +TEMPLATE_DIR = Path(__file__).parent.parent / 'templates' / 'evidence_graph' +VENDOR_FILES = ( + 'vendor/react.production.min.js', + 'vendor/react-dom.production.min.js', + 'vendor/dagre.min.js', +) + + +def _inline_script_safe(js: str) -> str: + """Make JS safe to inline in a ' breakout).""" + return js.replace(' - - - - - Evidence Graph Visualization - - - - - - - - - - - - """ + vendor_js = ';\n'.join( + (TEMPLATE_DIR / name).read_text(encoding='utf-8') for name in VENDOR_FILES + ) + app_js = (TEMPLATE_DIR / 'evidence_graph.js').read_text(encoding='utf-8') + + # Escaping '<' keeps the JSON valid JS while preventing a '' (or + # ' ☰ - + ↑ Navigation Datasheet Summary - Dataset Details + Release Overview AI Ready Information Composition @@ -717,24 +1071,36 @@ {% endif %} - {{ title }} - Version: {{ version }} + + {% if version %} + Version {{ version }} + {% endif %} + {% if doi %} + DOI ↗ + {% endif %} + {% if license_value %} + License ↗ + {% endif %} + {% if content_size %} + {{ content_size }} + {% endif %} + {% if release_date %} + Released {{ release_date }} + {% endif %} + {{ summary_section | safe }} - {{ overview_section | safe }} - {{ use_cases_section | safe }} - Composition (Datasets {{ subcrate_count }}) @@ -750,17 +1116,168 @@ Composition (Datasets {{ subcrate_count }}) >). -
. + strong_license = access_card_body.find( + 'strong', + string=lambda text: text and text.strip().startswith('License (for files):') + ) + license_p = strong_license.find_parent('p') if strong_license else None if license_p: - license_link = license_p.find('a') - if license_link and license_link.get('href'): + license_link = license_p.find('a', href=True) + if license_link: metadata['extracted_license'] = license_link['href'] return metadata + def _fetch_checksums(self) -> Tuple[Dict[str, str], Optional[str]]: + """Fetch and parse PhysioNet's checksum manifest for this dataset. + + Returns a (mapping, algorithm) tuple where mapping is {relative_path: hash} + and algorithm is one of "sha256", "md5", or None when no manifest is found. + Each manifest line is ` ` (single space separator). + """ + candidates = [ + ("SHA256SUMS.txt", "sha256"), + ("SHA256SUMS", "sha256"), + ("MD5SUMS.txt", "md5"), + ("MD5SUMS", "md5"), + ] + for filename, algorithm in candidates: + url = f"{self.base_file_download_url}{filename}" + try: + response = requests.get(url, timeout=30) + except requests.exceptions.RequestException as e: + print(f"Warning: Failed to fetch checksum file {url}: {e}", file=sys.stderr) + continue + if response.status_code != 200: + continue + + mapping: Dict[str, str] = {} + for line in response.text.splitlines(): + line = line.strip() + if not line: + continue + # Format: " " — split on first whitespace only, + # since paths may contain spaces. + parts = line.split(None, 1) + if len(parts) != 2: + continue + digest, rel_path = parts[0].strip(), parts[1].strip() + mapping[rel_path] = digest + if mapping: + print(f"Loaded {len(mapping)} {algorithm} checksums from {filename}", file=sys.stderr) + return mapping, algorithm + return {}, None + def _process_file_table_rows(self, table_body_soup: BeautifulSoup, current_subdir_path: str, all_files_list: List[Dict[str, Any]], subdirs_to_explore: List[str], crate_default_date_published: str): rows_to_process = list(table_body_soup.select('tr')) for row in rows_to_process: + row_classes = row.get('class', []) + + # PhysioNet renders a "Parent Directory" navigation row (class="parentdir") + # on every subdirectory page. It has no class="subdir", so without this skip + # it would be misclassified as a file and added as a Dataset per traversed dir. + if 'parentdir' in row_classes: + continue + name_cell_a_tag = row.select_one('td a') if not name_cell_a_tag: continue name = name_cell_a_tag.get_text(strip=True) - - if name in ['.', '..', '']: + + if name in ['.', '..', '', 'Parent Directory']: continue - is_directory = 'subdir' in row.get('class', []) - + is_directory = 'subdir' in row_classes or name_cell_a_tag.has_attr('data-dfp-dir') + item_relative_path: str = "" if is_directory: item_relative_path = name_cell_a_tag.get('data-dfp-dir') @@ -346,9 +412,9 @@ def _process_file_table_rows(self, table_body_soup: BeautifulSoup, current_subdi if item_relative_path not in subdirs_to_explore and item_relative_path != current_subdir_path : print(f"DEBUG: Found directory: {name} (data-dfp-dir: {item_relative_path}) in {current_subdir_path or ''}. Adding to queue.", file=sys.stderr) - subdirs_to_explore.append(item_relative_path) - else: - item_relative_path = os.path.join(current_subdir_path, name).replace('\\', '/') + subdirs_to_explore.append(item_relative_path) + else: + item_relative_path = os.path.join(current_subdir_path, name).replace('\\', '/') if not is_directory: @@ -532,6 +598,8 @@ def to_rocrate( print(f"Total files found after exploring subdirectories: {len(all_file_entries)}", file=sys.stderr) + checksum_map, checksum_algorithm = self._fetch_checksums() + crate_elements_to_add = [] for item_data in all_file_entries: item_path_guid_segment = item_data['relative_path'].replace('/', '_').replace('.', '_').replace(' ', '_').lower() @@ -554,6 +622,10 @@ def to_rocrate( if 'contentSize' in item_data: dataset_params['contentSize'] = item_data['contentSize'] if 'contentUrl' in item_data: dataset_params['contentUrl'] = item_data['contentUrl'] + digest = checksum_map.get(item_data['relative_path']) + if digest and checksum_algorithm: + dataset_params[checksum_algorithm] = digest + try: dataset_entity = GenerateDataset(**dataset_params) crate_elements_to_add.append(dataset_entity) diff --git a/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py b/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py index 605cd54..8710b54 100644 --- a/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py +++ b/src/fairscape_cli/data_fetcher/generic_data/connectors/dataverse_connector.py @@ -213,7 +213,9 @@ def fetch_data(self, doi: str, include_files: bool = True) -> ResearchData: if is_software: software_data.append({ **item_common_data, - "version": latest_version_info.get("versionNumber", "1.0"), # Use dataset version for software + # Dataverse returns versionNumber as a numeric type; the + # Software pydantic model requires a string. + "version": str(latest_version_info.get("versionNumber", "1.0")), "documentation_url": dataset_landing_url }) else: @@ -249,7 +251,7 @@ def fetch_data(self, doi: str, include_files: bool = True) -> ResearchData: "url": f"{self.server_url}/dataset.xhtml?persistentId=doi:{doi}", "contentUrl": f"{self.server_url}/dataset.xhtml?persistentId=doi:{doi}", "publisher": dataset_metadata_full.get("publisher", ""), - "version": latest_version_info.get("versionNumber"), + "version": str(latest_version_info.get("versionNumber")) if latest_version_info.get("versionNumber") is not None else None, } ) diff --git a/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py b/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py new file mode 100644 index 0000000..0900534 --- /dev/null +++ b/src/fairscape_cli/data_fetcher/generic_data/connectors/manifest_connector.py @@ -0,0 +1,244 @@ +"""Manifest connector. + +Reads a CSV manifest + JSON sidecar describing a published dataset that +doesn't have (or doesn't need) a dedicated repository connector, and returns a +ResearchData instance for `ResearchData.to_rocrate()` to write out. +""" +import csv +import json +import os +import pathlib +from datetime import datetime +from typing import Any, Dict, List, Optional + +from fairscape_cli.data_fetcher.generic_data.research_data import ResearchData + + +SIDECAR_REQUIRED_FIELDS = ("name", "description", "authors") +MANIFEST_REQUIRED_COLUMNS = ("name", "description", "contentUrl") +SIZE_UNITS = ("B", "KB", "MB", "GB", "TB", "PB") + +# Extensions that should default to Software entities rather than Datasets. +# Mirrors fairscape_cli.data_fetcher.generic_data.connectors.dataverse_connector +# with a few additional script suffixes. +SOFTWARE_EXTENSIONS = { + ".py", ".r", ".sh", ".bash", ".ipynb", ".jl", ".m", + ".exe", ".java", ".cpp", ".js", ".jsx", ".ts", ".tsx", ".css", +} + + +def _auto_detect_type(name: str) -> str: + """Return 'software' or 'dataset' based on file extension.""" + ext = os.path.splitext(name)[1].lower() + return "software" if ext in SOFTWARE_EXTENSIONS else "dataset" + + +def _human_size(n: int) -> str: + """Format byte count as a short human-readable string matching the other connectors.""" + if n is None: + return "" + size = float(n) + i = 0 + while size >= 1024 and i < len(SIZE_UNITS) - 1: + size /= 1024.0 + i += 1 + if i == 0: + return f"{int(size)} {SIZE_UNITS[i]}" + return f"{size:.1f} {SIZE_UNITS[i]}" + + +def _ext_format(name: str) -> str: + """Derive a format token from a filename extension. Returns 'unknown' if there's nothing.""" + ext = os.path.splitext(name)[1].lstrip(".") + return ext.lower() if ext else "unknown" + + +def _split_pipe(value: Optional[str]) -> List[str]: + if not value: + return [] + return [piece.strip() for piece in value.split("|") if piece.strip()] + + +def _resolve_sidecar(manifest_path: pathlib.Path, explicit: Optional[pathlib.Path]) -> pathlib.Path: + if explicit is not None: + if not explicit.exists(): + raise ValueError(f"Sidecar not found: {explicit}") + return explicit + + siblings = [ + manifest_path.with_suffix(".json"), + manifest_path.parent / "crate.json", + ] + for candidate in siblings: + if candidate.exists(): + return candidate + + raise ValueError( + "No sidecar found. Looked for " + + ", ".join(str(s) for s in siblings) + + ". Pass --sidecar to point at one explicitly." + ) + + +def _load_sidecar(path: pathlib.Path) -> Dict[str, Any]: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + missing = [field for field in SIDECAR_REQUIRED_FIELDS if not data.get(field)] + if missing: + raise ValueError( + f"Sidecar {path} is missing required field(s): {', '.join(missing)}" + ) + if not isinstance(data.get("authors"), list): + raise ValueError(f"Sidecar 'authors' must be a list (got {type(data.get('authors')).__name__}).") + return data + + +def _row_to_file_dict(row: Dict[str, str], sidecar: Dict[str, Any]) -> Dict[str, Any]: + """Translate one manifest row into a dict ready for ResearchData. + + The dict carries an extra '_type' key (`'dataset'` or `'software'`) so the + caller can route it into `ResearchData.files` or `ResearchData.software`. + The `_type` is sourced from the row's `type` column (case-insensitive), + falling back to extension-based auto-detection. + """ + name = (row.get("name") or "").strip() + if not name: + raise ValueError(f"Manifest row missing 'name': {row}") + + content_url = (row.get("contentUrl") or "").strip() + if not content_url: + raise ValueError(f"Manifest row '{name}' missing 'contentUrl'.") + + declared_type = (row.get("type") or "").strip().lower() + if declared_type in ("software", "dataset"): + row_type = declared_type + elif declared_type == "": + row_type = _auto_detect_type(name) + else: + raise ValueError( + f"Manifest row '{name}' has unknown type {declared_type!r}; " + f"expected 'dataset', 'software', or blank." + ) + + description = (row.get("description") or "").strip() or sidecar.get("description") or "" + fmt = (row.get("format") or "").strip() or _ext_format(name) + + # Size: prefer explicit byte count (we format it); else accept pre-formatted string; else nothing. + content_size: Optional[str] = None + size_bytes_raw = (row.get("size_bytes") or "").strip() + if size_bytes_raw: + try: + content_size = _human_size(int(size_bytes_raw)) + except ValueError: + raise ValueError(f"Manifest row '{name}' has non-integer size_bytes: {size_bytes_raw!r}") + elif (row.get("contentSize") or "").strip(): + content_size = row["contentSize"].strip() + + file_dict: Dict[str, Any] = { + "name": name, + "description": description, + "contentUrl": content_url, + "url": content_url, + "format": fmt, + "datePublished": (row.get("datePublished") or "").strip() or sidecar.get("publication_date"), + "version": (row.get("version") or "").strip() or "1.0", + } + + md5 = (row.get("md5") or "").strip() + if md5: + file_dict["md5"] = md5 + sha256 = (row.get("sha256") or "").strip() + if sha256: + file_dict["sha256"] = sha256 + if content_size: + file_dict["contentSize"] = content_size + # The Dataverse connector also stuffs raw bytes under 'size'; mirror that + # so anything downstream that reads 'size' keeps working. + if size_bytes_raw: + file_dict["size"] = int(size_bytes_raw) + + per_row_keywords = _split_pipe(row.get("keywords")) + if per_row_keywords: + file_dict["keywords"] = per_row_keywords + + group = (row.get("group") or "").strip() + if group: + # Reserved for future schema-inference grouping; stash it where it'll + # round-trip without breaking anything. + file_dict["group"] = group + + file_dict["_type"] = row_type + return file_dict + + +def _to_software_dict(record: Dict[str, Any]) -> Dict[str, Any]: + """Translate a file-dict (built for ResearchData.files) into the shape + ResearchData.software expects, which `to_rocrate` passes to GenerateSoftware. + + The only meaningful difference is `datePublished` → `dateModified`. + """ + software = dict(record) + if "datePublished" in software: + software["dateModified"] = software.pop("datePublished") + return software + + +class ManifestConnector: + """Reads a CSV manifest + JSON sidecar and produces ResearchData.""" + + def __init__(self, sidecar_path: Optional[pathlib.Path] = None): + self.sidecar_path = pathlib.Path(sidecar_path) if sidecar_path else None + + def fetch_data(self, manifest_path: str, include_files: bool = True) -> ResearchData: + manifest_p = pathlib.Path(manifest_path) + if not manifest_p.exists(): + raise ValueError(f"Manifest not found: {manifest_p}") + + sidecar_p = _resolve_sidecar(manifest_p, self.sidecar_path) + sidecar = _load_sidecar(sidecar_p) + + files_data: List[Dict[str, Any]] = [] + software_data: List[Dict[str, Any]] = [] + if include_files: + with manifest_p.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + missing_cols = [c for c in MANIFEST_REQUIRED_COLUMNS if c not in (reader.fieldnames or [])] + if missing_cols: + raise ValueError( + f"Manifest {manifest_p} is missing required column(s): {', '.join(missing_cols)}. " + f"Found columns: {reader.fieldnames}" + ) + for row in reader: + if not any((row.get(c) or "").strip() for c in row): + continue # skip blank lines + record = _row_to_file_dict(row, sidecar) + row_type = record.pop("_type") + if row_type == "software": + software_data.append(_to_software_dict(record)) + else: + files_data.append(record) + + publication_date = sidecar.get("publication_date") or datetime.utcnow().date().isoformat() + + metadata: Dict[str, Any] = {} + if sidecar.get("url"): + metadata["url"] = sidecar["url"] + if sidecar.get("associated_publication"): + metadata["contentUrl"] = sidecar["associated_publication"] + if sidecar.get("version"): + metadata["version"] = sidecar["version"] + + return ResearchData( + repository_name=sidecar.get("repository_name") or "Manifest", + project_id=sidecar.get("project_id") or sidecar.get("doi") or manifest_p.stem, + title=sidecar["name"], + description=sidecar["description"], + authors=list(sidecar["authors"]), + license=sidecar.get("license"), + keywords=list(sidecar.get("keywords") or []), + publication_date=publication_date, + doi=sidecar.get("doi"), + files=files_data, + software=software_data, + metadata=metadata, + ) diff --git a/src/fairscape_cli/data_fetcher/generic_data/research_data.py b/src/fairscape_cli/data_fetcher/generic_data/research_data.py index fc29888..6934c12 100644 --- a/src/fairscape_cli/data_fetcher/generic_data/research_data.py +++ b/src/fairscape_cli/data_fetcher/generic_data/research_data.py @@ -90,12 +90,14 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: } if file_info.get("md5"): dataset_params["md5"] = file_info["md5"] + if file_info.get("sha256"): + dataset_params["sha256"] = file_info["sha256"] if file_info.get("contentSize"): dataset_params["contentSize"] = file_info["contentSize"] - + dataset = GenerateDataset(**dataset_params) elements_to_append.append(dataset) - + for software_info in self.software: software_version = software_info.get("version", "0.1.0") @@ -105,8 +107,8 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: software_author = self.authors[0] if self.authors else "Unknown" - software_item = GenerateSoftware( - guid=None, + software_kwargs = dict( + guid=None, name=software_info.get("name", "Unnamed software"), author=software_author, dateModified=software_info.get("dateModified", datetime.now().isoformat()), @@ -114,10 +116,18 @@ def to_rocrate(self, output_dir: str, **kwargs) -> str: description=software_description, associatedPublication=self.doi or None, additionalDocumentation=software_info.get("documentation_url", None), - format=software_info.get("format", "application/zip"), + format=software_info.get("format", "application/zip"), usedByComputation=[], - contentUrl=software_info.get("contentUrl", software_info.get("url", "")) + contentUrl=software_info.get("contentUrl", software_info.get("url", "")), ) + if software_info.get("md5"): + software_kwargs["md5"] = software_info["md5"] + if software_info.get("sha256"): + software_kwargs["sha256"] = software_info["sha256"] + if software_info.get("contentSize"): + software_kwargs["contentSize"] = software_info["contentSize"] + + software_item = GenerateSoftware(**software_kwargs) elements_to_append.append(software_item) if elements_to_append: @@ -138,6 +148,10 @@ def from_repository(cls, repository_type: str, identifier: str, **kwargs) -> 'Re api_token=kwargs.get('token') ) return connector.fetch_data(identifier, include_files=kwargs.get('include_files', True)) + elif repository_type.lower() == 'manifest': + from fairscape_cli.data_fetcher.generic_data.connectors.manifest_connector import ManifestConnector + connector = ManifestConnector(sidecar_path=kwargs.get('sidecar_path')) + return connector.fetch_data(identifier, include_files=kwargs.get('include_files', True)) else: raise ValueError(f"Unsupported repository type: {repository_type}") diff --git a/src/fairscape_cli/datasheet_builder/__init__.py b/src/fairscape_cli/datasheet_builder/__init__.py index 97e0706..56635fb 100644 --- a/src/fairscape_cli/datasheet_builder/__init__.py +++ b/src/fairscape_cli/datasheet_builder/__init__.py @@ -2,6 +2,14 @@ Datasheet Builder module for RO-Crate metadata visualization and documentation. """ +from pathlib import Path + from fairscape_cli.datasheet_builder.rocrate.datasheet_generator import DatasheetGenerator -__all__ = ['DatasheetGenerator'] \ No newline at end of file + +def get_default_template_dir() -> Path: + """Return the Jinja template directory shipped with the package.""" + return Path(__file__).parent / 'templates' + + +__all__ = ['DatasheetGenerator', 'get_default_template_dir'] diff --git a/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py b/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py deleted file mode 100644 index 62842c8..0000000 --- a/src/fairscape_cli/datasheet_builder/evidence_graph/graph_builder.py +++ /dev/null @@ -1,242 +0,0 @@ -from typing import Dict, List, Optional, Any, Union, Set -import json -import pathlib -from pathlib import Path -import click - -class EvidenceNode: - def __init__(self, id: str, type: str): - self.id = id - self.type = type - # For Computation nodes - self.usedSoftware: Optional[List[str]] = None - self.usedDataset: Optional[List[str]] = None - self.usedSample: Optional[List[str]] = None - self.usedInstrument: Optional[List[str]] = None - # For Dataset/Sample/Instrument nodes - self.generatedBy: Optional[str] = None - -class EvidenceGraphJSON: - def __init__(self, guid: str, owner: str, description: str, name: str = "Evidence Graph"): - self.metadataType = "evi:EvidenceGraph" - self.guid = guid - self.owner = owner - self.description = description - self.name = name - self.graph = None - - def build_graph(self, node_id: str, json_data: Dict[str, Any]): - processed = set() - self.graph = self._build_graph_recursive(node_id, json_data, processed) - - def _build_graph_recursive(self, node_id: str, json_data: Dict[str, Any], processed: Set[str]) -> Dict: - if node_id in processed: - return {"@id": node_id} - - # Find node in json data graph - node = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == node_id: - node = entity - break - - if not node: - return {"@id": node_id} - - processed.add(node_id) - result = self._build_base_node(node) - - # Determine node type based on @type - node_type = None - if isinstance(node.get("@type"), list): - for type_entry in node["@type"]: - if "Dataset" in type_entry: - node_type = "Dataset" - break - elif "Computation" in type_entry: - node_type = "Computation" - break - elif "Sample" in type_entry: - node_type = "Sample" - break - elif "Instrument" in type_entry: - node_type = "Instrument" - break - elif "Experiment" in type_entry: - node_type = "Experiment" - break - elif isinstance(node.get("@type"), str): - type_str = node.get("@type") - if "Dataset" in type_str: - node_type = "Dataset" - elif "Computation" in type_str: - node_type = "Computation" - elif "Sample" in type_str: - node_type = "Sample" - elif "Instrument" in type_str: - node_type = "Instrument" - elif "Experiment" in type_str: - node_type = "Experiment" - - if node_type in ["Dataset", "Sample", "Instrument"]: - if "generatedBy" in node: - result["generatedBy"] = self._build_computation_node(node, json_data, processed) - elif node_type in ["Computation", "Experiment"]: - if "usedDataset" in node: - result["usedDataset"] = self._build_used_resources(node["usedDataset"], json_data, processed) - if "usedSoftware" in node: - result["usedSoftware"] = self._build_software_reference(node["usedSoftware"], json_data) - if "usedSample" in node: - result["usedSample"] = self._build_used_resources(node["usedSample"], json_data, processed) - if "usedInstrument" in node: - result["usedInstrument"] = self._build_used_resources(node["usedInstrument"], json_data, processed) - - return result - - def _build_base_node(self, node: Dict) -> Dict: - return { - "@id": node["@id"], - "@type": node.get("@type"), - "name": node.get("name"), - "description": node.get("description") - } - - def _build_computation_node(self, parent_node: Dict, json_data: Dict[str, Any], processed: Set[str]) -> Dict: - # If generatedBy is an empty list, don't add anything - if isinstance(parent_node["generatedBy"], list) and len(parent_node["generatedBy"]) == 0: - return {} - - comp_id = None - if isinstance(parent_node["generatedBy"], list): - if parent_node["generatedBy"] and isinstance(parent_node["generatedBy"][0], dict): - comp_id = parent_node["generatedBy"][0].get("@id") - elif isinstance(parent_node["generatedBy"], dict): - comp_id = parent_node["generatedBy"].get("@id") - - if not comp_id: - return {"@id": "unknown-computation"} - - comp = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == comp_id: - comp = entity - break - - if not comp: - return {"@id": comp_id} - - return self._build_graph_recursive(comp_id, json_data, processed) - - def _build_used_resources(self, used_resources: Union[Dict, List], json_data: Dict[str, Any], processed: Set[str]) -> List: - if isinstance(used_resources, dict): - resource_id = used_resources.get("@id") - if resource_id: - return [self._build_graph_recursive(resource_id, json_data, processed)] - return [] - - if isinstance(used_resources, list): - resources = [] - for resource in used_resources: - if isinstance(resource, dict) and resource.get("@id"): - resources.append(self._build_graph_recursive(resource.get("@id"), json_data, processed)) - return resources - - return [] - - def _build_software_reference(self, used_software: Union[Dict, List], json_data: Dict[str, Any]) -> Union[Dict, List[Dict]]: - if isinstance(used_software, list): - if not used_software: - return [] - - software_refs = [] - for sw in used_software: - if isinstance(sw, dict) and sw.get("@id"): - software_id = sw.get("@id") - software = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == software_id: - software = entity - break - - if software: - software_refs.append(self._build_base_node(software)) - else: - software_refs.append({"@id": software_id}) - - return software_refs - - elif isinstance(used_software, dict) and used_software.get("@id"): - software_id = used_software.get("@id") - software = None - for entity in json_data.get("@graph", []): - if entity.get("@id") == software_id: - software = entity - break - - if software: - return self._build_base_node(software) - return {"@id": software_id} - - return {"@id": "unknown-software"} - - def to_dict(self): - return { - "@type": self.metadataType, - "@id": self.guid, - "owner": self.owner, - "description": self.description, - "name": self.name, - "@graph": self.graph - } - - def save_to_file(self, file_path: Union[str, pathlib.Path]): - with open(file_path, 'w') as f: - json.dump(self.to_dict(), f, indent=2) - -def generate_evidence_graph_from_rocrate( - rocrate_path: Union[str, pathlib.Path], - output_path: Optional[Union[str, pathlib.Path]] = None, - node_id: str = "" -) -> Dict[str, Any]: - """ - Generate an evidence graph from an RO-Crate JSON file for a specific node ID - - Args: - rocrate_path: Path to the RO-Crate metadata JSON file - output_path: Optional path to save the evidence graph JSON - node_id: ID of the node to start building the graph from - - Returns: - The generated evidence graph as a dictionary - """ - # Load RO-Crate data - with open(rocrate_path, 'r') as f: - rocrate_data = json.load(f) - - # Find the root entity - root_entity = None - for entity in rocrate_data.get("@graph", []): - if entity.get("@id") == node_id: - root_entity = entity - break - - if not root_entity: - raise ValueError(f"Could not find entity with ID {node_id} in RO-Crate") - - # Create evidence graph - graph_id = f"{node_id}-evidence-graph" - graph = EvidenceGraphJSON( - guid=graph_id, - owner=node_id, - description=f"Evidence graph for {root_entity.get('name', 'Unknown')}", - name=f"Evidence Graph - {root_entity.get('name', 'Unknown')}" - ) - - # Build the graph - graph.build_graph(node_id, rocrate_data) - - # Save to file if output path provided - if output_path: - graph.save_to_file(output_path) - - return graph.to_dict() diff --git a/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py b/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py index fd272db..1635ab9 100644 --- a/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py +++ b/src/fairscape_cli/datasheet_builder/evidence_graph/html_builder.py @@ -1,33 +1,57 @@ -import json -import os +"""Generate a standalone, self-contained HTML visualization of an evidence graph. + +The HTML shell lives in templates/evidence_graph/evidence_graph.html.j2 and the +application JavaScript in evidence_graph.js next to it. React, ReactDOM and +dagre are vendored under templates/evidence_graph/vendor/ and inlined into the +output, so the generated file is a single offline-shareable HTML document with +no CDN dependency. +""" import argparse +import json +import logging from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +logger = logging.getLogger(__name__) + +TEMPLATE_DIR = Path(__file__).parent.parent / 'templates' / 'evidence_graph' +VENDOR_FILES = ( + 'vendor/react.production.min.js', + 'vendor/react-dom.production.min.js', + 'vendor/dagre.min.js', +) + + +def _inline_script_safe(js: str) -> str: + """Make JS safe to inline in a ' breakout).""" + return js.replace(' - - - - - Evidence Graph Visualization - - - - - - - - - - - - """ + vendor_js = ';\n'.join( + (TEMPLATE_DIR / name).read_text(encoding='utf-8') for name in VENDOR_FILES + ) + app_js = (TEMPLATE_DIR / 'evidence_graph.js').read_text(encoding='utf-8') + + # Escaping '<' keeps the JSON valid JS while preventing a '' (or + # ' ☰ - + ↑ Navigation Datasheet Summary - Dataset Details + Release Overview AI Ready Information Composition @@ -717,24 +1071,36 @@ {% endif %} - {{ title }} - Version: {{ version }} + + {% if version %} + Version {{ version }} + {% endif %} + {% if doi %} + DOI ↗ + {% endif %} + {% if license_value %} + License ↗ + {% endif %} + {% if content_size %} + {{ content_size }} + {% endif %} + {% if release_date %} + Released {{ release_date }} + {% endif %} + {{ summary_section | safe }} - {{ overview_section | safe }} - {{ use_cases_section | safe }} - Composition (Datasets {{ subcrate_count }}) @@ -750,17 +1116,168 @@ Composition (Datasets {{ subcrate_count }}) >). -