diff --git a/.gitignore b/.gitignore index 9702592..acf39fb 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,8 @@ simdb-coverage-report src/simdb/_version.py *.egg-info *.egg -*.whl \ No newline at end of file +*.whl +.simdb-instances/ +_study +.serena +myenv diff --git a/deploy/nginx.dev.conf.template b/deploy/nginx.dev.conf.template index 20dedc3..608b83a 100644 --- a/deploy/nginx.dev.conf.template +++ b/deploy/nginx.dev.conf.template @@ -13,8 +13,13 @@ server { return 200 '{"deployment":"${SIMDB_DEPLOYMENT}","git_branch":"${SIMDB_GIT_BRANCH}","git_commit":"${SIMDB_GIT_COMMIT}","deployed_at":"${SIMDB_DEPLOYED_AT}"}'; } + location ~ ^/(v1|v1\.1|v1\.2)/$ { + default_type application/json; + return 200 '{"api":"simdb","api_version":"$1","server_version":"0.0.0","endpoints":["$scheme://$http_host/$1/simulations","$scheme://$http_host/$1/files","$scheme://$http_host/$1/validation_schema","$scheme://$http_host/$1/metadata","$scheme://$http_host/$1/upload_options"],"documentation":"$scheme://$http_host/$1/docs"}'; + } + location / { - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/pyproject.toml b/pyproject.toml index 3e93cfb..fbde57d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "numpy>=1.14", "pydantic>=2.10.6", "python-dateutil>=2.6", + "plotext==5.3.2", "pyyaml>=3.13", "requests>=2.27.0", "semantic-version>=2.8", diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 5b8045a..ecf2f0c 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -18,7 +18,12 @@ from simdb.validation import ValidationError, Validator from . import check_meta_args, pass_config -from .utils import print_simulations +from .utils import ( + is_numeric_1d, + print_quantity, + print_simulations, + show_quantity_textual_plot, +) from .validators import validate_non_negative @@ -375,6 +380,69 @@ def simulation_query( ) +@simulation.command("data", cls=n_required_args_adaptor(2)) +@pass_config +@click.argument("remote", required=False) +@click.argument("sim_id") +@click.argument("ids_path") +@click.option("--username", help="Username used to authenticate with the remote.") +@click.option("--password", help="Password used to authenticate with the remote.") +def simulation_data( + config: Config, + remote: Optional[str], + sim_id: str, + ids_path: str, + username: Optional[str], + password: Optional[str], +): + """Fetch IDS field data for simulation SIM_ID (UUID or alias) from REMOTE. + + \b + IDS_PATH format: + ids_name[:]/path/to/field + + \b + Examples: + simdb sim data iter 4dd781b... profiles_1d[0]/grid/rho_tor_norm + simdb sim data 4dd781b... equilibrium:0/time_slice[0]/profiles_1d/psi + """ + api = RemoteAPI(remote, username, password, config) + + try: + result = api.get_simulation_data(sim_id, ids_path) + except Exception as err: + raise click.ClickException(str(err)) from err + + click.echo(f"simulation : {result['simulation']}") + click.echo(f"path : {result['path']} (occurrence {result['occurrence']})") + + coordinates = result.get("coordinates") or [] + plot_coordinate = next( + ( + coord + for coord in coordinates + if isinstance(coord.get("data"), list) + and isinstance(result["field"].get("data"), list) + and len(coord["data"]) == len(result["field"]["data"]) + ), + None, + ) + field_is_1d = is_numeric_1d(result["field"].get("data")) + if field_is_1d: + show_quantity_textual_plot( + result["field"], label="field", x_quantity=plot_coordinate + ) + else: + print_quantity(result["field"], label="field") + + if config.verbose and coordinates: + for coord in coordinates: + if field_is_1d and is_numeric_1d(coord.get("data")): + continue + if isinstance(coord.get("data"), list): + print_quantity(coord, label=f"coord {coord['name']}", show_stats=False) + + @simulation.command("validate", cls=n_required_args_adaptor(1)) @pass_config @click.argument("remote", required=False) diff --git a/src/simdb/cli/commands/utils.py b/src/simdb/cli/commands/utils.py index ab2c919..8f87bd3 100644 --- a/src/simdb/cli/commands/utils.py +++ b/src/simdb/cli/commands/utils.py @@ -1,7 +1,12 @@ from collections import OrderedDict -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, TypeVar +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, TypeVar import click +import plotext +from rich.console import Console, Group +from rich.panel import Panel +from rich.table import Table +from rich.text import Text if TYPE_CHECKING: # Only importing these for type checking and documentation generation in order to @@ -10,6 +15,230 @@ else: Config = TypeVar("Config") +_RICH_CONSOLE = Console() + + +def _get_shape(data: Any) -> Tuple[int, ...]: + """Recursively compute shape of a nested list""" + if not isinstance(data, list): + return () + if not data: + return (0,) + return (len(data), *_get_shape(data[0])) + + +def _fmt_val(v: Any) -> str: + if isinstance(v, float): + return f"{v:.6g}" + return str(v) + + +def _fmt_row(row: list) -> str: + """Format a 1-D list with numpy-style head/tail truncation.""" + if len(row) <= 8: + return " ".join(_fmt_val(v) for v in row) + head = " ".join(_fmt_val(v) for v in row[:3]) + tail = " ".join(_fmt_val(v) for v in row[-3:]) + return f"{head} ... {tail}" + + +def _is_numeric(v: Any) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def is_numeric_1d(data: Any) -> bool: + return isinstance(data, list) and bool(data) and all(_is_numeric(v) for v in data) + + +def _quantity_axis_label(q: dict, fallback: str = "") -> str: + name = q.get("name") or fallback + units = q.get("units") or "-" + label = str(name).rsplit("/", 1)[-1] or str(name) + return f"{label} [{units}]" + + +def _build_array_body(data: list, shape: Tuple[int, ...]) -> str: + """Build string for 1-D or 2-D arrays.""" + if len(shape) == 1: + return f"[{_fmt_row(data)}]" + + if len(shape) == 2: + if len(data) <= 8: + rows = data + lines = [f" [{_fmt_row(row)}]" for row in rows] + else: + lines = [f" [{_fmt_row(row)}]" for row in data[:3]] + lines.append(" ...") + lines += [f" [{_fmt_row(row)}]" for row in data[-3:]] + formatted_lines = "\n".join(lines) + return f"[\n{formatted_lines}\n]" + + return f"<{len(shape)}-D array, shape {shape}>" + + +def _iter_numeric(data: Any) -> Iterable[float]: + """Yield all numeric leaf values from a nested list, skipping None.""" + if isinstance(data, list): + for item in data: + yield from _iter_numeric(item) + elif _is_numeric(data): + yield float(data) + + +def _compute_stats(data: Any) -> Optional[Dict[str, float]]: + """Return basic statistics for numeric data, or None if not applicable.""" + values = list(_iter_numeric(data)) + if len(values) < 2: + return None + n = len(values) + vmin = min(values) + vmax = max(values) + mean = sum(values) / n + std = (sum((x - mean) ** 2 for x in values) / n) ** 0.5 + sorted_v = sorted(values) + mid = n // 2 + median = sorted_v[mid] if n % 2 else (sorted_v[mid - 1] + sorted_v[mid]) / 2 + return { + "n": n, + "min": vmin, + "max": vmax, + "mean": mean, + "std": std, + "median": median, + } + + +def _stats_table(stats: Dict[str, float]) -> Table: + table = Table(show_header=True, header_style="bold", box=None, padding=(0, 2)) + for key in ("n", "min", "max", "mean", "std", "median"): + table.add_column(key, justify="right") + table.add_row( + str(int(stats["n"])), + _fmt_val(stats["min"]), + _fmt_val(stats["max"]), + _fmt_val(stats["mean"]), + _fmt_val(stats["std"]), + _fmt_val(stats["median"]), + ) + return table + + +def _plot_stats_table(stats: Dict[str, float], shape: Tuple[int, ...]) -> Table: + table = Table(show_header=True, header_style="bold", box=None, padding=(0, 2)) + for key in ("n", "min", "max", "mean", "std", "median"): + table.add_column(key, justify="right") + table.add_row( + str(int(stats["n"])), + _fmt_val(stats["min"]), + _fmt_val(stats["max"]), + _fmt_val(stats["mean"]), + _fmt_val(stats["std"]), + _fmt_val(stats["median"]), + ) + return table + + +def _plot_panel( + *, + plot: Text, + title: str, + units: str, + stats: Optional[Dict[str, float]], + shape: Tuple[int, ...], +) -> None: + content = plot + if stats: + content = Group(plot, _plot_stats_table(stats, shape)) + + _RICH_CONSOLE.print( + Panel( + content, + title=f"[bold]{title}[/bold] [dim]\\[{units}][/dim]", + subtitle=f"shape {shape}", + ) + ) + + +def show_quantity_textual_plot( + q: dict, + label: str = "", + x_quantity: Optional[dict] = None, +) -> None: + """Print line plot for a 1-D numeric QuantityData dict.""" + name = q["name"] + units = q["units"] or "-" + data = q["data"] + if not is_numeric_1d(data): + print_quantity(q, label=label) + return + + y_values = [float(value) for value in data] + shape = _get_shape(data) + x_values = None + xlabel = "index [-]" + if ( + x_quantity + and is_numeric_1d(x_quantity.get("data")) + and len(x_quantity["data"]) == len(y_values) + ): + x_values = [float(value) for value in x_quantity["data"]] + xlabel = _quantity_axis_label(x_quantity, fallback="x") + + title = label or name + if x_values is None: + x_values = [float(index) for index in range(len(y_values))] + + console_width = _RICH_CONSOLE.size.width + plot_width = max(48, min(70, console_width - 12)) + _, terminal_height = plotext.terminal_size() + plot_height = max(12, min(24, terminal_height - 8)) + + plotext.clear_figure() + plotext.canvas_color("default") + plotext.axes_color("default") + plotext.ticks_color("default") + plotext.plot_size(plot_width, plot_height) + plotext.xlabel(xlabel) + plotext.ylabel(_quantity_axis_label(q, fallback=label or "field")) + plotext.plot(x_values, y_values, marker="braille", color="cyan") + plot = Text.from_ansi(plotext.build()) + stats = _compute_stats(y_values) + _plot_panel( + plot=plot, + title=title, + units=units, + stats=stats, + shape=shape, + ) + print_quantity(q, label=label) + + +def print_quantity(q: dict, label: str = "", show_stats: bool = True) -> None: + """Print a QuantityData dict with array display and stats.""" + name = q["name"] + units = q["units"] or "-" + data = q["data"] + title = f"[bold]{label or name}[/bold] [dim]\\[{units}][/dim]" + + if not isinstance(data, list): + _RICH_CONSOLE.print(Panel(f"{_fmt_val(data)}", title=title, subtitle="scalar")) + return + + shape = _get_shape(data) + stats = _compute_stats(data) + array_body = _build_array_body(data, shape) + subtitle = f"shape ({shape[0]},)" if len(shape) == 1 else f"shape {shape}" + if show_stats and stats: + _RICH_CONSOLE.print( + Panel( + Group(array_body, _stats_table(stats)), + title=title, + subtitle=subtitle, + ) + ) + else: + _RICH_CONSOLE.print(Panel(array_body, title=title, subtitle=subtitle)) + def _flatten_dict(values: Dict) -> List[Tuple[str, str]]: items = [] diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index b831206..a7c2017 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -662,6 +662,11 @@ def delete_metadata(self, sim_id: str, key: str) -> List[str]: res = self.delete("simulation/metadata/" + sim_id, {"key": key}) return [data["value"] for data in res.json()] + @try_request + def get_simulation_data(self, sim_id: str, path: str) -> Dict[str, Any]: + res = self.get(f"simulation/{sim_id}/data", params={"path": path}) + return res.json() + @try_request def get_directory(self) -> str: res = self.get("staging_dir") diff --git a/src/simdb/remote/apis/v1_2/__init__.py b/src/simdb/remote/apis/v1_2/__init__.py index 920f303..6b8bc01 100644 --- a/src/simdb/remote/apis/v1_2/__init__.py +++ b/src/simdb/remote/apis/v1_2/__init__.py @@ -10,6 +10,7 @@ from simdb.remote.core.typing import current_app from simdb.remote.models import StagingDirectoryResponse +from .simulation_data import api as data_ns from .simulations import api as sim_ns api = Api( @@ -31,7 +32,7 @@ ) api.add_namespace(sim_ns) -namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns] +namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns, data_ns] @api.route("/staging_dir", defaults={"sim_hex": None}) diff --git a/src/simdb/remote/apis/v1_2/simulation_data.py b/src/simdb/remote/apis/v1_2/simulation_data.py new file mode 100644 index 0000000..b59235b --- /dev/null +++ b/src/simdb/remote/apis/v1_2/simulation_data.py @@ -0,0 +1,321 @@ +"""IMAS simulation data endpoint: /data. + +TODO: Temporary solution to retrieve data (for IBEX backend) +""" + +from typing import Annotated, Any, NamedTuple, Optional + +import imas +import numpy as np +from flask_restx import Namespace, Resource +from imas import IDSFactory +from imas.ids_convert import dd_version_map_from_factories +from imas.ids_defs import EMPTY_FLOAT +from imas.ids_path import IDSPath +from imas.ids_primitive import IDSPrimitive +from imas.ids_toplevel import IDSToplevel + +from simdb.cli.manifest import DataType +from simdb.database import DatabaseError +from simdb.imas.utils import ( + ImasError, + SimDBUrl, + open_imas, +) +from simdb.remote.core.auth import User, requires_auth +from simdb.remote.core.pydantic_utils import ( + Query, + ResponseException, + ServerException, + pydantic_validate, +) +from simdb.remote.core.typing import current_app +from simdb.remote.models import ImasDataQueryParams, ImasDataResponse, QuantityData + +api = Namespace("data", path="/") + + +# Helpers + + +def _to_python(value: Any) -> Any: + """Convert a value returned by IDSPrimitive.value to a JSON-serialisable + Python object.""" + if isinstance(value, np.ndarray): + flat = value.tolist() + + def _clean(v): + if isinstance(v, float) and ( + v != v or v == float("inf") or v == float("-inf") or v == EMPTY_FLOAT + ): + return None + if isinstance(v, list): + return [_clean(x) for x in v] + return v + + return _clean(flat) + return value + + +def _parse_ids_path(path: str) -> tuple: + """Parse ``ids_name[:occurrence][/ids_path]`` into a 3-tuple""" + head, _, ids_path = path.partition("/") + if ":" in head: + ids_name, occ_str = head.split(":", 1) + try: + occurrence = int(occ_str) + except ValueError as exc: + raise ValueError( + f"Invalid occurrence in path '{path}': '{occ_str}'" + ) from exc + else: + ids_name, occurrence = head, 0 + return ids_name, occurrence, ids_path + + +def _get_coordinates(node: IDSPrimitive, ids_name: str) -> list: + """Return a :class:`QuantityData` for each coordinate dimension of *node*.""" + coords = [] + for i in range(node.metadata.ndim): + coord = node.coordinates[i] + if isinstance(coord, IDSPrimitive): + data = ( + _to_python(coord.value) + if coord.has_value + else list(range(node.shape[i])) + ) + coords.append( + QuantityData( + name=f"{ids_name}/{coord._path}", + units=coord.metadata.units or "", + data=data, + ) + ) + else: + # Index-based coordinate: coord is already a numpy arange + coords.append( + QuantityData( + name=f"dim_{i + 1}", + units="", + data=coord.tolist(), + ) + ) + return coords + + +def _resolve_renamed_ids_path( + ids_obj: Any, ids_name: str, ids_path: str +) -> Optional[str]: + """Return the stored DD path for a requested current-DD path, if renamed.""" + if not ids_path: + return None + + stored_version = getattr(ids_obj, "_version", None) or getattr( + ids_obj, "_dd_version", None + ) + if not stored_version: + return None + + ddmap, _source_is_older = dd_version_map_from_factories( + ids_name, + IDSFactory(stored_version), + IDSFactory(), + ) + return ddmap.new_to_old.path.get(ids_path) + + +def _copy_ids_properties(src_props: Any, dst_props: Any) -> None: + """Copy all populated scalar fields from *src_props* to *dst_props*.""" + for field in src_props._children: + src_node = getattr(src_props, field) + if isinstance(src_node, IDSPrimitive) and src_node.has_value: + getattr(dst_props, field).value = src_node.value + + +def _set_path_value(ids: IDSToplevel, node_path: Any, value: Any) -> None: + """Generic function to write *value* into *ids* at *node_path* considering + IDSStructArray. + """ + p = IDSPath(str(node_path)) + current: Any = ids + + # allocate all intermediate nodes (structs and arrays) along the path + for part, idx in zip(p.parts[:-1], p.indices[:-1]): + child = getattr(current, part) + if isinstance(idx, int): + if len(child) <= idx: + child.resize(idx + 1) + current = child[idx] + else: + current = child + # set the value at the leaf node, allocating array if needed + last_part, last_idx = p.parts[-1], p.indices[-1] + if isinstance(last_idx, int): + child = getattr(current, last_part) + if len(child) <= last_idx: + child.resize(last_idx + 1) + child[last_idx].value = value + else: + getattr(current, last_part).value = value + + +def _build_nonlazy_ids( + lazy_ids: Any, + ids_name: str, + resolved_node: IDSPrimitive, + stored_version: str, +) -> IDSToplevel: + """Return a non-lazy IDS in *stored_version*""" + nonlazy: IDSToplevel = IDSFactory(stored_version).new(ids_name) + _copy_ids_properties(lazy_ids.ids_properties, nonlazy.ids_properties) + _set_path_value(nonlazy, resolved_node._path, resolved_node.value) + return nonlazy + + +def _get_ids_node( + entry, + ids_name: str, + occurrence: int, + ids_path: str, + dd_version: "str | None" = None, +) -> IDSPrimitive: + """Return the :class:`IDSPrimitive` leaf node at *ids_path* inside *ids_name*. + + Args: + entry: Open IMAS data entry. + ids_name: Name of the IDS to read. + occurrence: Occurrence index of the IDS. + ids_path: Slash-separated path within the IDS to the leaf node. + dd_version: When provided, convert the field value to this DD + version (e.g. ``"3.42.0"`` or ``"4.1.1"``) before returning. + """ + ids_obj = entry.get( + ids_name, + occurrence, + lazy=True, + autoconvert=False, + ignore_unknown_dd_version=True, + ) + try: + node = ids_obj[ids_path] if ids_path else ids_obj + except (AttributeError, IndexError, KeyError) as exc: + renamed_path = _resolve_renamed_ids_path(ids_obj, ids_name, ids_path) + if not renamed_path: + raise exc + if dd_version is None: + _dd_version = getattr(ids_obj, "_version", None) or getattr( + ids_obj, "_dd_version", "unknown" + ) + raise ValueError( + f"Path '{ids_path}' does not exist in the stored DD version " + f"({_dd_version})" + f" but is known under the name '{renamed_path}' in that version. " + f"Pass dd_version to request an explicit DD conversion. " + f"Original error: {type(exc).__name__}: {exc}" + ) from exc + try: + node = ids_obj[renamed_path] + except (AttributeError, IndexError, KeyError): + raise exc from None + + if dd_version is not None: + stored_version = ( + getattr(ids_obj, "_version", None) + or getattr(ids_obj, "_dd_version", None) + or entry.factory.version + ) + minimal = _build_nonlazy_ids(ids_obj, ids_name, node, stored_version) + converted = imas.convert_ids(minimal, dd_version) + node = converted[ids_path] if ids_path else converted + + if not isinstance(node, IDSPrimitive): + raise ValueError( + f"path does not point to a scalar/array leaf " + f"(reached {type(node).__name__}); add more path segments" + ) + if not node.has_value: + raise ValueError("field is not populated (no data written)") + return node + + +class _SimulationImasFile(NamedTuple): + simulation: Any + imas_file: Any + + +def _get_simulation_and_imas_file(sim_id: str) -> _SimulationImasFile: + try: + simulation = current_app.db.get_simulation(sim_id) + except DatabaseError as exc: + raise ResponseException(str(exc), 404) from exc + + imas_outputs = [f for f in simulation.outputs if f.type == DataType.IMAS] + if not imas_outputs: + raise ResponseException(f"Simulation {sim_id} has no IMAS output files", 404) + + return _SimulationImasFile(simulation, imas_outputs[0]) + + +# Endpoints + + +@api.route("/simulation//data") +class SimulationImasData(Resource): + @requires_auth() + @pydantic_validate(api) + def get( + self, + sim_id: str, + user: User, + params: Annotated[ImasDataQueryParams, Query()], + ) -> ImasDataResponse: + """Return the value at a given IDS path for a simulation's IMAS output.""" + result = _get_simulation_and_imas_file(sim_id) + + try: + ids_name, occurrence, ids_path = _parse_ids_path(params.path) + except ValueError as exc: + raise ResponseException(str(exc)) from exc + + try: + imas_uri = SimDBUrl(str(result.imas_file.uri)) + if imas_uri.host: + qs = dict(imas_uri.query_params()) + if "cache_mode" not in qs: + imas_uri = SimDBUrl(str(imas_uri) + "&cache_mode=none") + entry = open_imas(imas_uri) + with entry: + node = _get_ids_node( + entry, + ids_name, + occurrence, + ids_path, + dd_version=params.dd_version, + ) + coordinates = _get_coordinates(node, ids_name) + field = QuantityData( + name=f"{ids_name}/{node._path}", + units=node.metadata.units or "", + data=_to_python(node.value), + ) + except (ValueError, AttributeError, IndexError, KeyError) as exc: + raise ResponseException( + f"Invalid IDS path '{params.path}': {type(exc).__name__}: {exc}" + ) from exc + except ImasError as exc: + raise ServerException( + f"Failed to open IMAS data: {type(exc).__name__}: {exc}" + ) from exc + except Exception as exc: + msg = str(exc) + if "is empty" in msg or "not found" in msg.lower(): + raise ResponseException(f"{type(exc).__name__}: {msg}", 404) from exc + raise ServerException(f"{type(exc).__name__}: {msg}") from exc + + return ImasDataResponse( + simulation=str(result.simulation.uuid), + path=params.path, + occurrence=occurrence, + field=field, + coordinates=coordinates, + ) diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index fcaffef..eb3827a 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -26,7 +26,9 @@ BeforeValidator, ConfigDict, Field, + InstanceOf, PlainSerializer, + field_validator, model_validator, ) from pydantic import ( @@ -108,21 +110,6 @@ class RangeValue(BaseModel): max: float -MetadataValue = Union[ - CustomUUID, - str, - int, - float, - bool, - list, - RangeValue, - dict[str, Any], - None, -] -"""Supported types for simulation metadata values. Numpy arrays and regular arrays -containing numeric data are automatically converted to RangeValue.""" - - class StatusPatchData(BaseModel): """Post data for updating simulation status.""" @@ -181,6 +168,53 @@ def __getitem__(self, item) -> FileData: return self.root[item] +def _deserialize_numpy(v: Any) -> Any: + if isinstance(v, np.ndarray): + return v + if isinstance(v, dict) and v.get("_type") == "numpy.ndarray": + np_bytes = base64.b64decode(v["bytes"].encode()) + arr = np.frombuffer(np_bytes, dtype=v["dtype"]) + if "shape" in v: + arr = arr.reshape(v["shape"]) + return arr + raise ValueError(f"Cannot deserialize {v} to np.ndarray") + + +def _serialize_numpy(o: np.ndarray) -> dict: + """Serialize numpy arrays to dict format for the web dashboard.""" + encoded_bytes = base64.b64encode(o.tobytes()).decode() + return { + "_type": "numpy.ndarray", + "dtype": o.dtype.name, + "shape": o.shape, + "bytes": encoded_bytes, + } + + +NumpyArray = Annotated[ + InstanceOf[np.ndarray], + BeforeValidator(_deserialize_numpy), + PlainSerializer(_serialize_numpy, return_type=dict), +] + + +MetadataValue = Union[ + CustomUUID, + str, + int, + float, + bool, + RangeValue, + list, + dict, + NumpyArray, + None, +] +"""Supported types for simulation metadata values. RangeValue, numpy arrays and +scalars are automatically converted to their plain Python equivalents before +validation.""" + + class MetadataData(BaseModel): """Key-value pair for simulation metadata.""" @@ -572,6 +606,53 @@ class StagingDirectoryResponse(BaseModel): """Path to the staging dir.""" +class ImasDataQueryParams(BaseModel): + """Query parameters for the IMAS field-data endpoint.""" + + path: str + """Full IDS path including IDS name and optional occurrence.""" + + dd_version: Optional[str] = None + """When provided, explicitly convert the loaded IDS to this DD version + string (e.g. ``"3.42.0"``) using :func:`imas.convert_ids` after + reading. When omitted, data is returned in its stored DD version.""" + + @field_validator("path", mode="before") + @classmethod + def _strip_path(cls, v: Any) -> str: + v = str(v).strip() + if not v: + raise ValueError("must not be empty") + return v + + +class QuantityData(BaseModel): + """A named, unit-bearing data quantity (field value or coordinate).""" + + name: str + """IDS path of this quantity relative to the IDS root""" + units: str + """Physical units of the quantity""" + data: Any + """Data value: a Python scalar for 0-D quantities, or a nested list for + arrays. """ + + +class ImasDataResponse(BaseModel): + """Response from the IMAS field-data endpoint.""" + + simulation: str + """UUID of the simulation.""" + path: str + """Requested IDS path.""" + occurrence: int + """IDS occurrence index.""" + field: QuantityData + """The requested quantity""" + coordinates: List[QuantityData] + """Coordinates for each dimension of *field*, in dimension order.""" + + class ErrorResponse(BaseModel): """Response model for server errors.""" diff --git a/tests/remote/api/test_simulation_data.py b/tests/remote/api/test_simulation_data.py new file mode 100644 index 0000000..316e82d --- /dev/null +++ b/tests/remote/api/test_simulation_data.py @@ -0,0 +1,48 @@ +"""Tests for the /simulation//data endpoint helpers.""" + +import imas +import pytest +from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS + +from simdb.remote.apis.v1_2 import simulation_data + + +@pytest.fixture +def summary_entry(tmp_path): + """Write a minimal summary IDS to HDF5, return a read-mode DBEntry.""" + uri = f"imas:hdf5?path={tmp_path}" + with imas.DBEntry(uri, "x") as entry: + summary = entry.factory.new("summary") + summary.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + summary.global_quantities.ip.value = [1.5e6] + summary.time = [0.0] + entry.put(summary) + return imas.DBEntry(uri, "r") + + +def test_get_ids_node_returns_correct_value(summary_entry): + """Happy path: _get_ids_node returns the real IDSPrimitive with correct value.""" + with summary_entry as entry: + node = simulation_data._get_ids_node( + entry, "summary", 0, "global_quantities/ip/value" + ) + + assert node.has_value + assert list(node.value) == [1.5e6] + assert node.metadata.units == "A" + + +def test_get_ids_node_with_dd_version_returns_converted_value(summary_entry): + """convert_ids round-trip with matching dd_version preserves value.""" + with summary_entry as entry: + stored_version = entry.factory.version + node = simulation_data._get_ids_node( + entry, + "summary", + 0, + "global_quantities/ip/value", + dd_version=stored_version, + ) + + assert node.has_value + assert list(node.value) == [1.5e6] diff --git a/uv.lock b/uv.lock index 94ddb1f..1175123 100644 --- a/uv.lock +++ b/uv.lock @@ -2734,6 +2734,7 @@ dependencies = [ { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "plotext" }, { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "pyjwt", version = "2.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, @@ -2871,6 +2872,7 @@ requires-dist = [ { name = "nbsphinx", marker = "extra == 'build-docs'", specifier = ">=0.8.0" }, { name = "netcdf4", specifier = ">=1.5" }, { name = "numpy", specifier = ">=1.14" }, + { name = "plotext", specifier = "==5.3.2" }, { name = "psycopg2-binary", marker = "extra == 'postgres'", specifier = ">=2.8.0" }, { name = "pydantic", specifier = ">=2.10.6" }, { name = "pyjwt", specifier = ">=1.4.0" }, @@ -4283,6 +4285,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] +[[package]] +name = "plotext" +version = "5.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/d7/f75f397af966fe252d0d34ffd3cae765317fce2134f925f95e7d6725d1ce/plotext-5.3.2.tar.gz", hash = "sha256:52d1e932e67c177bf357a3f0fe6ce14d1a96f7f7d5679d7b455b929df517068e", size = 61967, upload-time = "2024-09-24T15:13:37.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/1e/12fe7c40cd2099a1f454518754ed229b01beaf3bbb343127f0cc13ce6c22/plotext-5.3.2-py3-none-any.whl", hash = "sha256:394362349c1ddbf319548cfac17ca65e6d5dfc03200c40dfdc0503b3e95a2283", size = 64047, upload-time = "2024-09-24T15:13:36.296Z" }, +] + [[package]] name = "pluggy" version = "1.5.0"