From 1b314a664de6c87517c8279c239f5ad4dd108f62 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:17:01 -0400 Subject: [PATCH 1/2] Add spaday-based model registry browser Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- ccflow/base.py | 4 +- ccflow/examples/tpch/config/conf.yaml | 8 - ccflow/flow_model.py | 54 ------- ccflow/tests/test_base.py | 4 +- ccflow/tests/ui/panel/__init__.py | 0 ccflow/tests/ui/{ => panel}/test_cli.py | 4 +- ccflow/tests/ui/{ => panel}/test_model.py | 4 +- ccflow/tests/ui/{ => panel}/test_registry.py | 4 +- ccflow/tests/ui/{ => panel}/utils.py | 0 ccflow/tests/ui/spaday/__init__.py | 0 ccflow/tests/ui/spaday/test_cli.py | 81 ++++++++++ ccflow/tests/ui/spaday/test_model.py | 114 ++++++++++++++ ccflow/tests/ui/spaday/test_registry.py | 137 ++++++++++++++++ ccflow/tests/ui/spaday/utils.py | 50 ++++++ ccflow/ui/__init__.py | 4 +- ccflow/ui/panel/__init__.py | 3 + ccflow/ui/{ => panel}/cli.py | 14 +- ccflow/ui/{ => panel}/model.py | 0 ccflow/ui/{ => panel}/registry.py | 0 ccflow/ui/spaday/__init__.py | 3 + ccflow/ui/spaday/cli.py | 157 +++++++++++++++++++ ccflow/ui/spaday/model.py | 120 ++++++++++++++ ccflow/ui/spaday/registry.py | 97 ++++++++++++ ccflow/utils/hydra.py | 35 ++--- ccflow/utils/tokenize.py | 5 - pyproject.toml | 7 + 26 files changed, 798 insertions(+), 111 deletions(-) create mode 100644 ccflow/tests/ui/panel/__init__.py rename ccflow/tests/ui/{ => panel}/test_cli.py (94%) rename ccflow/tests/ui/{ => panel}/test_model.py (99%) rename ccflow/tests/ui/{ => panel}/test_registry.py (99%) rename ccflow/tests/ui/{ => panel}/utils.py (100%) create mode 100644 ccflow/tests/ui/spaday/__init__.py create mode 100644 ccflow/tests/ui/spaday/test_cli.py create mode 100644 ccflow/tests/ui/spaday/test_model.py create mode 100644 ccflow/tests/ui/spaday/test_registry.py create mode 100644 ccflow/tests/ui/spaday/utils.py create mode 100644 ccflow/ui/panel/__init__.py rename ccflow/ui/{ => panel}/cli.py (89%) rename ccflow/ui/{ => panel}/model.py (100%) rename ccflow/ui/{ => panel}/registry.py (100%) create mode 100644 ccflow/ui/spaday/__init__.py create mode 100644 ccflow/ui/spaday/cli.py create mode 100644 ccflow/ui/spaday/model.py create mode 100644 ccflow/ui/spaday/registry.py diff --git a/ccflow/base.py b/ccflow/base.py index f4a68672..2e49ac70 100644 --- a/ccflow/base.py +++ b/ccflow/base.py @@ -286,7 +286,7 @@ def __panel__(self): Requires ccflow UI dependencies (panel, panel_material_ui). """ try: - from ccflow.ui.model import ModelViewer + from ccflow.ui.panel.model import ModelViewer except ImportError: raise ImportError( "panel and other optional dependencies must be installed to use ModelViewer. Pip install ccflow[full] to install all optional dependencies." @@ -480,7 +480,7 @@ def __panel__(self): Requires ccflow UI dependencies (panel, panel_material_ui). """ - from ccflow.ui.registry import ModelRegistryViewer + from ccflow.ui.panel.registry import ModelRegistryViewer return ModelRegistryViewer(self) diff --git a/ccflow/examples/tpch/config/conf.yaml b/ccflow/examples/tpch/config/conf.yaml index b888697b..dcf6b76c 100644 --- a/ccflow/examples/tpch/config/conf.yaml +++ b/ccflow/examples/tpch/config/conf.yaml @@ -24,19 +24,15 @@ # (``load_config(overrides=["tpch.backend.scale_factor=1.0"])``) reconfigures # every table, answer and query consistently. -# --------------------------------------------------------------------------- # Shared DuckDB backend. Plain ``ccflow.BaseModel`` — not callable itself, # but registered so all providers share one connection and one ``dbgen`` call. -# --------------------------------------------------------------------------- tpch: backend: _target_: ccflow.examples.tpch.TPCHDuckDBBackend scale_factor: 0.1 -# --------------------------------------------------------------------------- # Per-table providers. One instance per TPC-H table; the output schema of # each instance is fixed by its ``table`` field. -# --------------------------------------------------------------------------- table: customer: _target_: ccflow.examples.tpch.TPCHTableProvider @@ -71,10 +67,8 @@ table: backend: /tpch/backend table: supplier -# --------------------------------------------------------------------------- # Reference answers, one per query, served straight from DuckDB's # ``tpch_answers()`` table at the configured scale factor. -# --------------------------------------------------------------------------- answer: Q1: _target_: ccflow.examples.tpch.TPCHAnswerProvider @@ -165,12 +159,10 @@ answer: backend: /tpch/backend query_id: 22 -# --------------------------------------------------------------------------- # The 22 TPC-H queries. Each ``TPCHQuery`` is the same Python class with a # different ``query_id`` and a different tuple of table-provider inputs. # Wiring the inputs in YAML makes each query's table dependencies explicit # and overridable per-query. -# --------------------------------------------------------------------------- query: Q1: _target_: ccflow.examples.tpch.TPCHQuery diff --git a/ccflow/flow_model.py b/ccflow/flow_model.py index 94f22a56..64944ce8 100644 --- a/ccflow/flow_model.py +++ b/ccflow/flow_model.py @@ -125,11 +125,6 @@ _AnyCallable = Callable[..., Any] -# --------------------------------------------------------------------------- -# Internal data structures -# --------------------------------------------------------------------------- - - class _UnsetFlowInput: def __repr__(self) -> str: return "" @@ -404,11 +399,6 @@ class _LocalFlowModelPicklePayload(NamedTuple): factory_kwargs: Dict[str, Any] -# --------------------------------------------------------------------------- -# Small value helpers -# --------------------------------------------------------------------------- - - def _context_values(context: ContextBase) -> Dict[str, Any]: return dict(context) @@ -480,11 +470,6 @@ def _concrete_context_type(context_type: Any) -> Optional[Type[ContextBase]]: return None -# --------------------------------------------------------------------------- -# Type coercion, lazy thunks, and registry references -# --------------------------------------------------------------------------- - - def _remember_type_adapter(cache: "OrderedDict[Any, Any]", key: Any, value: Any) -> Any: cache[key] = value cache.move_to_end(key) @@ -681,11 +666,6 @@ def _ensure_named_python_function(fn: _AnyCallable, *, decorator_name: str) -> N raise TypeError(f"{decorator_name} only supports named Python functions.") -# --------------------------------------------------------------------------- -# Context-transform serialization and generated-model persistence -# --------------------------------------------------------------------------- - - def _serialize_context_transform_config(config: _FlowModelConfig) -> str: payload = cloudpickle.dumps(_serialize_flow_model_config(config), protocol=5) return b64encode(payload).decode("ascii") @@ -878,11 +858,6 @@ def _register_generated_model_class(config: _FlowModelConfig, generated_cls: typ ) -# --------------------------------------------------------------------------- -# Runtime context contracts and dependency projection -# --------------------------------------------------------------------------- - - def _runtime_context_for_model(model: CallableModel, values: Dict[str, Any]) -> ContextBase: """Build the runtime context object expected by ``model`` from raw values.""" @@ -1037,11 +1012,6 @@ def _missing_regular_param_names(model: "_GeneratedFlowModelBase", config: _Flow return missing -# --------------------------------------------------------------------------- -# Generated model input resolution -# --------------------------------------------------------------------------- - - def _resolve_regular_param_value(model: "_GeneratedFlowModelBase", param: _FlowModelParam, context: ContextBase) -> Any: value = getattr(model, param.name, _UNSET_FLOW_INPUT) if _is_unset_flow_input(value): @@ -1481,10 +1451,6 @@ def _coerce_model_context_value(model: CallableModel, field_name: str, value: An return _coerce_value(field_name, value, contract.input_types[field_name], source) -# --------------------------------------------------------------------------- -# Effective identity helpers -# --------------------------------------------------------------------------- - # Identity terms used below: # - config identity: stable hash of the analyzed Flow.model contract, fixed at # generated-class construction time and carried through local restore. @@ -1854,11 +1820,6 @@ def _generated_model_identity_payload( ) -# --------------------------------------------------------------------------- -# Static binding resolution and with_context normalization -# --------------------------------------------------------------------------- - - def _resolved_static_contextual_values( model: "_GeneratedFlowModelBase", config: _FlowModelConfig, @@ -2115,11 +2076,6 @@ def _normalize_with_context(model: CallableModel, patches: Tuple[Any, ...], fiel return _validate_static_context_spec_declared_context(model, context_spec) -# --------------------------------------------------------------------------- -# Bound context application and compute context construction -# --------------------------------------------------------------------------- - - def _context_from_values_preserving_private_state(context: ContextBase, values: Dict[str, Any]) -> ContextBase: """Validate updated public values while preserving private context state.""" @@ -2548,11 +2504,6 @@ def _recursive_dependency_specs_for_flow( active.remove(model_id) -# --------------------------------------------------------------------------- -# model.flow API and BoundModel wrapper -# --------------------------------------------------------------------------- - - class FlowAPI: """API namespace exposed as ``model.flow``. @@ -3171,11 +3122,6 @@ def _evaluation_identity_payload( return _generated_model_identity_payload(self, context) -# --------------------------------------------------------------------------- -# Generated model method builders and decorators -# --------------------------------------------------------------------------- - - def _make_call_impl(config: _FlowModelConfig) -> _AnyCallable: """Create the ``__call__`` implementation for one generated model class.""" diff --git a/ccflow/tests/test_base.py b/ccflow/tests/test_base.py index 26623fb0..2ec33296 100644 --- a/ccflow/tests/test_base.py +++ b/ccflow/tests/test_base.py @@ -175,8 +175,8 @@ def test_widget(self): def test_panel(self): from ccflow import ModelRegistry - from ccflow.ui.model import ModelViewer - from ccflow.ui.registry import ModelRegistryViewer + from ccflow.ui.panel.model import ModelViewer + from ccflow.ui.panel.registry import ModelRegistryViewer m = ModelA(x="foo") panel_obj = m.__panel__() diff --git a/ccflow/tests/ui/panel/__init__.py b/ccflow/tests/ui/panel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ccflow/tests/ui/test_cli.py b/ccflow/tests/ui/panel/test_cli.py similarity index 94% rename from ccflow/tests/ui/test_cli.py rename to ccflow/tests/ui/panel/test_cli.py index c399fb73..45353553 100644 --- a/ccflow/tests/ui/test_cli.py +++ b/ccflow/tests/ui/panel/test_cli.py @@ -1,6 +1,6 @@ -"""Unit tests for ccflow.ui.cli module.""" +"""Unit tests for ccflow.ui.panel.cli module.""" -from ccflow.ui.cli import _get_ui_args_parser +from ccflow.ui.panel.cli import _get_ui_args_parser class TestGetUIArgsParser: diff --git a/ccflow/tests/ui/test_model.py b/ccflow/tests/ui/panel/test_model.py similarity index 99% rename from ccflow/tests/ui/test_model.py rename to ccflow/tests/ui/panel/test_model.py index 043dbd63..7cc15687 100644 --- a/ccflow/tests/ui/test_model.py +++ b/ccflow/tests/ui/panel/test_model.py @@ -1,10 +1,10 @@ -"""Unit tests for ccflow.ui.model module.""" +"""Unit tests for ccflow.ui.panel.model module.""" import panel as pn from pydantic import Field from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, MetaData, ModelRegistry -from ccflow.ui.model import ModelConfigViewer, ModelTypeViewer, ModelViewer +from ccflow.ui.panel.model import ModelConfigViewer, ModelTypeViewer, ModelViewer from .utils import find_components_by_type diff --git a/ccflow/tests/ui/test_registry.py b/ccflow/tests/ui/panel/test_registry.py similarity index 99% rename from ccflow/tests/ui/test_registry.py rename to ccflow/tests/ui/panel/test_registry.py index d9b1dd8a..a5f0f744 100644 --- a/ccflow/tests/ui/test_registry.py +++ b/ccflow/tests/ui/panel/test_registry.py @@ -1,11 +1,11 @@ -"""Unit tests for ccflow.ui.registry module.""" +"""Unit tests for ccflow.ui.panel.registry module.""" from unittest import mock import panel as pn from ccflow import BaseModel, ModelRegistry -from ccflow.ui.registry import ModelRegistryViewer, RegistryBrowser +from ccflow.ui.panel.registry import ModelRegistryViewer, RegistryBrowser from .utils import find_components_by_type diff --git a/ccflow/tests/ui/utils.py b/ccflow/tests/ui/panel/utils.py similarity index 100% rename from ccflow/tests/ui/utils.py rename to ccflow/tests/ui/panel/utils.py diff --git a/ccflow/tests/ui/spaday/__init__.py b/ccflow/tests/ui/spaday/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ccflow/tests/ui/spaday/test_cli.py b/ccflow/tests/ui/spaday/test_cli.py new file mode 100644 index 00000000..976b6855 --- /dev/null +++ b/ccflow/tests/ui/spaday/test_cli.py @@ -0,0 +1,81 @@ +"""Unit tests for ccflow.ui.spaday.cli module.""" + +from pathlib import Path + +from spaday.bootstrap import _ASSETS, bundles_dir + +from ccflow import BaseModel, ModelRegistry +from ccflow.ui.spaday.cli import _asset_layout, _get_ui_args_parser, serve_registry + + +class SimpleModel(BaseModel): + name: str + value: int = 0 + + +class TestGetUIArgsParser: + def test_parser_composition(self): + parser = _get_ui_args_parser() + args = parser.parse_args([]) + + # From add_hydra_config_args + assert hasattr(args, "overrides") + assert hasattr(args, "config_path") + assert hasattr(args, "config_name") + + # Server + viewer-specific + assert hasattr(args, "address") + assert hasattr(args, "port") + assert hasattr(args, "browser_width") + assert hasattr(args, "title") + assert hasattr(args, "sort_children") + + def test_defaults(self): + args = _get_ui_args_parser().parse_args([]) + assert args.address == "127.0.0.1" + assert args.port == 8080 + assert args.browser_width == 400 + assert args.title == "ccflow Model Registry" + assert args.sort_children is True + + def test_custom_values(self): + args = _get_ui_args_parser().parse_args(["--address", "0.0.0.0", "--port", "9000", "--browser-width", "500", "--title", "Mine"]) + assert args.address == "0.0.0.0" + assert args.port == 9000 + assert args.browser_width == 500 + assert args.title == "Mine" + + def test_no_sort_children_flag(self): + args = _get_ui_args_parser().parse_args(["--no-sort-children"]) + assert args.sort_children is False + + def test_overrides_positional(self): + args = _get_ui_args_parser().parse_args(["key1=value1", "key2=value2"]) + assert args.overrides == ["key1=value1", "key2=value2"] + + +class TestServeRegistry: + def test_builds_app_without_running(self): + registry = ModelRegistry(name="test") + registry.add("m", SimpleModel(name="m", value=1)) + app = serve_registry(registry, run=False) + paths = {getattr(route, "path", None) for route in app.routes} + assert "/" in paths + assert "/tree.json" in paths + + def test_tree_route_reflects_registry(self): + registry = ModelRegistry(name="test") + registry.add("widget", SimpleModel(name="widget")) + app = serve_registry(registry, title="T", run=False) + # The tree route serializes the viewer; the model path should appear in it. + tree_route = next(r for r in app.routes if getattr(r, "path", None) == "/tree.json") + assert tree_route is not None + + +class TestAssetLayout: + def test_selected_layout_has_runtime_asset(self): + # Guards the 404 regression: an unrelated top-level ``js`` package must not push us to the + # "source" layout, whose bundle directory would then lack spaday's runtime asset. + layout = _asset_layout() + runtime = _ASSETS[layout]["runtime"].lstrip("/") + assert (Path(bundles_dir(layout)) / runtime).is_file() diff --git a/ccflow/tests/ui/spaday/test_model.py b/ccflow/tests/ui/spaday/test_model.py new file mode 100644 index 00000000..1338d764 --- /dev/null +++ b/ccflow/tests/ui/spaday/test_model.py @@ -0,0 +1,114 @@ +"""Unit tests for ccflow.ui.spaday.model module.""" + +from typing import Type + +from pydantic import Field +from spaday.validate import validate + +from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, ModelRegistry +from ccflow.ui.spaday.model import model_config_view, model_type_view, model_view + +from .utils import all_text, nodes_with_tag, text_of + + +class SimpleModel(BaseModel): + """A documented test model.""" + + name: str = Field(description="the display name") + value: int = 0 + + +class Ctx(ContextBase): + """A test context.""" + + a: int = 1 + + +class MyCallable(CallableModel): + """A callable test model.""" + + x: str = "hi" + + @property + def context_type(self) -> Type[Ctx]: + return Ctx + + @Flow.call + def __call__(self, context: Ctx) -> GenericResult: + return GenericResult(value=self.x) + + +class TestModelTypeView: + def test_none_is_empty(self): + node = model_type_view(None).to_node() + assert node["tag"] == "spa-stack" + assert node.get("slots", {}) == {} + + def test_type_name_in_badge(self): + node = model_type_view(SimpleModel).to_node() + badges = nodes_with_tag(node, "wa-badge") + assert any(text_of(b) == "SimpleModel" for b in badges) + + def test_lists_fields(self): + text = " ".join(all_text(model_type_view(SimpleModel).to_node())) + assert "name" in text + assert "value" in text + + def test_includes_field_description(self): + text = " ".join(all_text(model_type_view(SimpleModel).to_node())) + assert "the display name" in text + + def test_includes_docstring(self): + text = " ".join(all_text(model_type_view(SimpleModel).to_node())) + assert "A documented test model." in text + + +class TestModelConfigView: + def test_includes_path(self): + model = SimpleModel(name="m") + text = " ".join(all_text(model_config_view(model, "reg/m").to_node())) + assert "reg/m" in text + + def test_no_metadata_message_when_empty(self): + model = SimpleModel(name="m") + text = " ".join(all_text(model_config_view(model).to_node())) + assert "No additional metadata." in text + + def test_dependencies_rendered(self): + registry = ModelRegistry(name="test") + dep = SimpleModel(name="dep") + registry.add("dep", dep) + holder = MyCallable() + registry.add("holder", holder) + # A model that depends on another shows its registry dependencies (if any). + node = model_config_view(holder, "holder").to_node() + assert node["tag"] == "spa-stack" + + +class TestModelView: + def test_is_card(self): + node = model_view(SimpleModel(name="m"), "m").to_node() + assert node["tag"] == "wa-card" + + def test_has_core_tabs(self): + text = all_text(model_view(SimpleModel(name="m"), "m").to_node()) + assert "Summary" in text + assert "Model Type" in text + assert "Parameters" in text + + def test_plain_model_has_no_callable_tabs(self): + text = all_text(model_view(SimpleModel(name="m"), "m").to_node()) + assert "Context Type" not in text + assert "Result Type" not in text + + def test_callable_model_has_callable_tabs(self): + text = all_text(model_view(MyCallable(), "m").to_node()) + assert "Context Type" in text + assert "Result Type" in text + + def test_parameters_include_field_values(self): + text = " ".join(all_text(model_view(SimpleModel(name="widget", value=7), "m").to_node())) + assert "widget" in text + + def test_validates(self): + validate(model_view(MyCallable(), "m").to_node()) diff --git a/ccflow/tests/ui/spaday/test_registry.py b/ccflow/tests/ui/spaday/test_registry.py new file mode 100644 index 00000000..13989c15 --- /dev/null +++ b/ccflow/tests/ui/spaday/test_registry.py @@ -0,0 +1,137 @@ +"""Unit tests for ccflow.ui.spaday.registry module.""" + +from spaday.validate import validate + +from ccflow import BaseModel, ModelRegistry +from ccflow.ui.spaday.registry import ( + SELECTED_FIELD, + registry_leaves, + registry_store, + registry_tree, + registry_viewer, +) + +from .utils import click_set_field, nodes_with_tag, prop_str, show_when_value + + +class SimpleModel(BaseModel): + """A simple test model.""" + + name: str + value: int = 0 + + +class AnotherModel(BaseModel): + """Another test model.""" + + data: str = "" + + +def _registry(): + root = ModelRegistry(name="root") + sub = ModelRegistry(name="sub") + sub.add("alpha", SimpleModel(name="a", value=1)) + root.add("sub", sub) + root.add("zeta", AnotherModel(data="z")) + return root + + +class TestRegistryStore: + def test_default_store(self): + assert registry_store() == {SELECTED_FIELD: ""} + + +class TestRegistryLeaves: + def test_empty_registry(self): + assert registry_leaves(ModelRegistry(name="empty")) == [] + + def test_flat_registry(self): + registry = ModelRegistry(name="test") + model = SimpleModel(name="m", value=1) + registry.add("my_model", model) + assert registry_leaves(registry) == [("my_model", model)] + + def test_nested_paths(self): + leaves = registry_leaves(_registry()) + paths = [path for path, _ in leaves] + assert paths == ["sub/alpha", "zeta"] + + def test_sort_children_orders_subregistries_first(self): + root = ModelRegistry(name="root") + root.add("zzz_leaf", SimpleModel(name="leaf")) + sub = ModelRegistry(name="sub") + sub.add("inner", SimpleModel(name="inner")) + root.add("aaa_sub", sub) + # Subregistries sort before leaf models regardless of name. + assert [p for p, _ in registry_leaves(root)] == ["aaa_sub/inner", "zzz_leaf"] + + def test_insertion_order_when_not_sorted(self): + root = ModelRegistry(name="root") + root.add("zebra", SimpleModel(name="z")) + root.add("alpha", SimpleModel(name="a")) + assert [p for p, _ in registry_leaves(root, sort_children=False)] == ["zebra", "alpha"] + + +class TestRegistryTree: + def test_leaf_items_carry_selection_action(self): + nodes = registry_tree(_registry()) + # Serialize the whole set of tree items and collect leaf selection targets. + selected = set() + for item in nodes: + for node in nodes_with_tag(item.to_node(), "wa-tree-item"): + value = click_set_field(node) + if value is not None: + selected.add(value) + assert selected == {"sub/alpha", "zeta"} + + def test_branch_items_have_no_selection_action(self): + nodes = registry_tree(_registry()) + # The top-level "sub" node is a branch; it must not carry a click action. + sub_item = next(n for n in nodes if any(t == "sub" for t in _labels(n.to_node()))) + assert click_set_field(sub_item.to_node()) is None + + +def _labels(node): + from .utils import text_of + + return [text_of(n) for n in node.get("slots", {}).get("default", [])] + + +class TestRegistryViewer: + def test_returns_app(self): + app = registry_viewer(_registry()) + assert app.to_node()["tag"] == "spa-app" + + def test_validates(self): + validate(registry_viewer(_registry()).to_node()) + + def test_title_in_header(self): + from .utils import all_text + + node = registry_viewer(_registry(), title="My Registry").to_node() + assert "My Registry" in all_text(node) + + def test_show_panel_per_leaf(self): + node = registry_viewer(_registry()).to_node() + show_targets = {show_when_value(n) for n in nodes_with_tag(node, "spa-show")} + # A panel per leaf plus the empty-selection placeholder. + assert "sub/alpha" in show_targets + assert "zeta" in show_targets + assert "" in show_targets + + def test_search_options_cover_all_leaves(self): + node = registry_viewer(_registry()).to_node() + options = [prop_str(n, "value") for n in nodes_with_tag(node, "wa-option")] + # First option is the empty placeholder; the rest are sorted leaf paths. + assert options[0] == "" + assert options[1:] == sorted(["sub/alpha", "zeta"]) + + def test_browser_width_sets_gutter(self): + node = registry_viewer(_registry(), browser_width=500).to_node() + gutters = nodes_with_tag(node, "spa-gutter") + assert prop_str(gutters[0], "width") == "500px" + + def test_empty_registry_renders(self): + node = registry_viewer(ModelRegistry(name="empty")).to_node() + # Only the placeholder show panel, no model panels. + assert [show_when_value(n) for n in nodes_with_tag(node, "spa-show")] == [""] diff --git a/ccflow/tests/ui/spaday/utils.py b/ccflow/tests/ui/spaday/utils.py new file mode 100644 index 00000000..6da9b73e --- /dev/null +++ b/ccflow/tests/ui/spaday/utils.py @@ -0,0 +1,50 @@ +"""Helpers for inspecting the serialized spaday component tree in tests.""" + + +def iter_nodes(node): + """Yield ``node`` and every descendant node (depth-first) of a ``to_node()`` dict.""" + yield node + for children in node.get("slots", {}).values(): + for child in children: + yield from iter_nodes(child) + + +def nodes_with_tag(node, tag): + """All nodes in the tree with the given element ``tag``.""" + return [n for n in iter_nodes(node) if n.get("tag") == tag] + + +def text_of(node): + """The node's ``textContent`` string, or None.""" + tc = node.get("props", {}).get("textContent") + return tc.get("Str") if isinstance(tc, dict) else None + + +def all_text(node): + """Every ``textContent`` string found in the tree.""" + return [t for t in (text_of(n) for n in iter_nodes(node)) if t is not None] + + +def prop_str(node, name): + """A node prop serialized as a string (the ``{"Str": value}`` tag), or None.""" + value = node.get("props", {}).get(name) + return value.get("Str") if isinstance(value, dict) else None + + +def click_set_field(node): + """The literal value written by a ``click`` SetField action on the node, or None.""" + event = node.get("events", {}).get("click") + if event and event.get("kind") == "set-field": + return event["value"]["value"] + return None + + +def show_when_value(node): + """The literal a ``spa-show`` compares ``selected`` against in its ``when`` binding, or None.""" + when = node.get("bindings", {}).get("when") + if not when or "compute" not in when: + return None + expr = when["compute"] + if expr.get("expr") == "eq": + return expr["b"].get("value") + return None diff --git a/ccflow/ui/__init__.py b/ccflow/ui/__init__.py index 417aeab3..a09aa86a 100644 --- a/ccflow/ui/__init__.py +++ b/ccflow/ui/__init__.py @@ -1,3 +1 @@ -from .cli import * -from .model import * -from .registry import * +from .panel import * # noqa: F401,F403 Back-compat: the Panel UI remains the default and is re-exported here. diff --git a/ccflow/ui/panel/__init__.py b/ccflow/ui/panel/__init__.py new file mode 100644 index 00000000..417aeab3 --- /dev/null +++ b/ccflow/ui/panel/__init__.py @@ -0,0 +1,3 @@ +from .cli import * +from .model import * +from .registry import * diff --git a/ccflow/ui/cli.py b/ccflow/ui/panel/cli.py similarity index 89% rename from ccflow/ui/cli.py rename to ccflow/ui/panel/cli.py index 23bd34bc..2e391329 100644 --- a/ccflow/ui/cli.py +++ b/ccflow/ui/panel/cli.py @@ -50,15 +50,11 @@ def registry_viewer_cli( ): """CLI entry point for serving ModelRegistryViewer. - Parameters - ---------- - config_path - The config_path specified in hydra.main() - config_name - The config_name specified in hydra.main() - hydra_main - The function decorated with hydra.main(). Used to resolve config_path - relative to the decorated function's file location. + Args: + config_path: The config_path specified in hydra.main() + config_name: The config_name specified in hydra.main() + hydra_main: The function decorated with hydra.main(). Used to resolve config_path + relative to the decorated function's file location. """ parser = _get_ui_args_parser() args = parser.parse_args() diff --git a/ccflow/ui/model.py b/ccflow/ui/panel/model.py similarity index 100% rename from ccflow/ui/model.py rename to ccflow/ui/panel/model.py diff --git a/ccflow/ui/registry.py b/ccflow/ui/panel/registry.py similarity index 100% rename from ccflow/ui/registry.py rename to ccflow/ui/panel/registry.py diff --git a/ccflow/ui/spaday/__init__.py b/ccflow/ui/spaday/__init__.py new file mode 100644 index 00000000..076d582e --- /dev/null +++ b/ccflow/ui/spaday/__init__.py @@ -0,0 +1,3 @@ +from .cli import * # noqa: F401,F403 +from .model import * # noqa: F401,F403 +from .registry import * # noqa: F401,F403 diff --git a/ccflow/ui/spaday/cli.py b/ccflow/ui/spaday/cli.py new file mode 100644 index 00000000..03bc54d0 --- /dev/null +++ b/ccflow/ui/spaday/cli.py @@ -0,0 +1,157 @@ +"""CLI for serving the ccflow ModelRegistry as a spaday application. + +Mirrors :mod:`ccflow.ui.panel.cli` but renders the spaday viewer and serves it with Starlette + uvicorn +instead of Panel. ``serve_registry`` is the importable entry point; ``registry_viewer_cli`` is the +hydra-config-driven command wrapped by the ``ccflow-ui-spaday`` console script. +""" + +import argparse +import os +from pathlib import Path +from typing import Callable, Optional + +from ccflow import ModelRegistry +from ccflow.utils.hydra import add_hydra_config_args, load_config, resolve_config_paths + +from .registry import registry_store, registry_viewer + +__all__ = ("serve_registry", "registry_viewer_cli", "main") + + +def _asset_layout() -> str: + """Select spaday's asset layout ("source" vs "installed"). + + spaday auto-detects this from whether ``/../js`` is a directory, but an unrelated + top-level ``js`` package on ``sys.path`` (common in site-packages) makes it wrongly choose the + "source" layout, whose bundle URLs then 404. Only a real spaday source checkout ships ``js/dist``, + so require that before trusting the source layout; otherwise use the packaged extension assets. + """ + import spaday + + source_js = Path(spaday.__file__).resolve().parent.parent / "js" + return "source" if (source_js / "dist").is_dir() else "installed" + + +def serve_registry( + registry: ModelRegistry, + *, + title: str = "ccflow Model Registry", + browser_width: int = 400, + sort_children: bool = True, + address: str = "127.0.0.1", + port: int = 8080, + run: bool = True, +): + """Build the spaday registry viewer and serve it as a Starlette app. + + Args: + registry: The registry to browse. The page tree is rebuilt per request, so it reflects the + registry's current contents. + title: Title shown in the page header. + browser_width: Initial width of the registry sidebar, in pixels. + sort_children: Sort registry entries alphabetically at every level (subregistries first). + address, port: Interface and port uvicorn binds to (only used when ``run`` is True). + run: When True, start a blocking uvicorn server. When False, return the app without serving. + + Returns: + starlette.applications.Starlette: The mounted spaday application. + """ + try: + import uvicorn + from spaday.backends.starlette import serve + except ImportError: + raise ImportError( + "spaday, starlette and uvicorn must be installed to serve the spaday UI. Pip install ccflow[full] to install all optional dependencies." + ) from None + + app = serve( + lambda: registry_viewer(registry, title=title, browser_width=browser_width, sort_children=sort_children), + bundles=["webawesome"], + store=registry_store(), + title=title, + layout=_asset_layout(), + ) + if run: + uvicorn.run(app, host=address, port=port) + return app + + +def _get_ui_args_parser() -> argparse.ArgumentParser: + """Create the argument parser for the spaday viewer server.""" + parser = argparse.ArgumentParser( + add_help=True, + description="Serve the ccflow ModelRegistry viewer as a spaday application", + ) + + add_hydra_config_args(parser) + + parser.add_argument("--address", type=str, default="127.0.0.1", help="Address to bind the server to (default: 127.0.0.1).") + parser.add_argument("--port", type=int, default=8080, help="Port to bind the server to (default: 8080).") + parser.add_argument( + "--browser-width", + type=int, + default=400, + help="Initial width of the registry browser sidebar in px (default: 400).", + ) + parser.add_argument( + "--title", + type=str, + default="ccflow Model Registry", + help="Title shown in the page header (default: 'ccflow Model Registry').", + ) + parser.add_argument( + "--no-sort-children", + dest="sort_children", + action="store_false", + help="Keep registry entries in insertion order instead of sorting them alphabetically.", + ) + + return parser + + +def registry_viewer_cli( + config_path: str = "", + config_name: str = "", + hydra_main: Optional[Callable] = None, +): + """CLI entry point for serving the spaday ModelRegistry viewer. + + Args: + config_path: The config_path specified in hydra.main(). + config_name: The config_name specified in hydra.main(). + hydra_main: The function decorated with hydra.main(). Used to resolve config_path relative to + the decorated function's file location. + """ + parser = _get_ui_args_parser() + args = parser.parse_args() + + root_config_dir, root_config_name = resolve_config_paths(args, config_path, config_name, hydra_main) + # hydra's initialize_config_dir requires an absolute directory; resolve a relative --config-path + # against the current working directory. + root_config_dir = os.path.abspath(root_config_dir) + + result = load_config( + root_config_dir=root_config_dir, + root_config_name=root_config_name, + config_dir=args.config_dir, + config_name=args.config_dir_config_name, + overrides=args.overrides, + basepath=args.basepath, + ) + + registry = ModelRegistry.root() + registry.load_config(cfg=result.cfg, overwrite=True) + + serve_registry( + registry, + title=args.title, + browser_width=args.browser_width, + sort_children=args.sort_children, + address=args.address, + port=args.port, + ) + + +def main(): + """Console-script entry point (``ccflow-ui-spaday``).""" + registry_viewer_cli() diff --git a/ccflow/ui/spaday/model.py b/ccflow/ui/spaday/model.py new file mode 100644 index 00000000..ba2c61c7 --- /dev/null +++ b/ccflow/ui/spaday/model.py @@ -0,0 +1,120 @@ +"""Model-detail components for the spaday registry viewer. + +Each function builds a piece of the model inspector as a :class:`spaday.Component` tree (rendered to the +browser by the spaday runtime), mirroring the tabs of the Panel viewer in :mod:`ccflow.ui.panel.model`: +an instance summary, the model / context / result types with their fields, and the serialized parameters. +""" + +import json + +from pydantic._internal._repr import display_as_type +from spaday import Component, Strong, Text, element +from spaday.components import Column, Row, Tabs, WaBadge, WaCard, WaDivider + +import ccflow + +__all__ = ("model_type_view", "model_config_view", "model_view") + +_PRE_STYLE = { + "white_space": "pre-wrap", + "font_family": "monospace", + "background": "#f6f8fa", + "padding": "8px", + "margin": "0", + "border_radius": "4px", + "overflow_wrap": "anywhere", +} + + +def _labeled(label: str, *body: Component) -> Component: + """A bold label above its content.""" + return Column(Strong(label), *body, gap="0.25rem") + + +def _code(text: str, *, color: str = "") -> Component: + """An inline ```` element that wraps long identifiers.""" + node = element("code").text(text).style(overflow_wrap="anywhere") + return node.style(color=color) if color else node + + +def _pre(text: str) -> Component: + """A preformatted code block.""" + return element("pre").text(text).style(**_PRE_STYLE) + + +def model_type_view(model_cls) -> Component: + """Show a Pydantic model type's name, class docstring, and fields.""" + if model_cls is None: + return Column() + + children = [Row(Strong("Type:"), WaBadge(variant="brand").text(display_as_type(model_cls)), gap="0.5rem", align="center")] + + docs = (model_cls.__doc__ or "").strip() + if docs: + children.append(_labeled("Class Documentation", _pre(docs))) + + fields = getattr(model_cls, "model_fields", {}) + if fields: + items = element("ul").style(margin="0", padding_left="18px") + for name, field in fields.items(): + entry = element("li").style(overflow_wrap="anywhere") + entry.child(_code(name, color="#0550ae")) + entry.child(Text(f" ({display_as_type(field.annotation)})")) + if field.description: + entry.child(Text(f" — {field.description}")) + items.child(entry) + children.append(_labeled("Fields", items)) + + return Column(*children, gap="0.75rem") + + +def _dependencies_view(model) -> Component: + """A bulleted list of the model's registry dependencies, or ``None`` if it has none.""" + deps = model.get_registry_dependencies() + if not deps: + return None + + rows = sorted({group[0] if len(group) == 1 else " | ".join(group) for group in deps}) + items = element("ul").style(margin="0", padding_left="18px") + for row in rows: + items.child(element("li").child(_code(row))) + return _labeled("Registry Dependencies", items) + + +def model_config_view(model, path: str = "") -> Component: + """Show instance-level metadata: registry path, description, and dependencies.""" + children = [] + + if path: + children.append(_labeled("Registry Path", _code(path))) + + description = model.meta.description.strip() if hasattr(model, "meta") and model.meta.description else "" + if description: + children.append(_labeled("Instance Description", element("div").text(description))) + + dependencies = _dependencies_view(model) + if dependencies is not None: + children.append(dependencies) + + if not children: + children.append(Text("No additional metadata.")) + + return Column(*children, gap="0.75rem") + + +def model_view(model, path: str = "") -> Component: + """A card with tabs inspecting a single ccflow model instance.""" + type_name = display_as_type(type(model)) + + tabs = Tabs(active="summary") + tabs.tab("Summary", model_config_view(model, path), name="summary") + tabs.tab("Model Type", model_type_view(type(model)), name="model-type") + if isinstance(model, ccflow.CallableModel): + tabs.tab("Context Type", model_type_view(model.context_type), name="context-type") + tabs.tab("Result Type", model_type_view(model.result_type), name="result-type") + + params = model.__pydantic_serializer__.to_python(model, fallback=str, mode="json") + tabs.tab("Parameters", _pre(json.dumps(params, indent=2, default=str)), name="parameters") + + header = Row(WaBadge(variant="brand").text(type_name), Strong(path or type_name), gap="0.5rem", align="center") + return WaCard(appearance="outlined").child(Column(header, WaDivider(), tabs, gap="0.75rem")) diff --git a/ccflow/ui/spaday/registry.py b/ccflow/ui/spaday/registry.py new file mode 100644 index 00000000..406e1266 --- /dev/null +++ b/ccflow/ui/spaday/registry.py @@ -0,0 +1,97 @@ +"""Registry browser and top-level viewer as a spaday component tree. + +Selection is driven entirely client-side through the runtime's signal store: clicking a leaf in the +``wa-tree`` (or picking it from the search ``wa-select``) writes the model's path to the ``selected`` +field, and each model's detail card is wrapped in a :class:`~spaday.components.shell.Show` that mounts +only when ``selected`` equals its path. No round-trip to Python is needed to change the selection. +""" + +from typing import List, Tuple + +from spaday import Component, Strong, Text +from spaday.actions import SetField, eq, field, lit +from spaday.components import App, Body, Column, Gutter, Main, Nav, Show, WaOption, WaSelect, WaTree, WaTreeItem + +import ccflow + +from .model import model_view + +__all__ = ("SELECTED_FIELD", "registry_store", "registry_leaves", "registry_tree", "registry_viewer") + +#: The signal-store field holding the selected model's registry path ("" when nothing is selected). +SELECTED_FIELD = "selected" + + +def registry_store() -> dict: + """The initial signal-store state the viewer is mounted with.""" + return {SELECTED_FIELD: ""} + + +def _sorted_items(registry, sort_children: bool): + """Registry entries, optionally with subregistries first and each group sorted alphabetically.""" + items = registry.models.items() + if sort_children: + items = sorted(items, key=lambda kv: (not isinstance(kv[1], ccflow.ModelRegistry), kv[0])) + return list(items) + + +def registry_leaves(registry, *, sort_children: bool = True, _prefix: str = "") -> List[Tuple[str, object]]: + """Return ``(path, model)`` for every leaf model in the registry, depth-first.""" + leaves: List[Tuple[str, object]] = [] + for name, model in _sorted_items(registry, sort_children): + path = f"{_prefix}/{name}" if _prefix else name + if isinstance(model, ccflow.ModelRegistry): + leaves.extend(registry_leaves(model, sort_children=sort_children, _prefix=path)) + else: + leaves.append((path, model)) + return leaves + + +def registry_tree(registry, *, sort_children: bool = True, _prefix: str = "") -> List[WaTreeItem]: + """Build the ``wa-tree-item`` nodes for the registry; leaf clicks select the model by path.""" + nodes: List[WaTreeItem] = [] + for name, model in _sorted_items(registry, sort_children): + path = f"{_prefix}/{name}" if _prefix else name + if isinstance(model, ccflow.ModelRegistry): + children = registry_tree(model, sort_children=sort_children, _prefix=path) + nodes.append(WaTreeItem(Text(name), *children)) + else: + nodes.append(WaTreeItem(Text(name)).on("click", SetField(SELECTED_FIELD, lit(path)))) + return nodes + + +def _placeholder() -> Component: + """The main-area hint shown when no model is selected.""" + return Column( + Strong("Select a model"), + Text("Choose a model from the registry on the left to inspect its configuration, type, and parameters."), + gap="0.5rem", + ) + + +def _search(leaves: List[Tuple[str, object]]) -> WaSelect: + """A select of every model path, two-way bound to the selection so it both jumps and reflects.""" + options = [WaOption(value="").text("— jump to a model —")] + options += [WaOption(value=path).text(path) for path, _ in sorted(leaves)] + return WaSelect(placeholder="Search / jump to model", with_clear=True).child(*options).bind("value", SELECTED_FIELD, mode="two-way") + + +def registry_viewer(registry, *, title: str = "ccflow Model Registry", browser_width: int = 400, sort_children: bool = True) -> App: + """Compose the full page: a sidebar registry tree + search, and the selected model's detail card.""" + leaves = registry_leaves(registry, sort_children=sort_children) + tree = WaTree(*registry_tree(registry, sort_children=sort_children), selection="leaf") + + sidebar = Gutter( + Column(Strong("Registry"), _search(leaves), tree, gap="0.75rem"), + width=f"{browser_width}px", + gap="0.75rem", + ) + + panels: List[Component] = [Show(_placeholder(), when=eq(field(SELECTED_FIELD), lit("")))] + for path, model in leaves: + panels.append(Show(model_view(model, path), when=eq(field(SELECTED_FIELD), lit(path)))) + + return App( + Nav(Strong(title)), + Body(sidebar, Main(Column(*panels, gap="1rem"))), + ) diff --git a/ccflow/utils/hydra.py b/ccflow/utils/hydra.py index 3f828abd..b7ce6f8c 100644 --- a/ccflow/utils/hydra.py +++ b/ccflow/utils/hydra.py @@ -349,28 +349,19 @@ def resolve_config_paths( This helper extracts the common logic for resolving config paths from either CLI arguments or default values provided by the decorated hydra.main function. - Parameters - ---------- - args - Parsed argparse namespace containing config_path and config_name attributes - config_path - Default config_path, typically from hydra.main() decorator - config_name - Default config_name, typically from hydra.main() decorator - hydra_main - The function decorated with hydra.main(). Used to resolve config_path - relative to the decorated function's file location. - - Returns - ------- - tuple - (root_config_dir, root_config_name) - - Raises - ------ - ValueError - If neither args.config_path nor hydra_main+config_path are provided - If neither args.config_name nor config_name are provided + Args: + args: Parsed argparse namespace containing config_path and config_name attributes + config_path: Default config_path, typically from hydra.main() decorator + config_name: Default config_name, typically from hydra.main() decorator + hydra_main: The function decorated with hydra.main(). Used to resolve config_path + relative to the decorated function's file location. + + Returns: + tuple: (root_config_dir, root_config_name) + + Raises: + ValueError: If neither args.config_path nor hydra_main+config_path are provided + If neither args.config_name nor config_name are provided """ if args.config_path: root_config_dir = args.config_path diff --git a/ccflow/utils/tokenize.py b/ccflow/utils/tokenize.py index 17efc584..c4feaaeb 100644 --- a/ccflow/utils/tokenize.py +++ b/ccflow/utils/tokenize.py @@ -408,11 +408,6 @@ def compute_cache_token(*, data_values: Iterable[Any] = (), behavior_classes: It ) -# --------------------------------------------------------------------------- -# Behavior hashing — bytecode-based fingerprinting of class methods -# --------------------------------------------------------------------------- - - def _unwrap_function(func: object) -> Optional[Callable]: """Unwrap descriptors and decorator chains to get the underlying function. diff --git a/pyproject.toml b/pyproject.toml index a4c441b8..41ec2545 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,9 @@ full = [ "ray", "scipy", "smart_open", + "spaday", + "starlette", + "uvicorn", "xarray", ] otel = [ @@ -96,6 +99,9 @@ develop = [ "ray", "scipy", "smart_open", + "spaday", + "starlette", + "uvicorn", "xarray", # Reporting deps "opentelemetry-api", @@ -119,6 +125,7 @@ test = [ ] [project.scripts] +ccflow-ui-spaday = "ccflow.ui.spaday.cli:main" [project.urls] Repository = "https://github.com/Point72/ccflow" From 4c583ee810003d1a42d91ed5c5de32333494577a Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 18 Jul 2026 12:45:12 -0400 Subject: [PATCH 2/2] add lazy registry models Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- ccflow/base.py | 244 +++++++++++++++++++++++- ccflow/tests/test_base_registry.py | 133 ++++++++++++- ccflow/tests/ui/spaday/test_registry.py | 19 +- ccflow/ui/spaday/model.py | 20 +- ccflow/ui/spaday/registry.py | 13 +- 5 files changed, 421 insertions(+), 8 deletions(-) diff --git a/ccflow/base.py b/ccflow/base.py index 2e49ac70..e01af7a9 100644 --- a/ccflow/base.py +++ b/ccflow/base.py @@ -7,9 +7,27 @@ import pathlib import platform import sys +import threading import warnings from types import GenericAlias, MappingProxyType -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Generic, List, Optional, Tuple, Type, TypeVar, Union, get_args, get_origin +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + Generic, + Iterator, + List, + Mapping, + Optional, + Tuple, + Type, + TypeVar, + Union, + get_args, + get_origin, +) import pydantic from packaging import version @@ -39,6 +57,7 @@ __all__ = ( "BaseModel", + "LazyRegistry", "ModelRegistry", "ModelType", "RegistryLookupContext", @@ -54,6 +73,7 @@ REGISTRY_SEPARATOR = "/" +_LAZY_REGISTRY_LOCK = threading.RLock() class RegistryKeyError(KeyError): @@ -586,6 +606,20 @@ def __getitem__(self, item) -> ModelType: else: raise KeyError(f"No registered model found by the name '{item}' in registry '{self._debug_name}'") + def __contains__(self, item: object) -> bool: + if not isinstance(item, str): + return False + if REGISTRY_SEPARATOR in item: + if "." in item: + return False + registry_name, name = item.split(REGISTRY_SEPARATOR, 1) + if registry_name == "": + registry = ModelRegistry.root() + else: + registry = self._models.get(registry_name) + return isinstance(registry, ModelRegistry) and name in registry + return item in self._models + def __iter__(self): for key, model in self._models.items(): yield key @@ -675,6 +709,210 @@ def load_config_from_path( return self.load_config(cfg, overwrite=overwrite) +class _LazyRegistryModels(Mapping[str, BaseModel]): + """Read-only direct-child view of a lazy registry.""" + + def __init__(self, registry: "LazyRegistry"): + self._registry = registry + + def __getitem__(self, name: str) -> BaseModel: + if REGISTRY_SEPARATOR in name: + raise KeyError(name) + return self._registry[name] + + def __iter__(self) -> Iterator[str]: + return iter(tuple(self._registry._entry_order)) + + def __len__(self) -> int: + return len(self._registry._entry_order) + + def __contains__(self, name: object) -> bool: + return isinstance(name, str) and REGISTRY_SEPARATOR not in name and name in self._registry._entry_order + + +class LazyRegistry(ModelRegistry): + """Registry that instantiates configured models when they are first accessed.""" + + name: str = "" + _pending: Dict[str, Any] = PrivateAttr(default_factory=dict) + _entry_order: List[str] = PrivateAttr(default_factory=list) + _lookup_registries: List[ModelRegistry] = PrivateAttr(default_factory=list) + _loading: List[str] = PrivateAttr(default_factory=list) + + @field_validator("name") + @classmethod + def _validate_name(cls, value): + return value + + def __init__(self, name: str = "", **config): + super().__init__(name=name) + lookup_registries = list(RegistryLookupContext.registry_search_paths() or []) + self._lookup_registries = [*lookup_registries, self] if lookup_registries else [self] + self._load_config(config, overwrite=False, lookup_registries=self._lookup_registries) + + def __eq__(self, other: Any) -> bool: + return ( + isinstance(other, LazyRegistry) + and self.name == other.name + and self._models == other._models + and self._pending == other._pending + and self._entry_order == other._entry_order + ) + + @model_serializer(mode="wrap") + def _lazy_registry_serializer(self, handler): + self.materialize_all() + values = handler(self) + values["models"] = self._models + return values + + @property + def models(self) -> Mapping[str, BaseModel]: + """Return a read-only view that materializes values only when accessed.""" + return _LazyRegistryModels(self) + + def is_loaded(self, name: str) -> bool: + """Return whether a direct child has been instantiated.""" + return name in self._models + + def get_loaded(self, name: str) -> Optional[BaseModel]: + """Return an instantiated direct child without loading a pending entry.""" + return self._models.get(name) + + def get_pending_config(self, name: str) -> Optional[Mapping[str, Any]]: + """Return a pending direct child's configuration without instantiating it.""" + return self._pending.get(name) + + def _load_config(self, cfg, overwrite: bool, lookup_registries: List[ModelRegistry]) -> None: + from omegaconf import DictConfig, OmegaConf + + if isinstance(cfg, DictConfig): + cfg = OmegaConf.to_container(cfg, resolve=True) + for key, value in cfg.items(): + if not isinstance(value, (dict, DictConfig)): + continue + if isinstance(value, DictConfig): + value = OmegaConf.to_container(value, resolve=True) + if _is_config_model(value): + self._add_pending(key, value, overwrite=overwrite) + elif _is_config_subregistry(value): + child = LazyRegistry(name=key) + child._lookup_registries = [*lookup_registries, child] + child._load_config(value, overwrite=False, lookup_registries=child._lookup_registries) + self.add(key, child, overwrite=overwrite) + + def _add_pending(self, name: str, cfg: Any, overwrite: bool) -> None: + if REGISTRY_SEPARATOR in name: + raise ValueError(f"Cannot add '{name}' to '{self._debug_name}' because it contains '{REGISTRY_SEPARATOR}'") + if name in self._entry_order and not overwrite: + raise ValueError(f"Cannot add '{name}' to '{self._debug_name}' as it already exists!") + if name in self._models: + ModelRegistry.remove(self, name) + if name not in self._entry_order: + self._entry_order.append(name) + self._pending[name] = cfg + + def _materialize(self, name: str) -> BaseModel: + from hydra.utils import instantiate + + with _LAZY_REGISTRY_LOCK: + if name in self._models: + return self._models[name] + if name not in self._pending: + raise KeyError(f"No registered model found by the name '{name}' in registry '{self._debug_name}'") + if name in self._loading: + cycle = " -> ".join([*self._loading, name]) + raise RegistryKeyError(f"Circular lazy registry dependency detected: {cycle}") + + self._loading.append(name) + try: + with RegistryLookupContext(registries=self._lookup_registries): + model = instantiate(self._pending[name], _convert_="all") + if not isinstance(model, BaseModel): + raise TypeError(f"Lazy registry entry '{name}' produced '{type(model)}', not a child class of {BaseModel}.") + if hasattr(model, "meta") and hasattr(model.meta, "name") and model.meta.name == "": + model.meta.name = name + ModelRegistry.add(self, name, model) + del self._pending[name] + return model + finally: + self._loading.pop() + + def add(self, name: str, model: ModelType, overwrite: bool = False) -> ModelType: + with _LAZY_REGISTRY_LOCK: + if name in self._pending: + if not overwrite: + raise ValueError(f"Cannot add '{name}' to '{self._debug_name}' as it already exists!") + del self._pending[name] + if name not in self._entry_order: + self._entry_order.append(name) + return super().add(name, model, overwrite=overwrite) + + def remove(self, name: str) -> None: + with _LAZY_REGISTRY_LOCK: + if name in self._pending: + del self._pending[name] + elif name in self._models: + super().remove(name) + else: + raise ValueError(f"Cannot remove '{name}' from '{self._debug_name}' as it does not exist there!") + self._entry_order.remove(name) + + def clear(self) -> Self: + for name in list(self._entry_order): + self.remove(name) + return self + + def __getitem__(self, item) -> ModelType: + if REGISTRY_SEPARATOR in item: + return super().__getitem__(item) + return self._materialize(item) if item in self._pending else super().__getitem__(item) + + def __contains__(self, item: object) -> bool: + if not isinstance(item, str): + return False + if REGISTRY_SEPARATOR in item: + return super().__contains__(item) + return item in self._entry_order + + def __iter__(self): + for key in tuple(self._entry_order): + yield key + model = self._models.get(key) + if isinstance(model, ModelRegistry): + for subkey in model: + yield REGISTRY_SEPARATOR.join((key, subkey)) + + def __len__(self) -> int: + count = len(self._entry_order) + for model in self._models.values(): + if isinstance(model, ModelRegistry): + count += len(model) + return count + + def load_config( + self, + cfg: "DictConfig", + overwrite: bool = False, + skip_exceptions: bool = False, + resolve_from: Optional[ModelRegistry] = None, + ) -> Self: + if skip_exceptions: + raise ValueError("skip_exceptions is not supported by LazyRegistry") + lookup_registries = [resolve_from, self] if resolve_from is not None and resolve_from is not self else self._lookup_registries + with _LAZY_REGISTRY_LOCK: + self._load_config(cfg, overwrite=overwrite, lookup_registries=lookup_registries) + return self + + def materialize_all(self) -> Self: + """Instantiate every pending model in this registry tree.""" + for name in tuple(self._entry_order): + model = self[name] + if isinstance(model, LazyRegistry): + model.materialize_all() + return self + + class RootModelRegistry(ModelRegistry): """ Class to represent the singleton, i.e. "root" ModelRegistry, @@ -773,6 +1011,8 @@ def load_config( if hasattr(model, "meta") and hasattr(model.meta, "name") and model.meta.name == "": model.meta.name = k + if isinstance(model, LazyRegistry) and not model.name: + model.name = k registries[-1].add(k, model, overwrite=self._overwrite) if not unresolved_models: @@ -866,6 +1106,8 @@ def resolve_str(v: str) -> ModelType: search_registry = search_registries[idx] try: return search_registry[v] + except RegistryKeyError: + raise except KeyError: # A common mistake is to forget to start an absolute lookup with a forward slash. Return a better error message in that case. if not original_v.startswith("/"): diff --git a/ccflow/tests/test_base_registry.py b/ccflow/tests/test_base_registry.py index 657a4167..7de0b56c 100644 --- a/ccflow/tests/test_base_registry.py +++ b/ccflow/tests/test_base_registry.py @@ -3,7 +3,7 @@ import os import pickle import sys -from typing import Dict, List +from typing import ClassVar, Dict, List from unittest import TestCase import pytest @@ -12,7 +12,7 @@ from omegaconf.errors import InterpolationKeyError from pydantic import ConfigDict -from ccflow import BaseModel, ModelRegistry, RegistryLookupContext, RootModelRegistry, model_alias +from ccflow import BaseModel, LazyRegistry, ModelRegistry, RegistryLookupContext, RootModelRegistry, model_alias from ccflow.base import RegistryKeyError, resolve_str @@ -31,6 +31,14 @@ class MyTestModelSubclass(MyTestModel): pass +class LazyTestModel(MyTestModel): + constructions: ClassVar[int] = 0 + + def __init__(self, **data): + type(self).constructions += 1 + super().__init__(**data) + + class MyClass: def __init__(self, p="p", q=10.0): self.p = p @@ -664,6 +672,127 @@ def test_load_config_resolve_from_self_noop(self): self.assertEqual(r["foo"], MyTestModel(a="test", b=0.0)) +class TestLazyRegistry(TestCase): + def setUp(self) -> None: + ModelRegistry.root().clear() + LazyTestModel.constructions = 0 + + def tearDown(self) -> None: + ModelRegistry.root().clear() + + @staticmethod + def _load_registry(): + cfg = OmegaConf.create( + { + "lazy": { + "_target_": "ccflow.base.LazyRegistry", + "_recursive_": False, + "group": { + "source": { + "_target_": "ccflow.tests.test_base_registry.LazyTestModel", + "a": "source", + "b": 1.0, + }, + "dependent": { + "_target_": "ccflow.tests.test_base_registry.MyNestedModel", + "x": "/lazy/group/source", + "y": "source", + }, + }, + } + } + ) + root = ModelRegistry.root() + root.load_config(cfg) + return root + + def test_defers_models_until_access(self): + root = self._load_registry() + lazy = root["lazy"] + + self.assertIsInstance(lazy, LazyRegistry) + self.assertEqual(lazy.name, "lazy") + self.assertEqual(LazyTestModel.constructions, 0) + self.assertIn("lazy/group/source", root) + self.assertIn("group/source", lazy) + self.assertIn("source", lazy["group"].models) + self.assertIn("lazy/group/source", list(root.keys())) + self.assertEqual(LazyTestModel.constructions, 0) + + source = root["/lazy/group/source"] + self.assertIsInstance(source, LazyTestModel) + self.assertEqual(LazyTestModel.constructions, 1) + self.assertIs(root["/lazy/group/source"], source) + self.assertEqual(LazyTestModel.constructions, 1) + + def test_materializes_dependency_closure(self): + root = self._load_registry() + + dependent = root["/lazy/group/dependent"] + + self.assertEqual(LazyTestModel.constructions, 1) + self.assertIs(dependent.x, root["/lazy/group/source"]) + self.assertIs(dependent.y, root["/lazy/group/source"]) + + def test_pending_entries_support_mutation(self): + registry = LazyRegistry( + name="lazy", + pending={ + "_target_": "ccflow.tests.test_base_registry.LazyTestModel", + "a": "pending", + "b": 1.0, + }, + ) + replacement = MyTestModel(a="replacement", b=2.0) + + with self.assertRaises(ValueError): + registry.add("pending", replacement) + registry.add("pending", replacement, overwrite=True) + self.assertIs(registry["pending"], replacement) + self.assertEqual(LazyTestModel.constructions, 0) + + registry.load_config( + OmegaConf.create( + { + "other": { + "_target_": "ccflow.tests.test_base_registry.LazyTestModel", + "a": "other", + "b": 3.0, + } + } + ) + ) + registry.remove("other") + self.assertNotIn("other", registry) + self.assertEqual(LazyTestModel.constructions, 0) + + def test_cycle_reports_dependency_chain(self): + root = ModelRegistry.root() + root.load_config( + OmegaConf.create( + { + "lazy": { + "_target_": "ccflow.base.LazyRegistry", + "_recursive_": False, + "a": { + "_target_": "ccflow.tests.test_base_registry.MyNestedModel", + "x": "b", + "y": "b", + }, + "b": { + "_target_": "ccflow.tests.test_base_registry.MyNestedModel", + "x": "a", + "y": "a", + }, + } + } + ) + ) + + with self.assertRaisesRegex(Exception, "Circular lazy registry dependency detected: a -> b -> a"): + root["/lazy/a"] + + class TestRegistryLoadingErrors(TestCase): def setUp(self) -> None: ModelRegistry.root().clear() diff --git a/ccflow/tests/ui/spaday/test_registry.py b/ccflow/tests/ui/spaday/test_registry.py index 13989c15..6d2998bf 100644 --- a/ccflow/tests/ui/spaday/test_registry.py +++ b/ccflow/tests/ui/spaday/test_registry.py @@ -2,7 +2,7 @@ from spaday.validate import validate -from ccflow import BaseModel, ModelRegistry +from ccflow import BaseModel, LazyRegistry, ModelRegistry from ccflow.ui.spaday.registry import ( SELECTED_FIELD, registry_leaves, @@ -135,3 +135,20 @@ def test_empty_registry_renders(self): node = registry_viewer(ModelRegistry(name="empty")).to_node() # Only the placeholder show panel, no model panels. assert [show_when_value(n) for n in nodes_with_tag(node, "spa-show")] == [""] + + def test_lazy_registry_renders_without_materializing_models(self): + lazy = LazyRegistry( + name="lazy", + group={ + "model": { + "_target_": "ccflow.tests.ui.spaday.test_registry.SimpleModel", + "name": "pending", + } + }, + ) + + node = registry_viewer(lazy).to_node() + + assert not lazy["group"].is_loaded("model") + show_targets = {show_when_value(item) for item in nodes_with_tag(node, "spa-show")} + assert "group/model" in show_targets diff --git a/ccflow/ui/spaday/model.py b/ccflow/ui/spaday/model.py index ba2c61c7..8901135b 100644 --- a/ccflow/ui/spaday/model.py +++ b/ccflow/ui/spaday/model.py @@ -13,7 +13,7 @@ import ccflow -__all__ = ("model_type_view", "model_config_view", "model_view") +__all__ = ("model_type_view", "model_config_view", "model_view", "pending_model_view") _PRE_STYLE = { "white_space": "pre-wrap", @@ -118,3 +118,21 @@ def model_view(model, path: str = "") -> Component: header = Row(WaBadge(variant="brand").text(type_name), Strong(path or type_name), gap="0.5rem", align="center") return WaCard(appearance="outlined").child(Column(header, WaDivider(), tabs, gap="0.75rem")) + + +def pending_model_view(config, path: str) -> Component: + """A card showing configuration for a model that has not been instantiated.""" + target = str(config.get("_target_", "Pending model")) + tabs = Tabs(active="summary") + tabs.tab( + "Summary", + Column( + _labeled("Registry Path", _code(path)), + Text("This model will be instantiated when accessed from Python."), + gap="0.75rem", + ), + name="summary", + ) + tabs.tab("Configuration", _pre(json.dumps(config, indent=2, default=str)), name="configuration") + header = Row(WaBadge(variant="neutral").text("Pending"), Strong(target), gap="0.5rem", align="center") + return WaCard(appearance="outlined").child(Column(header, WaDivider(), tabs, gap="0.75rem")) diff --git a/ccflow/ui/spaday/registry.py b/ccflow/ui/spaday/registry.py index 406e1266..519c37dc 100644 --- a/ccflow/ui/spaday/registry.py +++ b/ccflow/ui/spaday/registry.py @@ -14,7 +14,7 @@ import ccflow -from .model import model_view +from .model import model_view, pending_model_view __all__ = ("SELECTED_FIELD", "registry_store", "registry_leaves", "registry_tree", "registry_viewer") @@ -29,7 +29,13 @@ def registry_store() -> dict: def _sorted_items(registry, sort_children: bool): """Registry entries, optionally with subregistries first and each group sorted alphabetically.""" - items = registry.models.items() + if isinstance(registry, ccflow.LazyRegistry): + items = [] + for name in registry.models: + loaded = registry.get_loaded(name) + items.append((name, loaded if loaded is not None else registry.get_pending_config(name))) + else: + items = list(registry.models.items()) if sort_children: items = sorted(items, key=lambda kv: (not isinstance(kv[1], ccflow.ModelRegistry), kv[0])) return list(items) @@ -89,7 +95,8 @@ def registry_viewer(registry, *, title: str = "ccflow Model Registry", browser_w panels: List[Component] = [Show(_placeholder(), when=eq(field(SELECTED_FIELD), lit("")))] for path, model in leaves: - panels.append(Show(model_view(model, path), when=eq(field(SELECTED_FIELD), lit(path)))) + detail = pending_model_view(model, path) if isinstance(model, dict) and "_target_" in model else model_view(model, path) + panels.append(Show(detail, when=eq(field(SELECTED_FIELD), lit(path)))) return App( Nav(Strong(title)),