Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,16 +182,16 @@ Every campaign fact is recorded as an immutable, hash-chained event. The write s

| Layer | Technology |
|-------|-----------|
| Language | Python 3.11+ |
| Schemas | Pydantic v2 |
| Event store | SQLite (WAL mode, append-only) |
| Language | [![Python](https://img.shields.io/badge/Python-3.11%2B-3776AB?logo=python&logoColor=white)](https://docs.python.org/3/) |
| Schemas | [![Pydantic](https://img.shields.io/badge/Pydantic-v2-E92063?logo=pydantic&logoColor=white)](https://docs.pydantic.dev/) |
| Event store | [![SQLite](https://img.shields.io/badge/SQLite-WAL_mode-003B57?logo=sqlite&logoColor=white)](https://www.sqlite.org/docs.html) |
| Hash chain | SHA-256, linear tamper-evident chain |
| CLI | Typer |
| API | FastAPI + uvicorn |
| Analytics | numpy (core), FAISS (optional) |
| Charts | Matplotlib |
| Type checking | mypy (strict) |
| Linting | ruff |
| CLI | [![Typer](https://img.shields.io/badge/Typer-0.9%2B-009688)](https://typer.tiangolo.com/) |
| API | [![FastAPI](https://img.shields.io/badge/FastAPI-0.100%2B-009688?logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) |
| Analytics | [![NumPy](https://img.shields.io/badge/NumPy-1.24%2B-013243?logo=numpy&logoColor=white)](https://numpy.org/doc/) [![FAISS](https://img.shields.io/badge/FAISS-optional-4285F4)](https://faiss.ai/) |
| Charts | [![Matplotlib](https://img.shields.io/badge/Matplotlib-3.7%2B-11557C)](https://matplotlib.org/stable/) |
| Type checking | [![mypy](https://img.shields.io/badge/mypy-strict-blue)](https://mypy.readthedocs.io/) |
| Linting | [![Ruff](https://img.shields.io/badge/Ruff-zero_errors-D7FF64?logo=ruff&logoColor=black)](https://docs.astral.sh/ruff/) |

### Event Model

Expand Down Expand Up @@ -234,6 +234,8 @@ src/matledger/

matledger's differentiators — trajectory analytics that CRUD experiment trackers structurally cannot provide.

*BO = Bayesian optimisation — the closed-loop surrogate-driven search that matledger records.*

| Analytic | What it detects | Signal | Field relevance |
|----------|----------------|--------|----------------|
| **Proxy gaming** | Surrogate predictions rising while validated results stay flat | Goodhart's Law in action | All BO |
Expand Down
137 changes: 13 additions & 124 deletions examples/notebook_demo.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,11 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": null,
"id": "setup",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Campaign created.\n"
]
}
],
"source": [
"%matplotlib inline\n",
"from matledger.sdk import CampaignClient\n",
"\n",
"# Create an in-memory campaign (use a file path for persistence)\n",
"client = CampaignClient(\":memory:\")\n",
"client.create_campaign(\n",
" campaign_id=\"perovskite-opt\",\n",
" goal=\"maximize bandgap\",\n",
" budget_cap=50000.0,\n",
")\n",
"print(\"Campaign created.\")"
]
"outputs": [],
"source": "%matplotlib inline\nfrom matledger.sdk import CampaignClient\n\n# Create an in-memory campaign (use a file path for persistence)\nclient = CampaignClient(\":memory:\")\nclient.create_campaign(\n campaign_id=\"perovskite-opt\",\n goal=\"maximize bandgap\",\n budget_cap=50_000.0,\n)\nprint(\"Campaign created.\")"
},
{
"cell_type": "markdown",
Expand Down Expand Up @@ -83,43 +63,11 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": null,
"id": "iter1-results",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iteration 1 complete.\n"
]
}
],
"source": [
"# Start experiments (commits budget)\n",
"for cid, _, _ in candidates_iter1:\n",
" client.start_experiment(cid, cost=5000.0, iteration=1)\n",
"\n",
"# Record validated lab results\n",
"client.record_result(\"CsPbI3\", value=1.68, uncertainty=0.05, iteration=1)\n",
"client.record_result(\"CsPbBr3\", value=2.25, uncertainty=0.03, iteration=1)\n",
"client.record_result(\"MAPbI3\", value=1.50, uncertainty=0.04, iteration=1)\n",
"client.record_result(\"FAPbI3\", value=1.45, uncertainty=0.06, iteration=1)\n",
"\n",
"# CsSnI3 failed — tin oxidises in air\n",
"client.record_failure(\"CsSnI3\", failure_class=\"stability\", iteration=1,\n",
" detail=\"Sn2+ oxidation under ambient conditions\")\n",
"\n",
"# Select the best, reject the failure\n",
"client.select(\"CsPbBr3\", rationale=\"Highest validated bandgap\",\n",
" model_version=\"v1.0\", iteration=1)\n",
"client.reject(\"CsSnI3\", rationale=\"Stability failure\",\n",
" model_version=\"v1.0\", iteration=1)\n",
"\n",
"# Close iteration 1\n",
"client.close_iteration(1, summary=\"5 candidates tested, 1 stability failure\")\n",
"print(\"Iteration 1 complete.\")"
]
"outputs": [],
"source": "# Start experiments (commits budget)\nfor cid, _, _ in candidates_iter1:\n client.start_experiment(cid, cost=5_000.0, iteration=1)\n\n# Record validated lab results\nclient.record_result(\"CsPbI3\", value=1.68, uncertainty=0.05, iteration=1)\nclient.record_result(\"CsPbBr3\", value=2.25, uncertainty=0.03, iteration=1)\nclient.record_result(\"MAPbI3\", value=1.50, uncertainty=0.04, iteration=1)\nclient.record_result(\"FAPbI3\", value=1.45, uncertainty=0.06, iteration=1)\n\n# CsSnI3 failed — tin oxidises in air\nclient.record_failure(\"CsSnI3\", failure_class=\"stability\", iteration=1,\n detail=\"Sn2+ oxidation under ambient conditions\")\n\n# Select the best, reject the failure\nclient.select(\"CsPbBr3\", rationale=\"Highest validated bandgap\",\n model_version=\"v1.0\", iteration=1)\nclient.reject(\"CsSnI3\", rationale=\"Stability failure\",\n model_version=\"v1.0\", iteration=1)\n\n# Close iteration 1\nclient.close_iteration(1, summary=\"5 candidates tested, 1 stability failure\")\nprint(\"Iteration 1 complete.\")"
},
{
"cell_type": "markdown",
Expand All @@ -131,42 +79,11 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": null,
"id": "iter2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iteration 2 complete.\n"
]
}
],
"source": [
"# Surrogate retrained on iteration 1 data\n",
"client.update_model(model_version=\"v2.0\", iteration=2,\n",
" summary=\"Retrained on 4 results + 1 failure\")\n",
"\n",
"candidates_iter2 = [\n",
" (\"CsPb05Sn05I3\", {\"Cs\": 1, \"Pb\": 0.5, \"Sn\": 0.5, \"I\": 3}, 1.60),\n",
" (\"FA08Cs02PbI3\", {\"FA\": 0.8, \"Cs\": 0.2, \"Pb\": 1, \"I\": 3}, 1.52),\n",
" (\"CsPbI2Br\", {\"Cs\": 1, \"Pb\": 1, \"I\": 2, \"Br\": 1}, 1.92),\n",
"]\n",
"\n",
"for cid, features, predicted in candidates_iter2:\n",
" client.propose(cid, features, predicted, model_version=\"v2.0\", iteration=2)\n",
" client.start_experiment(cid, cost=5000.0, iteration=2)\n",
"\n",
"client.record_result(\"CsPb05Sn05I3\", value=1.55, uncertainty=0.04, iteration=2)\n",
"client.record_result(\"FA08Cs02PbI3\", value=1.58, uncertainty=0.03, iteration=2)\n",
"client.record_result(\"CsPbI2Br\", value=1.88, uncertainty=0.04, iteration=2)\n",
"\n",
"client.select(\"CsPbI2Br\", rationale=\"Best mixed-halide bandgap\",\n",
" model_version=\"v2.0\", iteration=2)\n",
"client.close_iteration(2, summary=\"3 candidates tested, all successful\")\n",
"print(\"Iteration 2 complete.\")"
]
"outputs": [],
"source": "# Surrogate retrained on iteration 1 data\nclient.update_model(model_version=\"v2.0\", iteration=2,\n summary=\"Retrained on 4 results + 1 failure\")\n\ncandidates_iter2 = [\n (\"CsPb05Sn05I3\", {\"Cs\": 1, \"Pb\": 0.5, \"Sn\": 0.5, \"I\": 3}, 1.60),\n (\"FA08Cs02PbI3\", {\"FA\": 0.8, \"Cs\": 0.2, \"Pb\": 1, \"I\": 3}, 1.52),\n (\"CsPbI2Br\", {\"Cs\": 1, \"Pb\": 1, \"I\": 2, \"Br\": 1}, 1.92),\n]\n\nfor cid, features, predicted in candidates_iter2:\n client.propose(cid, features, predicted, model_version=\"v2.0\", iteration=2)\n client.start_experiment(cid, cost=5_000.0, iteration=2)\n\nclient.record_result(\"CsPb05Sn05I3\", value=1.55, uncertainty=0.04, iteration=2)\nclient.record_result(\"FA08Cs02PbI3\", value=1.58, uncertainty=0.03, iteration=2)\nclient.record_result(\"CsPbI2Br\", value=1.88, uncertainty=0.04, iteration=2)\n\nclient.select(\"CsPbI2Br\", rationale=\"Best mixed-halide bandgap\",\n model_version=\"v2.0\", iteration=2)\nclient.close_iteration(2, summary=\"3 candidates tested, all successful\")\nprint(\"Iteration 2 complete.\")"
},
{
"cell_type": "markdown",
Expand All @@ -178,39 +95,11 @@
},
{
"cell_type": "code",
"execution_count": 36,
"execution_count": null,
"id": "iter3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iteration 3 complete.\n"
]
}
],
"source": [
"client.update_model(model_version=\"v3.0\", iteration=3,\n",
" summary=\"Retrained on 7 results + 1 failure\")\n",
"\n",
"candidates_iter3 = [\n",
" (\"CsPbIBr2\", {\"Cs\": 1, \"Pb\": 1, \"I\": 1, \"Br\": 2}, 2.10),\n",
" (\"CsPbBr2Cl\", {\"Cs\": 1, \"Pb\": 1, \"Br\": 2, \"Cl\": 1}, 2.40),\n",
"]\n",
"\n",
"for cid, features, predicted in candidates_iter3:\n",
" client.propose(cid, features, predicted, model_version=\"v3.0\", iteration=3)\n",
" client.start_experiment(cid, cost=5000.0, iteration=3)\n",
"\n",
"client.record_result(\"CsPbIBr2\", value=2.05, uncertainty=0.03, iteration=3)\n",
"client.record_result(\"CsPbBr2Cl\", value=2.35, uncertainty=0.04, iteration=3)\n",
"\n",
"client.select(\"CsPbBr2Cl\", rationale=\"Highest bandgap observed\",\n",
" model_version=\"v3.0\", iteration=3)\n",
"client.close_iteration(3, summary=\"2 candidates, converging on mixed halides\")\n",
"print(\"Iteration 3 complete.\")"
]
"outputs": [],
"source": "client.update_model(model_version=\"v3.0\", iteration=3,\n summary=\"Retrained on 7 results + 1 failure\")\n\ncandidates_iter3 = [\n (\"CsPbIBr2\", {\"Cs\": 1, \"Pb\": 1, \"I\": 1, \"Br\": 2}, 2.10),\n (\"CsPbBr2Cl\", {\"Cs\": 1, \"Pb\": 1, \"Br\": 2, \"Cl\": 1}, 2.40),\n]\n\nfor cid, features, predicted in candidates_iter3:\n client.propose(cid, features, predicted, model_version=\"v3.0\", iteration=3)\n client.start_experiment(cid, cost=5_000.0, iteration=3)\n\nclient.record_result(\"CsPbIBr2\", value=2.05, uncertainty=0.03, iteration=3)\nclient.record_result(\"CsPbBr2Cl\", value=2.35, uncertainty=0.04, iteration=3)\n\nclient.select(\"CsPbBr2Cl\", rationale=\"Highest bandgap observed\",\n model_version=\"v3.0\", iteration=3)\nclient.close_iteration(3, summary=\"2 candidates, converging on mixed halides\")\nprint(\"Iteration 3 complete.\")"
},
{
"cell_type": "markdown",
Expand Down Expand Up @@ -1054,4 +943,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
}
}
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ classifiers = [
dependencies = [
"pydantic>=2.0",
"numpy>=1.24",
"typing_extensions>=4.0; python_version < '3.12'",
]

[project.optional-dependencies]
Expand Down
7 changes: 5 additions & 2 deletions src/matledger/analytics/acquisition_efficiency.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
from dataclasses import dataclass, field
from typing import Any

from typing_extensions import override

from matledger.events.envelope import Event, EventType
from matledger.projections.framework import Projector

Expand Down Expand Up @@ -59,7 +61,7 @@ def _rank(vals: list[float]) -> list[float]:
return 1.0 - (6.0 * d_sq) / (n * (n * n - 1))


@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class AcquisitionEfficiencyVerdict:
"""Evidence-bearing verdict from the acquisition efficiency tracker.

Expand All @@ -79,7 +81,7 @@ class AcquisitionEfficiencyVerdict:
threshold: float


@dataclass
@dataclass(slots=True)
class _IterPredictions:
pairs: list[tuple[str, float]] = field(default_factory=list)

Expand All @@ -105,6 +107,7 @@ def __init__(
self._predictions: dict[int, dict[str, float]] = {}
self._results: dict[str, float] = {}

@override
def apply(self, event: Event) -> None:
if event.type == EventType.CANDIDATE_PROPOSED:
cid = event.payload.get("candidate_id", "")
Expand Down
7 changes: 5 additions & 2 deletions src/matledger/analytics/calibration.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,19 @@
from dataclasses import dataclass
from typing import Any

from typing_extensions import override

from matledger.events.envelope import Event, EventType
from matledger.projections.framework import Projector


@dataclass
@dataclass(slots=True)
class _CandidatePrediction:
predicted_value: float
iteration: int


@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class CalibrationVerdict:
"""Evidence-bearing verdict from the calibration drift detector.

Expand Down Expand Up @@ -83,6 +85,7 @@ def __init__(
self._predictions: dict[str, _CandidatePrediction] = {}
self._results: dict[str, list[tuple[float, float, int]]] = {}

@override
def apply(self, event: Event) -> None:
if event.type == EventType.CANDIDATE_PROPOSED:
cid = event.payload.get("candidate_id", "")
Expand Down
5 changes: 4 additions & 1 deletion src/matledger/analytics/data_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@
from dataclasses import dataclass
from typing import Any

from typing_extensions import override

from matledger.events.envelope import Event, EventType
from matledger.projections.framework import Projector


@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class DataQualityVerdict:
"""Evidence-bearing verdict from the data quality tracker.

Expand Down Expand Up @@ -81,6 +83,7 @@ def __init__(
self._min_observations = min_observations
self._values_by_iter: dict[int, list[float]] = {}

@override
def apply(self, event: Event) -> None:
if event.type == EventType.RESULT_RECORDED:
val = event.payload.get("value")
Expand Down
38 changes: 21 additions & 17 deletions src/matledger/analytics/discovery_rate.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
from dataclasses import dataclass, field
from typing import Any

from typing_extensions import override

from matledger.events.envelope import Event, EventType
from matledger.events.failures import FailureClass
from matledger.projections.framework import Projector
Expand All @@ -36,7 +38,7 @@
)


@dataclass
@dataclass(slots=True)
class IterationDiscoveryStats:
"""Discovery statistics for a single iteration."""

Expand Down Expand Up @@ -69,7 +71,7 @@ def cost_per_result(self) -> float:
return self.cost / self.results


@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class DiscoveryRateVerdict:
"""Summary verdict on campaign discovery efficiency.

Expand Down Expand Up @@ -109,27 +111,29 @@ def _get_or_create(self, iteration: int) -> IterationDiscoveryStats:
self._iterations[iteration] = IterationDiscoveryStats(iteration=iteration)
return self._iterations[iteration]

@override
def apply(self, event: Event) -> None:
stats = self._get_or_create(event.iteration)

if event.type == EventType.EXPERIMENT_STARTED:
stats.cost += event.payload.get("cost", 0.0)
match event.type:
case EventType.EXPERIMENT_STARTED:
stats.cost += event.payload.get("cost", 0.0)

elif event.type == EventType.RESULT_RECORDED:
stats.results += 1
case EventType.RESULT_RECORDED:
stats.results += 1

elif event.type == EventType.FAILURE_RECORDED:
stats.failures += 1
fc = event.payload.get("failure_class", "unknown")
stats.failure_classes[fc] = stats.failure_classes.get(fc, 0) + 1
if fc in _DATA_QUALITY_FAILURES:
stats.data_quality_failures += 1
case EventType.FAILURE_RECORDED:
stats.failures += 1
fc = event.payload.get("failure_class", "unknown")
stats.failure_classes[fc] = stats.failure_classes.get(fc, 0) + 1
if fc in _DATA_QUALITY_FAILURES:
stats.data_quality_failures += 1

elif event.type == EventType.CANDIDATE_SELECTED:
stats.selections += 1
case EventType.CANDIDATE_SELECTED:
stats.selections += 1

elif event.type == EventType.ITERATION_CLOSED:
self._closed_iterations.add(event.iteration)
case EventType.ITERATION_CLOSED:
self._closed_iterations.add(event.iteration)

def reset(self) -> None:
self._iterations.clear()
Expand All @@ -156,7 +160,7 @@ def get_state(self) -> dict[str, Any]:

def analyze(self) -> DiscoveryRateVerdict:
"""Compute the discovery rate verdict."""
all_iters = set(it for it in self._iterations if it > 0)
all_iters = {it for it in self._iterations if it > 0}
all_iters |= self._closed_iterations
sorted_iters = sorted(all_iters)

Expand Down
Loading