From dc7d5e2672e048ac9344b9a7dc3608ae77bb6cf7 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 20:10:00 -0300 Subject: [PATCH 01/29] chore(workspace): hoist [workspace.dependencies] + [workspace.lints] Adds a `[workspace.dependencies]` table to the root manifest consolidating every dep used by 2+ crates across the full nullis- shepherd stack (anyhow, thiserror, tokio, futures, serde, serde_json, tracing, tracing-subscriber, strum, alloy-*, cowprotocol, reqwest, wit-bindgen, clap). Per-crate manifests inherit with `dep.workspace = true`, and may add features per call site via `dep = { workspace = true, features = ["extra"] }`. Single-consumer deps (wasmtime, toml, redb, getrandom, url, hex, axum, rand, ...) stay per-crate. Adds `[workspace.lints]` with light-touch defaults: `dbg_macro` and `todo` denied via clippy, `unsafe_op_in_unsafe_fn` warned via rust. `unsafe_code = deny` cannot be applied workspace-wide because every wit-bindgen guest module emits an `unsafe extern "C"` shim. Also pre-declares `auto_impl` and `derive_more` in the workspace deps table so future `Arc` boundaries and newtype-heavy crates can opt in without touching the root manifest. The version-drift failure mode (cowprotocol pinned to `1.0.0-alpha` in nexum-engine but `1.0.0-alpha.3` in shepherd-sdk, flagged in the 2026-06-25 audit) is now impossible by construction: every consumer inherits the single workspace pin. Audit reference: milestone-rubric-grant-audit-2026-06-25.md, judgment calls 1 + 3. --- Cargo.toml | 85 ++++++++++++++++++++++++++++++++++ crates/nexum-engine/Cargo.toml | 9 ++-- modules/example/Cargo.toml | 5 +- 3 files changed, 95 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d14c23e..4155a40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,91 @@ edition = "2024" license = "AGPL-3.0" repository = "https://github.com/nullisLabs/shepherd" +# Shared dependency table. Only deps consumed by 2+ crates across the +# full workspace (nexum-engine + every downstream module crate) are +# hoisted here; single-consumer deps stay per-crate. Crates inherit +# with `dep.workspace = true` and may add features per call site via +# `dep = { workspace = true, features = ["extra"] }`. Version drift +# across crates (the failure mode that prompted hoisting in the first +# place, e.g. cowprotocol on `1.0.0-alpha` vs `1.0.0-alpha.3`) is now +# impossible by construction. +[workspace.dependencies] +# Error + async plumbing. +anyhow = "1" +thiserror = "2" +tokio = { version = "1", features = ["full"] } +futures = "0.3" + +# Serde + config. +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +# Observability. +tracing = "0.1" +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "ansi", "json"] } + +# `strum::IntoStaticStr` on every error / event enum gives a free +# snake_case `&'static str` for every variant, which feeds directly +# into `metrics::counter!(..., "error_kind" => name)` and +# `tracing::warn!(error_kind = name, ...)` recordings without an +# ad-hoc `match err { ... => "connect" ... }` ladder per call site. +strum = { version = "0.26", features = ["derive"] } + +# `auto_impl::auto_impl(&, Arc, Box)` forwarding impls for traits +# held through smart pointers. Available workspace-wide so any future +# `Arc` boundary can opt in without touching root manifest. +auto_impl = "1" + +# `derive_more` newtype boilerplate (`Deref`, `From`, `Display`, ...). +# `default-features = false, features = ["full"]` keeps the proc-macro +# surface predictable; per-derive opt-in via the standard `#[derive(...)]` +# syntax. Available workspace-wide; not pulled in by default. +derive_more = { version = "1", default-features = false, features = ["full"] } + +# CLI parser. Used by every binary crate (engine, load-gen, +# orderbook-mock, shepherd-backtest) via the derive macro. +clap = { version = "4", features = ["derive"] } + +# alloy stack. Engine uses the full provider/transport surface; +# guest-facing crates use `alloy-primitives` + `alloy-sol-types` for +# typed protocol values. Pinned together so a single workspace bump +# moves every consumer at once. +alloy-primitives = { version = "1.5", default-features = false, features = ["std", "serde"] } +alloy-sol-types = { version = "1.5", default-features = false, features = ["std"] } +alloy-provider = { version = "1.5", default-features = false, features = ["ws", "ipc", "pubsub", "reqwest"] } +alloy-rpc-types-eth = { version = "1.5", default-features = false, features = ["std"] } +alloy-transport-ws = { version = "1.5", default-features = false } + +# CoW Protocol bindings. Pinned to one version across the workspace +# (was `1.0.0-alpha` in engine vs `1.0.0-alpha.3` in SDK before +# hoisting). +cowprotocol = { version = "1.0.0-alpha.3", default-features = false } + +# HTTP transport for `cow_api::request` REST passthrough and the +# orderbook-mock test surface. +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } + +# `wit-bindgen` is consumed by every guest module crate (example + +# every strategy + every fixture). Hoisted so a single bump moves +# them in lock-step. +wit-bindgen = { version = "0.58", default-features = false, features = ["macros", "realloc"] } + +# Workspace-standard lint set. New crates inherit via +# `[lints] workspace = true` in their package manifest. `unsafe_code` +# cannot be denied workspace-wide because every wit-bindgen guest +# module emits an `unsafe extern "C"` shim; modules carrying that +# macro keep the default-warn allowance, and unsafe in non-binding +# code still trips review by convention. +[workspace.lints.rust] +unsafe_op_in_unsafe_fn = "warn" + +[workspace.lints.clippy] +# Deny the easy footguns. Each crate carries its own narrower +# `#![deny(...)]` where the cost of a violation is high (e.g. the +# binary entrypoints carry `unused_crate_dependencies` warn). +dbg_macro = "deny" +todo = "deny" + [profile.dev] panic = "abort" diff --git a/crates/nexum-engine/Cargo.toml b/crates/nexum-engine/Cargo.toml index 65768c3..bf5cc27 100644 --- a/crates/nexum-engine/Cargo.toml +++ b/crates/nexum-engine/Cargo.toml @@ -5,11 +5,14 @@ edition.workspace = true license.workspace = true repository.workspace = true +[lints] +workspace = true + [dependencies] wasmtime = { version = "45", features = ["component-model"] } wasmtime-wasi = "45" -anyhow = "1" -tokio = { version = "1", features = ["full"] } +anyhow.workspace = true +tokio.workspace = true getrandom = "0.4" -serde = { version = "1", features = ["derive"] } +serde.workspace = true toml = "1" diff --git a/modules/example/Cargo.toml b/modules/example/Cargo.toml index eea9911..9039003 100644 --- a/modules/example/Cargo.toml +++ b/modules/example/Cargo.toml @@ -5,8 +5,11 @@ edition.workspace = true license.workspace = true repository.workspace = true +[lints] +workspace = true + [lib] crate-type = ["cdylib"] [dependencies] -wit-bindgen = { version = "0.58", default-features = false, features = ["macros", "realloc"] } +wit-bindgen.workspace = true From a2b6f4bfda36bcc8a6590e881086ecc2d148477f Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 20:27:10 -0300 Subject: [PATCH 02/29] feat(nexum-engine): migrate CLI from hand-rolled parser to clap Replaces the `std::env::args().skip(1)` walker with a `#[derive(clap:: Parser)]` struct so the engine binary picks up `--help`, `--version`, proper argument validation, and structured error reporting for free. The positional surface is preserved one-for-one (` [manifest-path]`); behaviour for callers that already pass two paths is identical. Help output now documents each argument inline rather than hiding the usage in an anyhow message that only fires on misuse. `clap.workspace = true` consumes the workspace dep added in the prior commit; no new direct version pin in this crate. Audit reference: milestone-rubric-grant-audit-2026-06-25.md, judgment call 2. --- crates/nexum-engine/Cargo.toml | 1 + crates/nexum-engine/src/main.rs | 48 +++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/crates/nexum-engine/Cargo.toml b/crates/nexum-engine/Cargo.toml index bf5cc27..926b11d 100644 --- a/crates/nexum-engine/Cargo.toml +++ b/crates/nexum-engine/Cargo.toml @@ -12,6 +12,7 @@ workspace = true wasmtime = { version = "45", features = ["component-model"] } wasmtime-wasi = "45" anyhow.workspace = true +clap.workspace = true tokio.workspace = true getrandom = "0.4" serde.workspace = true diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index f013f22..ec09672 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -2,11 +2,39 @@ mod manifest; use std::path::PathBuf; use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use clap::Parser; use wasmtime::component::{Component, Linker, ResourceTable}; use wasmtime::error::Context as _; use wasmtime::{Engine, Store}; use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; +/// Reference CLI for the 0.2 `nexum-engine` runtime. +/// +/// Loads a Wasm Component, links the `shepherd:cow/shepherd` host +/// world plus the WASI p2 set, calls `init` once, then dispatches a +/// single synthetic block event so the host stubs exercise their +/// timing paths. Production deployments invoke the engine through +/// the supervisor entrypoint introduced in later milestones; this +/// CLI is the M1 smoke-test surface. +#[derive(Parser, Debug)] +#[command( + name = "nexum-engine", + about = "Load a Wasm Component and dispatch a synthetic block event", + long_about = None, + version, +)] +struct Cli { + /// Path to the Wasm Component file to load. + wasm_path: PathBuf, + + /// Optional explicit path to the module's `nexum.toml` manifest. + /// When omitted, the engine looks for `nexum.toml` next to the + /// component file and falls back to a permissive default (with + /// a deprecation warning) when none is found. + manifest_path: Option, +} + // Both packages are listed explicitly so wit-parser can resolve the // cross-package reference natively — no vendored deps/ tree needed. // World name is fully qualified. @@ -339,22 +367,20 @@ impl nexum::host::http::Host for HostState { #[tokio::main] async fn main() -> anyhow::Result<()> { - let mut args = std::env::args().skip(1); - let wasm_path = args.next().ok_or_else(|| { - anyhow::anyhow!("usage: nexum-engine []") - })?; - let explicit_manifest = args.next().map(PathBuf::from); + let cli = Cli::parse(); + let wasm_path = cli.wasm_path; + let explicit_manifest = cli.manifest_path; - println!("nexum-engine: loading component from {wasm_path}"); + println!( + "nexum-engine: loading component from {}", + wasm_path.display() + ); // Load the manifest from the explicit path if given, otherwise from // `nexum.toml` next to the component file. Missing → fallback (with // deprecation warning). - let manifest_path = explicit_manifest.or_else(|| { - PathBuf::from(&wasm_path) - .parent() - .map(|p| p.join("nexum.toml")) - }); + let manifest_path = + explicit_manifest.or_else(|| wasm_path.parent().map(|p| p.join("nexum.toml"))); let loaded = match manifest_path.as_deref() { Some(p) if p.exists() => { println!("nexum-engine: loading manifest from {}", p.display()); From 747efdbab4e6bcc9f72188891782ed3e12027ffd Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 20:28:03 -0300 Subject: [PATCH 03/29] docs(07-rpc-namespace-design): mark allowlist enforcement as future direction A casual reader of `07-rpc-namespace-design.md` hitting the file top or the "Method Allowlisting" subsection could plausibly walk away believing the 0.2 runtime gates RPC methods on a read-only allowlist and intercepts signing methods to delegate them to the identity backend. The shipped host implementation does neither: `chain::request` forwards any method string through to the configured alloy provider. Adds an explicit `Status: Future direction (0.3+ target)` callout both at the file top and right above the "Method Allowlisting" subsection so the gap between design intent and shipped behaviour is visible without having to scroll the design narrative end-to-end. Audit reference: milestone-rubric-grant-audit-2026-06-25.md, judgment call 4. --- docs/07-rpc-namespace-design.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/07-rpc-namespace-design.md b/docs/07-rpc-namespace-design.md index 8f443a3..8cd1c93 100755 --- a/docs/07-rpc-namespace-design.md +++ b/docs/07-rpc-namespace-design.md @@ -1,5 +1,15 @@ # RPC Namespace Design: Generic JSON-RPC Passthrough +> **Status: Future direction (0.3+ target).** The 0.2 reference runtime +> ships the single `chain::request(chain_id, method, params)` WIT entry +> point and forwards any method to the configured alloy provider without +> a per-module allowlist. The "Method Allowlisting" section below +> describes the **intended** 0.3+ enforcement (read-only default set, +> per-module `[module.chain] extra_allowed_methods`, identity-delegated +> signing methods); none of it is wired into `chain::request` in the +> shipped binary. Treat the section as design intent until a tracking +> issue lands the runtime check. + > **Naming note (0.2):** This document describes the `chain` interface in the > `nexum:host` WIT package. In the 0.1 design history it was called `chain` > (short for "consensus"); 0.2 renamed it to `chain` because `chain.request(...)` @@ -212,6 +222,15 @@ That's it. The alloy provider already has the timeout/retry/rate-limit/fallback ### Method Allowlisting +> **Status: Future direction (0.3+ target).** The shipped 0.2 host +> implementation of `chain::request` forwards any method string to +> the alloy provider; it does **not** consult a read-only allowlist +> and it does **not** intercept signing methods to delegate to the +> identity backend. The categorisation below is the planned 0.3 +> enforcement model. Until that tracking issue lands the gating +> code, operators must treat any chain-capable module as having +> access to the full RPC surface their configured provider exposes. + The host maintains two categories of methods: **read-only methods** (always allowed through the RPC passthrough) and **signing methods** (delegated to the `identity` backend). #### Read-Only Methods (RPC Passthrough) From ef9164744a89ebb75cee645e24715ab06c90c21b Mon Sep 17 00:00:00 2001 From: brunota20 Date: Mon, 1 Jun 2026 14:19:42 -0300 Subject: [PATCH 04/29] chore(deps): pull cowprotocol, alloy, redb, reqwest, tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the dependencies the 0.2 host backends need: - cowprotocol (1.0.0-alpha) for the cow-api submission path (OrderBookApi, OrderCreation, OrderUid, Chain). - alloy-provider / -rpc-client / -transport-ws / -primitives (1.5) for the chain JSON-RPC dispatch. The reqwest feature on alloy-provider engages connect_http; the pubsub/ws features back eth_subscribe-class methods. - redb (2) for local-store. Same crate cowprotocol's own watch-tower picked, so the dep tree does not bifurcate when both are used in the same workspace. - reqwest (0.12, rustls-tls) — direct, so the import survives any future cowprotocol feature rearrangement. - tracing + tracing-subscriber (env-filter + fmt) — replaces the 0.1 eprintln! debug log so the engine can drop into a structured log pipeline without re-instrumenting every host call. - thiserror (2) — typed error enums in each backend. - tempfile + wiremock as dev-deps for the host backend tests. Adds engine.example.toml documenting the [engine] state_dir + per- chain RPC URLs the chain backend reads at boot; data/ is now ignored so a local run does not leave the redb file in tree. --- .gitignore | 1 + crates/nexum-engine/Cargo.toml | 48 ++++++++++++++++++++++++++++++++-- engine.example.toml | 34 ++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 engine.example.toml diff --git a/.gitignore b/.gitignore index 357bddc..e43a15c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ Thumbs.db # Environment .env .env.* +data/ diff --git a/crates/nexum-engine/Cargo.toml b/crates/nexum-engine/Cargo.toml index 926b11d..9d450b1 100644 --- a/crates/nexum-engine/Cargo.toml +++ b/crates/nexum-engine/Cargo.toml @@ -9,11 +9,55 @@ repository.workspace = true workspace = true [dependencies] +# WASM Component Model runtime. wasmtime = { version = "45", features = ["component-model"] } wasmtime-wasi = "45" + +# Async + error plumbing. anyhow.workspace = true -clap.workspace = true +thiserror.workspace = true tokio.workspace = true -getrandom = "0.4" +clap.workspace = true + +# Manifest parsing. serde.workspace = true toml = "1" +serde_json.workspace = true + +# Observability. `tracing` replaces the prior `eprintln!` debug log +# so the engine can drop into a structured log pipeline in production. +tracing.workspace = true +tracing-subscriber = { workspace = true, default-features = false, features = ["fmt", "env-filter", "ansi"] } + +# `cow-api` backend. cowprotocol pulls `OrderBookApi`, `OrderCreation`, +# `OrderUid`, the orderbook base URL table per `Chain`, and the typed +# error surface the host re-projects into `HostError`. Pinned via the +# workspace so every crate that touches cowprotocol moves in lockstep. +cowprotocol.workspace = true +# REST passthrough for `cow_api::request`. cowprotocol pulls reqwest +# transitively for its own client; we depend on it directly so the +# import is explicit and survives any future cowprotocol feature +# rearrangement. +reqwest.workspace = true + +# `chain` backend. Each configured chain owns a `DynProvider` built +# from a `WsConnect`/`Http` transport so the host's `request` / +# `request-batch` impls can hand a raw `(method, params)` pair to +# alloy's JSON-RPC layer without reimplementing the codec. +alloy-provider.workspace = true +alloy-rpc-client = { version = "1.5", default-features = false } +alloy-transport = { version = "1.5", default-features = false } +alloy-transport-ws.workspace = true +alloy-primitives.workspace = true + +# `local-store` backend. Per-module namespacing is enforced +# host-side via a `[len:u8][module_name][raw_key]` prefix. +redb = "2" + +# Misc. +getrandom = "0.4" +url = "2" + +[dev-dependencies] +tempfile = "3" +wiremock = "0.6" diff --git a/engine.example.toml b/engine.example.toml new file mode 100644 index 0000000..d6513b0 --- /dev/null +++ b/engine.example.toml @@ -0,0 +1,34 @@ +# Engine-side runtime configuration for `nexum-engine`. +# +# Distinct from `nexum.toml` (per-module manifest): this file +# describes the *engine*'s I/O wiring. Copy to `engine.toml` next to +# the binary, or pass the path as the third positional argument. + +[engine] +# Directory the local-store redb file (and future engine artefacts) +# will be created under. Created automatically at boot. +state_dir = "./data" + +# `tracing_subscriber::EnvFilter`-compatible directive. `RUST_LOG` +# overrides at process start. +log_level = "info" + +# One [chains.] table per chain the engine should be able to talk +# to. Chain ids are EVM decimal. `ws://` and `wss://` URLs engage +# alloy's pubsub transport (needed for `eth_subscribe`); `http://` and +# `https://` use the HTTP transport. + +[chains.1] +rpc_url = "https://ethereum-rpc.publicnode.com" + +[chains.100] +rpc_url = "https://rpc.gnosischain.com" + +[chains.11155111] +rpc_url = "wss://ethereum-sepolia-rpc.publicnode.com" + +[chains.42161] +rpc_url = "https://arb1.arbitrum.io/rpc" + +[chains.8453] +rpc_url = "https://mainnet.base.org" From d578626ecda8cf22e66770b48d6d573d1a483e88 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Mon, 1 Jun 2026 14:20:01 -0300 Subject: [PATCH 05/29] runtime: implement cow-api, chain, local-store host backends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the 0.2 Unsupported stubs with working backends. Each capability lives in its own host submodule so the trait impls in main.rs stay thin (dispatch + project the backend's typed error onto HostError). cow_api::submit_order - Parses the guest's bytes as JSON cowprotocol::OrderCreation. - Dispatches via cowprotocol::OrderBookApi::post_order. - Returns the assigned OrderUid as a 0x-prefixed hex string. cow_api::request - REST passthrough. The base URL is whichever URL the pool's OrderBookApi client carries — so OrderBookApi::new_with_base_url overrides (staging, wiremock) flow through transparently. - Method/path validated host-side; orderbook 4xx/5xx bodies are surfaced verbatim so the guest can decode {errorType,description}. chain::request - Raw JSON-RPC dispatch over an alloy DynProvider opened from engine.toml at boot. WebSocket URLs engage pubsub (eth_subscribe); HTTP URLs use the HTTP transport. Params are passed as serde_json::RawValue so alloy does not re-encode. - request-batch falls back to per-call dispatch (same shape as the earlier stub but now backed by real RPC). local_store - redb file under engine_config.engine.state_dir. - Single shared table. Per-module namespacing is enforced host-side via [len:u8][module_name][raw_key] prefix on every key. list_keys strips the prefix before returning to the guest. logging - Routes through tracing::event! tagged with module=. - Engine boot installs an EnvFilter-based subscriber; RUST_LOG overrides the engine.toml log_level. identity / remote-store / messaging / http stay at Unsupported per the 0.2 roadmap (keystore / Swarm / Waku land in 0.3). Tests (14, all green): - cow_orderbook: pool default chains, unknown-chain typing, REST GET passthrough, relative-path resolution, unknown-method rejection, submit_order round-trip — last three under wiremock so the full HTTP path is exercised without hitting api.cow.fi. - provider_pool: empty pool surfaces UnknownChain. - local_store: roundtrip, namespace isolation, delete, list_keys prefix-stripping, empty-namespace rejection. End-to-end against modules/example: example.wasm loads under the new wiring, logs init + on_event through the tracing pipeline. --- Cargo.toml | 7 +- crates/nexum-engine/src/engine_config.rs | 98 +++++ crates/nexum-engine/src/host/cow_orderbook.rs | 294 +++++++++++++ .../nexum-engine/src/host/local_store_redb.rs | 211 +++++++++ crates/nexum-engine/src/host/mod.rs | 12 + crates/nexum-engine/src/host/provider_pool.rs | 152 +++++++ crates/nexum-engine/src/main.rs | 406 ++++++++++++------ 7 files changed, 1035 insertions(+), 145 deletions(-) create mode 100644 crates/nexum-engine/src/engine_config.rs create mode 100644 crates/nexum-engine/src/host/cow_orderbook.rs create mode 100644 crates/nexum-engine/src/host/local_store_redb.rs create mode 100644 crates/nexum-engine/src/host/mod.rs create mode 100644 crates/nexum-engine/src/host/provider_pool.rs diff --git a/Cargo.toml b/Cargo.toml index 4155a40..4652c25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,8 +67,11 @@ alloy-transport-ws = { version = "1.5", default-features = false } # CoW Protocol bindings. Pinned to one version across the workspace # (was `1.0.0-alpha` in engine vs `1.0.0-alpha.3` in SDK before -# hoisting). -cowprotocol = { version = "1.0.0-alpha.3", default-features = false } +# hoisting). Default features stay on so the engine picks up +# `http-client` for `OrderBookApi`; guest-side consumers (SDK, +# strategies) opt out with `default-features = false` for the +# `cdylib` wasm-target builds. +cowprotocol = "1.0.0-alpha.3" # HTTP transport for `cow_api::request` REST passthrough and the # orderbook-mock test surface. diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs new file mode 100644 index 0000000..4ec271c --- /dev/null +++ b/crates/nexum-engine/src/engine_config.rs @@ -0,0 +1,98 @@ +//! Engine-side runtime configuration. +//! +//! Distinct from `nexum.toml` (module manifest): this file describes +//! the *engine*'s I/O wiring — chain RPC endpoints and the on-disk +//! location of the `local-store` database. Both are required for the +//! 0.2 reference engine to do anything other than print stubs. +//! +//! Lookup order: +//! +//! 1. `--engine-config ` CLI flag (future), or third positional +//! argument today; +//! 2. `engine.toml` in the current working directory; +//! 3. defaults — no chains configured, `state_dir = ./data`. +//! +//! A missing config is OK for the example module (it only logs); for +//! the cow-api / chain backends it surfaces as `HostError { +//! kind: unsupported }` so guests learn early. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use tracing::{info, warn}; + +/// Engine-side configuration loaded from `engine.toml`. +#[derive(Debug, Default, Deserialize)] +pub struct EngineConfig { + #[serde(default)] + pub engine: EngineSection, + /// Per-chain RPC URLs keyed by EVM chain id (decimal in TOML). + /// Used by the `chain::request` host call and as the alloy provider + /// pool seed. + #[serde(default)] + pub chains: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub struct EngineSection { + #[serde(default = "default_state_dir")] + pub state_dir: PathBuf, + /// `tracing_subscriber::EnvFilter`-compatible directive. Defaults to + /// `info` when absent; `RUST_LOG` overrides at process start. + #[serde(default = "default_log_level")] + pub log_level: String, +} + +impl Default for EngineSection { + fn default() -> Self { + Self { + state_dir: default_state_dir(), + log_level: default_log_level(), + } + } +} + +#[derive(Debug, Deserialize)] +pub struct ChainConfig { + /// JSON-RPC endpoint. `ws://` and `wss://` engage alloy's pubsub + /// transport (required for `eth_subscribe`); `http://` and `https://` + /// engage the HTTP transport (request/response only). + pub rpc_url: String, +} + +fn default_state_dir() -> PathBuf { + PathBuf::from("./data") +} + +fn default_log_level() -> String { + "info".to_owned() +} + +/// Read an engine config from disk, returning defaults if the file is +/// missing. Parse errors propagate. +pub fn load_or_default(path: Option<&Path>) -> anyhow::Result { + let path = match path { + Some(p) => p.to_path_buf(), + None => PathBuf::from("engine.toml"), + }; + + if !path.exists() { + warn!( + path = %path.display(), + "engine.toml not found — running with defaults (no chain RPC endpoints; \ + chain::request and cow_api::submit_order will return Unsupported)" + ); + return Ok(EngineConfig::default()); + } + + let raw = std::fs::read_to_string(&path)?; + let cfg: EngineConfig = toml::from_str(&raw)?; + info!( + path = %path.display(), + chains = cfg.chains.len(), + state_dir = %cfg.engine.state_dir.display(), + "engine config loaded", + ); + Ok(cfg) +} diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs new file mode 100644 index 0000000..cd11173 --- /dev/null +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -0,0 +1,294 @@ +//! `shepherd:cow/cow-api` backend. +//! +//! Two responsibilities: +//! +//! 1. `request` — generic REST passthrough. Module gives the HTTP +//! method, path (relative to the chain's orderbook base URL), and +//! optional JSON body. We dispatch via `reqwest`, return the +//! response body verbatim. +//! 2. `submit_order` — typed submission. Module gives a JSON-encoded +//! `cowprotocol::OrderCreation`; we parse, dispatch via +//! `cowprotocol::OrderBookApi::post_order`, return the assigned +//! `OrderUid` as a `0x`-prefixed hex string. +//! +//! Per-chain `OrderBookApi` instances are constructed once at engine +//! boot from the discriminated chain set in `cowprotocol::Chain`. +//! Chains the SDK does not know about return `Unsupported` at the +//! host call boundary. + +use std::collections::BTreeMap; + +use cowprotocol::{Chain, OrderBookApi, OrderCreation, OrderUid}; +use thiserror::Error; + +/// Process-wide pool of `OrderBookApi` clients keyed by EVM chain id. +#[derive(Debug, Clone)] +pub struct OrderBookPool { + clients: BTreeMap, + http: reqwest::Client, +} + +impl OrderBookPool { + /// Build a pool covering every `cowprotocol::Chain` variant. The + /// default `OrderBookApi::new(chain)` constructor uses the canonical + /// `api.cow.fi/{slug}/api/v1` base URL from the SDK; callers that + /// need barn or a custom staging URL override per chain. + pub fn with_default_chains() -> Self { + let http = reqwest::Client::new(); + let chains = [ + Chain::Mainnet, + Chain::Gnosis, + Chain::Sepolia, + Chain::ArbitrumOne, + Chain::Base, + ]; + let clients = chains + .iter() + .map(|c| (c.id(), OrderBookApi::new(*c))) + .collect(); + Self { clients, http } + } + + /// Look up the client for a chain. + pub fn get(&self, chain_id: u64) -> Result<&OrderBookApi, CowApiError> { + self.clients + .get(&chain_id) + .ok_or(CowApiError::UnknownChain(chain_id)) + } + + /// REST passthrough. The base URL is whichever URL the pool's + /// `OrderBookApi` client carries — overrides set via + /// `OrderBookApi::new_with_base_url` (staging, wiremock) flow + /// through here too, which keeps the passthrough and the typed + /// `submit_order_json` path aimed at the same orderbook. + pub async fn request( + &self, + chain_id: u64, + method: &str, + path: &str, + body: Option<&str>, + ) -> Result { + let api = self.get(chain_id)?; + let base = api.base_url().clone(); + // `path` may or may not lead with a slash; `Url::join` handles + // both, but we strip a single leading `/` so consumers can + // write either `/orders/...` or `orders/...` interchangeably. + let trimmed = path.strip_prefix('/').unwrap_or(path); + let url = base + .join(trimmed) + .map_err(|e| CowApiError::BadPath(format!("{path:?}: {e}")))?; + + let request = match method.to_ascii_uppercase().as_str() { + "GET" => self.http.get(url), + "POST" => self.http.post(url), + "PUT" => self.http.put(url), + "DELETE" => self.http.delete(url), + other => return Err(CowApiError::BadMethod(other.to_owned())), + }; + let request = if let Some(body) = body { + request + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(body.to_owned()) + } else { + request + }; + + let response = request.send().await.map_err(CowApiError::Network)?; + // Surface the orderbook's structured 4xx / 5xx bodies verbatim + // so the guest can decode `{"errorType": "...", "description": + // "..."}` — projecting them into HostError here loses the + // detail the guest needs to recover. + let text = response.text().await.map_err(CowApiError::Network)?; + Ok(text) + } + + /// Typed submission. `body` is the JSON encoding of + /// `cowprotocol::OrderCreation`. The chain's orderbook validates + /// `from`, the EIP-712 hash, and (if `Eip1271`) the contract + /// signature; we return whatever UID it assigns. + pub async fn submit_order_json( + &self, + chain_id: u64, + body: &[u8], + ) -> Result { + let creation: OrderCreation = serde_json::from_slice(body).map_err(CowApiError::Decode)?; + let api = self.get(chain_id)?; + let uid = api + .post_order(&creation) + .await + .map_err(|e| CowApiError::Orderbook(e.to_string()))?; + Ok(uid) + } +} + +#[derive(Debug, Error)] +pub enum CowApiError { + #[error("unknown chain {0} (no cowprotocol::Chain variant)")] + UnknownChain(u64), + #[error("bad HTTP method `{0}` (expected GET/POST/PUT/DELETE)")] + BadMethod(String), + #[error("invalid path: {0}")] + BadPath(String), + #[error("network: {0}")] + Network(#[from] reqwest::Error), + #[error("decode OrderCreation JSON: {0}")] + Decode(#[from] serde_json::Error), + #[error("orderbook rejected: {0}")] + Orderbook(String), +} + +#[cfg(test)] +mod tests { + use super::*; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + #[test] + fn pool_indexes_default_chains() { + let pool = OrderBookPool::with_default_chains(); + assert!(pool.get(1).is_ok(), "mainnet present"); + assert!(pool.get(100).is_ok(), "gnosis present"); + assert!(pool.get(11_155_111).is_ok(), "sepolia present"); + assert!(pool.get(42_161).is_ok(), "arbitrum present"); + assert!(pool.get(8_453).is_ok(), "base present"); + } + + #[test] + fn unknown_chain_surfaces_typed_error() { + let pool = OrderBookPool::with_default_chains(); + assert!(matches!( + pool.get(99_999), + Err(CowApiError::UnknownChain(99_999)) + )); + } + + /// Build a pool whose Mainnet entry points at `mock.uri()`. + /// `OrderBookApi::new_with_base_url` ships in cowprotocol; we + /// rely on it so wiremock-driven tests can exercise the full + /// request path without re-implementing the HTTP client. + fn pool_with_mainnet_at(mock: &MockServer) -> OrderBookPool { + let mut clients = std::collections::BTreeMap::new(); + clients.insert( + Chain::Mainnet.id(), + OrderBookApi::new_with_base_url(mock.uri().parse().expect("mock uri parses")), + ); + OrderBookPool { + clients, + http: reqwest::Client::new(), + } + } + + #[tokio::test] + async fn request_passes_get_path_through() { + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v1/version")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"version":"x.y.z"}"#)) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request(Chain::Mainnet.id(), "GET", "/api/v1/version", None) + .await + .expect("request succeeds"); + assert_eq!(body, r#"{"version":"x.y.z"}"#); + } + + #[tokio::test] + async fn request_relative_path_works() { + // Module passes a path without a leading slash. The + // passthrough should still resolve against the orderbook + // base URL. + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v1/native_price/0xabc")) + .respond_with(ResponseTemplate::new(200).set_body_string("1.23")) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request( + Chain::Mainnet.id(), + "GET", + "api/v1/native_price/0xabc", + None, + ) + .await + .expect("relative path resolves"); + assert_eq!(body, "1.23"); + } + + #[tokio::test] + async fn request_rejects_unknown_method() { + let pool = OrderBookPool::with_default_chains(); + let err = pool + .request(Chain::Mainnet.id(), "PATCH", "/x", None) + .await + .unwrap_err(); + assert!(matches!(err, CowApiError::BadMethod(_))); + } + + #[tokio::test] + async fn submit_order_propagates_orderbook_response() { + let mock = MockServer::start().await; + let body_json = sample_order_json(); + // cowprotocol POST /api/v1/orders returns the order UID + // (56-byte hex) as a JSON string body. + let returned_uid = format!("\"0x{}\"", "ab".repeat(56)); + Mock::given(method("POST")) + .and(path("/api/v1/orders")) + .respond_with(ResponseTemplate::new(201).set_body_string(returned_uid.clone())) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let uid = pool + .submit_order_json(Chain::Mainnet.id(), body_json.as_bytes()) + .await + .expect("submit succeeds"); + assert_eq!(uid.as_slice().len(), 56); + assert_eq!(uid.as_slice(), &[0xab; 56]); + } + + /// A minimal but accepted-by-cowprotocol OrderCreation JSON. We + /// generate it inside the test so the JSON shape stays in lockstep + /// with the published `cowprotocol` version. + fn sample_order_json() -> String { + use alloy_primitives::{Address, U256}; + use cowprotocol::OrderCreation; + use cowprotocol::app_data::{EMPTY_APP_DATA_HASH, EMPTY_APP_DATA_JSON}; + use cowprotocol::order::{BuyTokenDestination, OrderData, OrderKind, SellTokenSource}; + use cowprotocol::signature::Signature; + use cowprotocol::signing_scheme::SigningScheme; + + let order_data = OrderData { + sell_token: Address::from([0x01; 20]), + buy_token: Address::from([0x02; 20]), + receiver: None, + sell_amount: U256::from(100u64), + buy_amount: U256::from(99u64), + valid_to: u32::MAX, + app_data: EMPTY_APP_DATA_HASH, + fee_amount: U256::ZERO, + kind: OrderKind::Sell, + partially_fillable: false, + sell_token_balance: SellTokenSource::Erc20, + buy_token_balance: BuyTokenDestination::Erc20, + }; + let signature = Signature::from_bytes(SigningScheme::PreSign, &[]).expect("presign empty"); + let creation = OrderCreation::from_signed_order_data( + &order_data, + signature, + Address::from([0x03; 20]), + EMPTY_APP_DATA_JSON.to_owned(), + None, + ) + .expect("valid OrderCreation"); + serde_json::to_string(&creation).expect("serialise OrderCreation") + } +} diff --git a/crates/nexum-engine/src/host/local_store_redb.rs b/crates/nexum-engine/src/host/local_store_redb.rs new file mode 100644 index 0000000..4e535e0 --- /dev/null +++ b/crates/nexum-engine/src/host/local_store_redb.rs @@ -0,0 +1,211 @@ +//! `nexum:host/local-store` backend. +//! +//! Single redb file under `EngineConfig.engine.state_dir`. Per-module +//! namespacing is enforced host-side via a `[len:u8][module_name][raw_key]` +//! prefix on every redb key. Two modules using the same key string see +//! disjoint data. +//! +//! The runtime supplies the namespace; modules see plain key strings. +//! Module names longer than 255 bytes are rejected at construction +//! (matches the one-byte length prefix). + +// The redb error enum is large by construction (Txn / Storage / +// Commit each carry a redb backtrace ≈ 160 bytes). Allowing the +// cap-on-Result-size lint here is the lesser evil: boxing every +// variant pushes the error path to the heap just to humour the lint. +#![allow(clippy::result_large_err)] + +use std::path::Path; +use std::sync::Arc; + +use redb::{Database, ReadableTable, TableDefinition}; +use thiserror::Error; + +const TABLE: TableDefinition<'static, &[u8], &[u8]> = TableDefinition::new("nexum:local-store"); +const MAX_NAMESPACE_LEN: usize = u8::MAX as usize; + +/// Process-wide handle to the local-store redb database. Cheap to +/// clone; the per-module view is constructed by setting the +/// namespace prefix at call time. +#[derive(Debug, Clone)] +pub struct LocalStore { + db: Arc, +} + +impl LocalStore { + /// Open (or create) the redb file at `path`. Materialises the + /// shared table so subsequent read transactions never hit + /// `TableDoesNotExist`. + pub fn open(path: impl AsRef) -> Result { + let db = Database::create(path).map_err(StorageError::Open)?; + { + let txn = db.begin_write().map_err(StorageError::Txn)?; + txn.open_table(TABLE).map_err(StorageError::Table)?; + txn.commit().map_err(StorageError::Commit)?; + } + Ok(Self { db: Arc::new(db) }) + } + + /// Fetch a value for `(namespace, key)`. Returns `Ok(None)` when + /// no entry exists; module never observes the prefix. + pub fn get(&self, namespace: &str, key: &str) -> Result>, StorageError> { + let full = build_key(namespace, key)?; + let txn = self.db.begin_read().map_err(StorageError::Txn)?; + let table = txn.open_table(TABLE).map_err(StorageError::Table)?; + let value = table + .get(full.as_slice()) + .map_err(StorageError::Storage)? + .map(|v| v.value().to_vec()); + Ok(value) + } + + /// Insert or overwrite. + pub fn set(&self, namespace: &str, key: &str, value: &[u8]) -> Result<(), StorageError> { + let full = build_key(namespace, key)?; + let txn = self.db.begin_write().map_err(StorageError::Txn)?; + { + let mut table = txn.open_table(TABLE).map_err(StorageError::Table)?; + table + .insert(full.as_slice(), value) + .map_err(StorageError::Storage)?; + } + txn.commit().map_err(StorageError::Commit)?; + Ok(()) + } + + /// Delete. Idempotent — deleting a missing key is a no-op. + pub fn delete(&self, namespace: &str, key: &str) -> Result<(), StorageError> { + let full = build_key(namespace, key)?; + let txn = self.db.begin_write().map_err(StorageError::Txn)?; + { + let mut table = txn.open_table(TABLE).map_err(StorageError::Table)?; + table + .remove(full.as_slice()) + .map_err(StorageError::Storage)?; + } + txn.commit().map_err(StorageError::Commit)?; + Ok(()) + } + + /// Enumerate keys in `namespace` whose raw key (post-prefix) + /// starts with `prefix`. Returns only the module-visible key + /// strings — the host strips the namespace prefix. + pub fn list_keys(&self, namespace: &str, prefix: &str) -> Result, StorageError> { + let ns_prefix = namespace_prefix(namespace)?; + let full_prefix = build_key(namespace, prefix)?; + let txn = self.db.begin_read().map_err(StorageError::Txn)?; + let table = txn.open_table(TABLE).map_err(StorageError::Table)?; + let mut out = Vec::new(); + for entry in table.iter().map_err(StorageError::Storage)? { + let (k, _v) = entry.map_err(StorageError::Storage)?; + let key_bytes = k.value(); + if key_bytes.starts_with(&full_prefix) + && let Ok(s) = std::str::from_utf8(&key_bytes[ns_prefix.len()..]) + { + out.push(s.to_owned()); + } + } + Ok(out) + } +} + +fn namespace_prefix(namespace: &str) -> Result, StorageError> { + if namespace.is_empty() { + return Err(StorageError::InvalidNamespace( + "module namespace must not be empty".into(), + )); + } + let bytes = namespace.as_bytes(); + if bytes.len() > MAX_NAMESPACE_LEN { + return Err(StorageError::InvalidNamespace(format!( + "namespace `{namespace}` is {} bytes; max is {MAX_NAMESPACE_LEN}", + bytes.len() + ))); + } + let mut out = Vec::with_capacity(1 + bytes.len()); + out.push(bytes.len() as u8); + out.extend_from_slice(bytes); + Ok(out) +} + +fn build_key(namespace: &str, key: &str) -> Result, StorageError> { + let mut out = namespace_prefix(namespace)?; + out.extend_from_slice(key.as_bytes()); + Ok(out) +} + +/// Errors surfaced by [`LocalStore`]. +#[derive(Debug, Error)] +pub enum StorageError { + #[error("open redb: {0}")] + Open(#[source] redb::DatabaseError), + #[error("redb txn: {0}")] + Txn(#[source] redb::TransactionError), + #[error("redb table: {0}")] + Table(#[source] redb::TableError), + #[error("redb storage: {0}")] + Storage(#[source] redb::StorageError), + #[error("redb commit: {0}")] + Commit(#[source] redb::CommitError), + #[error("invalid namespace: {0}")] + InvalidNamespace(String), +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fresh() -> (tempfile::TempDir, LocalStore) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = LocalStore::open(dir.path().join("ls.redb")).expect("open"); + (dir, store) + } + + #[test] + fn set_get_roundtrip() { + let (_dir, store) = fresh(); + store.set("twap", "k", b"v").unwrap(); + assert_eq!(store.get("twap", "k").unwrap().as_deref(), Some(&b"v"[..])); + } + + #[test] + fn namespaces_isolate_modules() { + let (_dir, store) = fresh(); + store.set("a", "k", b"from-a").unwrap(); + store.set("b", "k", b"from-b").unwrap(); + assert_eq!( + store.get("a", "k").unwrap().as_deref(), + Some(&b"from-a"[..]) + ); + assert_eq!( + store.get("b", "k").unwrap().as_deref(), + Some(&b"from-b"[..]) + ); + } + + #[test] + fn delete_then_get_is_none() { + let (_dir, store) = fresh(); + store.set("twap", "k", b"v").unwrap(); + store.delete("twap", "k").unwrap(); + assert!(store.get("twap", "k").unwrap().is_none()); + } + + #[test] + fn list_keys_strips_namespace_prefix() { + let (_dir, store) = fresh(); + store.set("twap", "posted:1", b"x").unwrap(); + store.set("twap", "posted:2", b"y").unwrap(); + store.set("twap", "other", b"z").unwrap(); + let keys = store.list_keys("twap", "posted:").unwrap(); + assert_eq!(keys.len(), 2); + assert!(keys.iter().all(|k| k.starts_with("posted:"))); + } + + #[test] + fn rejects_empty_namespace() { + let (_dir, store) = fresh(); + let err = store.set("", "k", b"v").unwrap_err(); + assert!(matches!(err, StorageError::InvalidNamespace(_))); + } +} diff --git a/crates/nexum-engine/src/host/mod.rs b/crates/nexum-engine/src/host/mod.rs new file mode 100644 index 0000000..b76fe99 --- /dev/null +++ b/crates/nexum-engine/src/host/mod.rs @@ -0,0 +1,12 @@ +//! Host-side backends for the `nexum:host` / `shepherd:cow` +//! interfaces. +//! +//! Each submodule owns one capability. The trait impls in `main.rs` +//! stay thin: they validate inputs, dispatch to the backend, and +//! project the backend's typed error onto the bindgen-generated +//! `HostError`. Keeping the backends pure (no bindgen types) means +//! each can be unit-tested without spinning up a wasmtime store. + +pub mod cow_orderbook; +pub mod local_store_redb; +pub mod provider_pool; diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs new file mode 100644 index 0000000..0b6d94b --- /dev/null +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -0,0 +1,152 @@ +//! `nexum:host/chain` backend. +//! +//! Per-chain alloy provider, opened from the engine config at boot. +//! `request` is a raw JSON-RPC dispatch: the host hands `(method, +//! params)` straight to alloy's transport and returns the result body +//! verbatim. No method allowlist, no re-encoding of params — the +//! contract is "give us a JSON-RPC pair, we'll return what the node +//! returns". +//! +//! Transports: +//! - `ws://` / `wss://` — `WsConnect`; required for `eth_subscribe`. +//! - `http://` / `https://` — alloy's HTTP transport; request/response only. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use alloy_provider::{DynProvider, Provider, ProviderBuilder, WsConnect}; +use serde_json::value::RawValue; +use thiserror::Error; +use tracing::info; + +use crate::engine_config::EngineConfig; + +/// Pool of alloy providers keyed by chain id. +#[derive(Debug, Clone)] +pub struct ProviderPool { + providers: Arc>, +} + +impl ProviderPool { + /// Open one provider per chain in `cfg.chains`. WebSocket URLs + /// engage alloy's pubsub transport; HTTP URLs use the HTTP + /// transport. Connection failures propagate to the caller; the + /// engine treats them as fatal at boot. + pub async fn from_config(cfg: &EngineConfig) -> Result { + let mut providers: BTreeMap = BTreeMap::new(); + for (chain_id, chain_cfg) in &cfg.chains { + let url = chain_cfg.rpc_url.as_str(); + info!(chain_id, url, "opening chain RPC provider"); + let provider = if url.starts_with("ws://") || url.starts_with("wss://") { + ProviderBuilder::new() + .connect_ws(WsConnect::new(url)) + .await + .map_err(|e| ProviderError::Connect { + chain_id: *chain_id, + detail: e.to_string(), + })? + .erased() + } else { + let parsed: url::Url = + url.parse() + .map_err(|e: url::ParseError| ProviderError::Connect { + chain_id: *chain_id, + detail: e.to_string(), + })?; + ProviderBuilder::new().connect_http(parsed).erased() + }; + providers.insert(*chain_id, provider); + } + Ok(Self { + providers: Arc::new(providers), + }) + } + + /// Empty pool — used by tests and as a default when no + /// `engine.toml` is found. Every `request` call returns + /// `UnknownChain`. + #[cfg_attr(not(test), allow(dead_code))] + pub fn empty() -> Self { + Self { + providers: Arc::new(BTreeMap::new()), + } + } + + /// Raw JSON-RPC dispatch. `params_json` must be the JSON encoding + /// of the params array (e.g. `"[\"0x...\",\"latest\"]"`), as + /// produced by the SDK's `chain::request` glue. + pub async fn request( + &self, + chain_id: u64, + method: String, + params_json: String, + ) -> Result { + let provider = self + .providers + .get(&chain_id) + .ok_or(ProviderError::UnknownChain(chain_id))?; + // Pass the params through as a raw JSON value so alloy does + // not re-encode them on the way to the node. + let params: Box = RawValue::from_string(params_json.clone()).map_err(|e| { + ProviderError::InvalidParams { + method: method.clone(), + detail: e.to_string(), + } + })?; + let result: Box = provider + .raw_request(method.clone().into(), params) + .await + .map_err(|e| ProviderError::Rpc { + method, + detail: e.to_string(), + })?; + Ok(result.get().to_owned()) + } +} + +/// Errors surfaced by [`ProviderPool`]. +#[derive(Debug, Error)] +pub enum ProviderError { + /// Chain id absent from the engine config. + #[error("unknown chain {0} (no engine.toml entry)")] + UnknownChain(u64), + /// Could not open the underlying transport. + #[error("connect chain {chain_id}: {detail}")] + Connect { + /// Chain id we failed to dial. + chain_id: u64, + /// Transport-side error string. + detail: String, + }, + /// The guest-supplied JSON params did not parse. + #[error("invalid params JSON for `{method}`: {detail}")] + InvalidParams { + /// RPC method name. + method: String, + /// JSON-parser detail. + detail: String, + }, + /// The node returned an error for the dispatched call. + #[error("rpc `{method}` failed: {detail}")] + Rpc { + /// RPC method name. + method: String, + /// Transport-side error string. + detail: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn empty_pool_rejects_lookups() { + let pool = ProviderPool::empty(); + let err = pool + .request(1, "eth_blockNumber".into(), "[]".into()) + .await + .unwrap_err(); + assert!(matches!(err, ProviderError::UnknownChain(1))); + } +} diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index ec09672..57be5a4 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -1,9 +1,13 @@ +mod engine_config; +mod host; mod manifest; use std::path::PathBuf; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use clap::Parser; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; use wasmtime::component::{Component, Linker, ResourceTable}; use wasmtime::error::Context as _; use wasmtime::{Engine, Store}; @@ -33,6 +37,11 @@ struct Cli { /// component file and falls back to a permissive default (with /// a deprecation warning) when none is found. manifest_path: Option, + + /// Optional explicit path to the engine-wide `engine.toml` config. + /// When omitted, the engine resolves the default search path + /// documented in `engine_config::load_or_default`. + engine_config_path: Option, } // Both packages are listed explicitly so wit-parser can resolve the @@ -56,6 +65,15 @@ struct HostState { /// Per-module `[capabilities.http].allow` allowlist (from nexum.toml). /// Consulted by `http::fetch` before any outbound call. http_allowlist: Vec, + /// Namespace for the running module's `local-store` rows. Set from + /// `manifest.module.name` at instantiation. + module_namespace: String, + /// `cow-api` backend — per-chain `OrderBookApi` clients + reqwest. + cow: host::cow_orderbook::OrderBookPool, + /// `chain` backend — per-chain alloy `DynProvider` pool. + chain: host::provider_pool::ProviderPool, + /// `local-store` backend — redb file with host-side namespacing. + store: host::local_store_redb::LocalStore, } impl WasiView for HostState { @@ -77,58 +95,135 @@ fn unimplemented(domain: &str, detail: impl Into) -> HostError { } } -// -- Stub implementations for host interfaces -- +fn internal_error(domain: &str, detail: impl Into) -> HostError { + HostError { + domain: domain.into(), + kind: HostErrorKind::Internal, + code: 0, + message: detail.into(), + data: None, + } +} + +// -- nexum:host/types is empty (declarations only). -- impl nexum::host::types::Host for HostState {} +// -- shepherd:cow/cow-api: REST passthrough + typed submission. -- + impl shepherd::cow::cow_api::Host for HostState { async fn request( &mut self, - _chain_id: u64, + chain_id: u64, method: String, path: String, - _body: Option, + body: Option, ) -> Result { let start = Instant::now(); - eprintln!("[cow-api] {method} {path}"); - let result = Err(unimplemented( - "cow-api", - format!("not implemented: {method} {path}"), - )); - eprintln!("[timing] cow-api::request: {:?}", start.elapsed()); + tracing::debug!(chain_id, %method, %path, "cow-api::request"); + let result = match self + .cow + .request(chain_id, &method, &path, body.as_deref()) + .await + { + Ok(body) => Ok(body), + Err(host::cow_orderbook::CowApiError::UnknownChain(id)) => Err(unimplemented( + "cow-api", + format!("chain {id} not in cowprotocol"), + )), + Err(host::cow_orderbook::CowApiError::BadMethod(m)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: format!("unsupported HTTP method: {m}"), + data: None, + }), + Err(host::cow_orderbook::CowApiError::BadPath(msg)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: msg, + data: None, + }), + Err(err) => Err(internal_error("cow-api", err.to_string())), + }; + tracing::trace!(elapsed_ms = ?start.elapsed(), "cow-api::request done"); result } async fn submit_order( &mut self, - _chain_id: u64, - _order_data: Vec, + chain_id: u64, + order_data: Vec, ) -> Result { let start = Instant::now(); - eprintln!("[cow-api] submit-order"); - let result = Err(unimplemented("cow-api", "submit-order not implemented")); - eprintln!("[timing] cow-api::submit-order: {:?}", start.elapsed()); + tracing::debug!(chain_id, bytes = order_data.len(), "cow-api::submit-order"); + let result = match self.cow.submit_order_json(chain_id, &order_data).await { + Ok(uid) => Ok(format!("0x{}", hex_encode(uid.as_slice()))), + Err(host::cow_orderbook::CowApiError::UnknownChain(id)) => Err(unimplemented( + "cow-api", + format!("chain {id} not in cowprotocol"), + )), + Err(host::cow_orderbook::CowApiError::Decode(err)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: format!("invalid OrderCreation JSON: {err}"), + data: None, + }), + Err(host::cow_orderbook::CowApiError::Orderbook(msg)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::Denied, + code: 0, + message: msg, + data: None, + }), + Err(err) => Err(internal_error("cow-api", err.to_string())), + }; + tracing::trace!(elapsed_ms = ?start.elapsed(), "cow-api::submit-order done"); result } } +// -- nexum:host/chain: raw JSON-RPC dispatch over alloy. -- + impl nexum::host::chain::Host for HostState { async fn request( &mut self, - _chain_id: u64, + chain_id: u64, method: String, - _params: String, + params: String, ) -> Result { let start = Instant::now(); - eprintln!("[chain] request: {method}"); - let result = Err(HostError { - domain: "chain".into(), - kind: HostErrorKind::Unsupported, - code: -32601, - message: format!("method not implemented: {method}"), - data: None, - }); - eprintln!("[timing] chain::request: {:?}", start.elapsed()); + tracing::debug!(chain_id, %method, "chain::request"); + let result = match self.chain.request(chain_id, method.clone(), params).await { + Ok(body) => Ok(body), + Err(host::provider_pool::ProviderError::UnknownChain(id)) => Err(HostError { + domain: "chain".into(), + kind: HostErrorKind::Unsupported, + code: 0, + message: format!("chain {id} has no engine.toml RPC entry"), + data: None, + }), + Err(host::provider_pool::ProviderError::InvalidParams { detail, .. }) => { + Err(HostError { + domain: "chain".into(), + kind: HostErrorKind::InvalidInput, + code: -32602, + message: detail, + data: None, + }) + } + Err(host::provider_pool::ProviderError::Rpc { detail, .. }) => Err(HostError { + domain: "chain".into(), + kind: HostErrorKind::Internal, + code: -32603, + message: detail, + data: None, + }), + Err(err) => Err(internal_error("chain", err.to_string())), + }; + tracing::trace!(elapsed_ms = ?start.elapsed(), "chain::request done"); result } @@ -138,34 +233,30 @@ impl nexum::host::chain::Host for HostState { requests: Vec, ) -> Result, HostError> { let start = Instant::now(); - eprintln!("[chain] request-batch: {} calls", requests.len()); + tracing::debug!(chain_id, count = requests.len(), "chain::request-batch"); let mut out = Vec::with_capacity(requests.len()); for req in requests { - match self.request(chain_id, req.method, req.params).await { + match nexum::host::chain::Host::request(self, chain_id, req.method, req.params).await { Ok(s) => out.push(nexum::host::chain::RpcResult::Ok(s)), Err(e) => out.push(nexum::host::chain::RpcResult::Err(e)), } } - eprintln!("[timing] chain::request-batch: {:?}", start.elapsed()); + tracing::trace!(elapsed_ms = ?start.elapsed(), "chain::request-batch done"); Ok(out) } } +// -- nexum:host/identity: deferred to 0.3 (keystore/KMS backend). -- + impl nexum::host::identity::Host for HostState { async fn accounts(&mut self) -> Result>, HostError> { - let start = Instant::now(); - eprintln!("[identity] accounts"); - let result = Ok(vec![]); - eprintln!("[timing] identity::accounts: {:?}", start.elapsed()); - result + // No keystore wired yet — return an empty roster so guests can + // probe-then-skip without erroring. Real keystore lands in 0.3. + Ok(vec![]) } async fn sign(&mut self, _account: Vec, _message: Vec) -> Result, HostError> { - let start = Instant::now(); - eprintln!("[identity] sign"); - let result = Err(unimplemented("identity", "sign not implemented")); - eprintln!("[timing] identity::sign: {:?}", start.elapsed()); - result + Err(unimplemented("identity", "sign requires a keystore (0.3)")) } async fn sign_typed_data( @@ -173,61 +264,54 @@ impl nexum::host::identity::Host for HostState { _account: Vec, _typed_data: String, ) -> Result, HostError> { - let start = Instant::now(); - eprintln!("[identity] sign-typed-data"); - let result = Err(unimplemented("identity", "sign-typed-data not implemented")); - eprintln!("[timing] identity::sign-typed-data: {:?}", start.elapsed()); - result + Err(unimplemented( + "identity", + "sign-typed-data requires a keystore (0.3)", + )) } } +// -- nexum:host/local-store: redb backend with host-side namespacing. -- + impl nexum::host::local_store::Host for HostState { async fn get(&mut self, key: String) -> Result>, HostError> { - let start = Instant::now(); - eprintln!("[local-store] get: {key}"); - let result = Ok(None); - eprintln!("[timing] local-store::get: {:?}", start.elapsed()); - result + self.store + .get(&self.module_namespace, &key) + .map_err(|err| internal_error("local-store", err.to_string())) } - async fn set(&mut self, key: String, _value: Vec) -> Result<(), HostError> { - let start = Instant::now(); - eprintln!("[local-store] set: {key}"); - let result = Ok(()); - eprintln!("[timing] local-store::set: {:?}", start.elapsed()); - result + async fn set(&mut self, key: String, value: Vec) -> Result<(), HostError> { + self.store + .set(&self.module_namespace, &key, &value) + .map_err(|err| internal_error("local-store", err.to_string())) } async fn delete(&mut self, key: String) -> Result<(), HostError> { - let start = Instant::now(); - eprintln!("[local-store] delete: {key}"); - let result = Ok(()); - eprintln!("[timing] local-store::delete: {:?}", start.elapsed()); - result + self.store + .delete(&self.module_namespace, &key) + .map_err(|err| internal_error("local-store", err.to_string())) } async fn list_keys(&mut self, prefix: String) -> Result, HostError> { - let start = Instant::now(); - eprintln!("[local-store] list-keys: {prefix}"); - let result = Ok(vec![]); - eprintln!("[timing] local-store::list-keys: {:?}", start.elapsed()); - result + self.store + .list_keys(&self.module_namespace, &prefix) + .map_err(|err| internal_error("local-store", err.to_string())) } } impl nexum::host::remote_store::Host for HostState { async fn upload(&mut self, _data: Vec) -> Result, HostError> { - let start = Instant::now(); - let result = Err(unimplemented("remote-store", "upload not implemented")); - eprintln!("[timing] remote-store::upload: {:?}", start.elapsed()); - result + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) } async fn download(&mut self, _reference: Vec) -> Result, HostError> { - let start = Instant::now(); - let result = Err(unimplemented("remote-store", "download not implemented")); - eprintln!("[timing] remote-store::download: {:?}", start.elapsed()); - result + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) } async fn read_feed( @@ -235,56 +319,51 @@ impl nexum::host::remote_store::Host for HostState { _owner: Vec, _topic: Vec, ) -> Result>, HostError> { - let start = Instant::now(); - let result = Err(unimplemented("remote-store", "read-feed not implemented")); - eprintln!("[timing] remote-store::read-feed: {:?}", start.elapsed()); - result + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) } async fn write_feed(&mut self, _topic: Vec, _data: Vec) -> Result, HostError> { - let start = Instant::now(); - let result = Err(unimplemented("remote-store", "write-feed not implemented")); - eprintln!("[timing] remote-store::write-feed: {:?}", start.elapsed()); - result + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) } } impl nexum::host::messaging::Host for HostState { - async fn publish(&mut self, content_topic: String, _payload: Vec) -> Result<(), HostError> { - let start = Instant::now(); - eprintln!("[messaging] publish: {content_topic}"); - let result = Err(unimplemented("messaging", "publish not implemented")); - eprintln!("[timing] messaging::publish: {:?}", start.elapsed()); - result + async fn publish( + &mut self, + _content_topic: String, + _payload: Vec, + ) -> Result<(), HostError> { + Err(unimplemented("messaging", "Waku backend deferred to 0.3")) } async fn query( &mut self, - content_topic: String, + _content_topic: String, _start_time: Option, _end_time: Option, _limit: Option, ) -> Result, HostError> { - let start = Instant::now(); - eprintln!("[messaging] query: {content_topic}"); - let result = Ok(vec![]); - eprintln!("[timing] messaging::query: {:?}", start.elapsed()); - result + // Empty result — same posture as `identity::accounts`. + Ok(vec![]) } } impl nexum::host::logging::Host for HostState { async fn log(&mut self, level: nexum::host::logging::Level, message: String) { - let start = Instant::now(); - let level_str = match level { - nexum::host::logging::Level::Trace => "TRACE", - nexum::host::logging::Level::Debug => "DEBUG", - nexum::host::logging::Level::Info => "INFO", - nexum::host::logging::Level::Warn => "WARN", - nexum::host::logging::Level::Error => "ERROR", - }; - eprintln!("[{level_str}] {message}"); - eprintln!("[timing] logging::log: {:?}", start.elapsed()); + let module = self.module_namespace.as_str(); + match level { + nexum::host::logging::Level::Trace => tracing::trace!(module, "{}", message), + nexum::host::logging::Level::Debug => tracing::debug!(module, "{}", message), + nexum::host::logging::Level::Info => tracing::info!(module, "{}", message), + nexum::host::logging::Level::Warn => tracing::warn!(module, "{}", message), + nexum::host::logging::Level::Error => tracing::error!(module, "{}", message), + } } } @@ -320,16 +399,12 @@ impl nexum::host::http::Host for HostState { &mut self, req: nexum::host::http::Request, ) -> Result { - let start = Instant::now(); - eprintln!("[http] {} {}", req.method, req.url); - // Manifest allowlist enforcement runs before any I/O. Hosts that // never link a manifest leave `http_allowlist` empty, which denies // every request — matching the "no implicit network" stance. let host = match manifest::extract_host(&req.url) { Some(h) => h, None => { - eprintln!("[timing] http::fetch: {:?}", start.elapsed()); return Err(HostError { domain: "http".into(), kind: HostErrorKind::InvalidInput, @@ -340,8 +415,7 @@ impl nexum::host::http::Host for HostState { } }; if !manifest::host_allowed(host, &self.http_allowlist) { - eprintln!("[http] denied by allowlist: {host}"); - eprintln!("[timing] http::fetch: {:?}", start.elapsed()); + warn!(host, "[http] denied by allowlist"); return Err(HostError { domain: "http".into(), kind: HostErrorKind::Denied, @@ -353,50 +427,86 @@ impl nexum::host::http::Host for HostState { data: None, }); } - // 0.2: allowlist passed, but the reference runtime does not perform // real HTTP yet. Real fetch lands in 0.3. - let result = Err(unimplemented( + Err(unimplemented( "http", "fetch not implemented in 0.2 reference runtime (allowlist passed)", - )); - eprintln!("[timing] http::fetch: {:?}", start.elapsed()); - result + )) } } +/// Lowercase hex encoder. Kept in the engine binary rather than +/// pulling a `hex` crate just for one call site. +fn hex_encode(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push_str(&format!("{b:02x}")); + } + s +} + #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); let wasm_path = cli.wasm_path; let explicit_manifest = cli.manifest_path; + let explicit_engine_config = cli.engine_config_path; - println!( - "nexum-engine: loading component from {}", - wasm_path.display() - ); + // -- 1. Load engine config (optional). -- + let engine_cfg = engine_config::load_or_default(explicit_engine_config.as_deref())?; + + // -- 2. Install tracing subscriber. -- + let env_filter = EnvFilter::try_from_default_env() + .or_else(|_| EnvFilter::try_new(&engine_cfg.engine.log_level)) + .unwrap_or_else(|_| EnvFilter::new("info")); + tracing_subscriber::fmt() + .with_env_filter(env_filter) + .with_target(true) + .init(); + + info!("nexum-engine starting"); + info!(wasm = %wasm_path.display(), "loading component"); - // Load the manifest from the explicit path if given, otherwise from - // `nexum.toml` next to the component file. Missing → fallback (with - // deprecation warning). + // -- 3. Load the module manifest. -- let manifest_path = explicit_manifest.or_else(|| wasm_path.parent().map(|p| p.join("nexum.toml"))); let loaded = match manifest_path.as_deref() { Some(p) if p.exists() => { - println!("nexum-engine: loading manifest from {}", p.display()); + info!(manifest = %p.display(), "loading nexum.toml"); manifest::load(p)? } _ => manifest::fallback_manifest(), }; + // -- 4. Bring up the host backends. -- + std::fs::create_dir_all(&engine_cfg.engine.state_dir).with_context(|| { + format!( + "create state directory {}", + engine_cfg.engine.state_dir.display() + ) + })?; + let store_path = engine_cfg.engine.state_dir.join("local-store.redb"); + let local_store = host::local_store_redb::LocalStore::open(&store_path) + .with_context(|| format!("open local-store at {}", store_path.display()))?; + let cow_pool = host::cow_orderbook::OrderBookPool::with_default_chains(); + let provider_pool = host::provider_pool::ProviderPool::from_config(&engine_cfg) + .await + .context("open chain providers")?; + + // -- 5. Build the wasmtime engine + component. -- let mut config = wasmtime::Config::new(); config.wasm_component_model(true); + // `async_support` was deprecated in wasmtime 45 — the engine + // resolves async on its own. Keeping the call out of the Config + // chain silences the `deprecated` warning under + // `RUSTFLAGS=-D warnings`. let engine = Engine::new(&config)?; - let start = Instant::now(); + let load_start = Instant::now(); let component = Component::from_file(&engine, &wasm_path).context("failed to load component")?; - eprintln!("[timing] component load: {:?}", start.elapsed()); + tracing::debug!(elapsed_ms = ?load_start.elapsed(), "component load"); let mut linker = Linker::::new(&engine); Shepherd::add_to_linker::>( @@ -406,6 +516,11 @@ async fn main() -> anyhow::Result<()> { wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; let wasi = WasiCtxBuilder::new().inherit_stdio().build(); + let module_namespace = if loaded.manifest.module.name.is_empty() { + "module".to_owned() + } else { + loaded.manifest.module.name.clone() + }; let mut store = Store::new( &engine, @@ -414,36 +529,39 @@ async fn main() -> anyhow::Result<()> { table: ResourceTable::new(), monotonic_baseline: Instant::now(), http_allowlist: loaded.http_allowlist, + module_namespace, + cow: cow_pool, + chain: provider_pool, + store: local_store, }, ); - let start = Instant::now(); + let inst_start = Instant::now(); let bindings = Shepherd::instantiate_async(&mut store, &component, &linker) .await .context("failed to instantiate component")?; - eprintln!("[timing] component instantiate: {:?}", start.elapsed()); + tracing::debug!(elapsed_ms = ?inst_start.elapsed(), "component instantiate"); - println!("nexum-engine: calling init..."); - // 0.2: [config] is stringly-typed (typed variant deferred to 0.3). - // Fall back to a single ("name", "") pair if the manifest has - // no [config] section so the example module still has something to log. + info!("calling init"); let config_entries: Config = if loaded.config.is_empty() { vec![("name".into(), loaded.manifest.module.name.clone())] } else { loaded.config }; - let start = Instant::now(); + let init_start = Instant::now(); match bindings.call_init(&mut store, &config_entries).await? { - Ok(()) => println!("nexum-engine: init succeeded"), - Err(e) => println!( - "nexum-engine: init failed: {}::{:?} {} ({})", - e.domain, e.kind, e.message, e.code + Ok(()) => info!(elapsed_ms = ?init_start.elapsed(), "init succeeded"), + Err(e) => warn!( + domain = %e.domain, + kind = ?e.kind, + code = e.code, + message = %e.message, + "init failed", ), } - eprintln!("[timing] call_init: {:?}", start.elapsed()); // Dispatch a test block event (timestamps are ms since Unix epoch, UTC). - println!("nexum-engine: dispatching test block event..."); + info!("dispatching test block event"); let block = nexum::host::types::Block { chain_id: 1, number: 19_000_000, @@ -451,16 +569,18 @@ async fn main() -> anyhow::Result<()> { timestamp: 1_700_000_000_000, }; let event = nexum::host::types::Event::Block(block); - let start = Instant::now(); + let evt_start = Instant::now(); match bindings.call_on_event(&mut store, &event).await? { - Ok(()) => println!("nexum-engine: on-event succeeded"), - Err(e) => println!( - "nexum-engine: on-event failed: {}::{:?} {} ({})", - e.domain, e.kind, e.message, e.code + Ok(()) => info!(elapsed_ms = ?evt_start.elapsed(), "on-event succeeded"), + Err(e) => warn!( + domain = %e.domain, + kind = ?e.kind, + code = e.code, + message = %e.message, + "on-event failed", ), } - eprintln!("[timing] call_on_event: {:?}", start.elapsed()); - println!("nexum-engine: done"); + info!("done"); Ok(()) } From 76523ae8c3649f568cd0c6bff40ba73d679cbadc Mon Sep 17 00:00:00 2001 From: brunota20 Date: Mon, 1 Jun 2026 15:43:42 -0300 Subject: [PATCH 06/29] runtime: multi-module supervisor + block/log event loop --- crates/nexum-engine/Cargo.toml | 2 + crates/nexum-engine/src/engine_config.rs | 21 + crates/nexum-engine/src/host/provider_pool.rs | 49 +++ crates/nexum-engine/src/main.rs | 309 +++++++++----- crates/nexum-engine/src/manifest.rs | 48 +++ crates/nexum-engine/src/supervisor.rs | 387 ++++++++++++++++++ 6 files changed, 719 insertions(+), 97 deletions(-) create mode 100644 crates/nexum-engine/src/supervisor.rs diff --git a/crates/nexum-engine/Cargo.toml b/crates/nexum-engine/Cargo.toml index 9d450b1..b054ff0 100644 --- a/crates/nexum-engine/Cargo.toml +++ b/crates/nexum-engine/Cargo.toml @@ -46,9 +46,11 @@ reqwest.workspace = true # alloy's JSON-RPC layer without reimplementing the codec. alloy-provider.workspace = true alloy-rpc-client = { version = "1.5", default-features = false } +alloy-rpc-types-eth = { version = "1.5", default-features = false, features = ["std"] } alloy-transport = { version = "1.5", default-features = false } alloy-transport-ws.workspace = true alloy-primitives.workspace = true +futures.workspace = true # `local-store` backend. Per-module namespacing is enforced # host-side via a `[len:u8][module_name][raw_key]` prefix. diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs index 4ec271c..595a688 100644 --- a/crates/nexum-engine/src/engine_config.rs +++ b/crates/nexum-engine/src/engine_config.rs @@ -32,6 +32,27 @@ pub struct EngineConfig { /// pool seed. #[serde(default)] pub chains: BTreeMap, + /// Modules the supervisor should boot. Each entry resolves a + /// `(component.wasm, nexum.toml)` pair on the local filesystem + /// for 0.2 — content-addressed resolution (Swarm / OCI / + /// `[[content.sources]]`) lands in 0.3 per + /// `docs/03-module-discovery.md`. + #[serde(default)] + pub modules: Vec, +} + +/// One `[[modules]]` table from `engine.toml`. +/// +/// Both fields are filesystem paths in 0.2. `manifest` defaults to +/// `nexum.toml` next to `path` if omitted, matching the bundle layout +/// in `docs/02-modules-events-packaging.md`. +#[derive(Debug, Deserialize)] +pub struct ModuleEntry { + /// Path to the compiled `.wasm` component. + pub path: std::path::PathBuf, + /// Path to the module's `nexum.toml`. Defaults to `/nexum.toml`. + #[serde(default)] + pub manifest: Option, } #[derive(Debug, Deserialize)] diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index 0b6d94b..adf589f 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -12,9 +12,13 @@ //! - `http://` / `https://` — alloy's HTTP transport; request/response only. use std::collections::BTreeMap; +use std::pin::Pin; use std::sync::Arc; use alloy_provider::{DynProvider, Provider, ProviderBuilder, WsConnect}; +use alloy_rpc_types_eth::{Filter, Header, Log}; +use futures::stream::Stream; +use futures::stream::StreamExt as _; use serde_json::value::RawValue; use thiserror::Error; use tracing::info; @@ -72,6 +76,46 @@ impl ProviderPool { } } + /// Open a new-blocks (`eth_subscribe newHeads`) stream on + /// `chain_id`. Requires a WS / IPC transport at construction + /// time; HTTP-only providers surface `UnknownChain` here. + pub async fn subscribe_blocks(&self, chain_id: u64) -> Result { + let provider = self + .providers + .get(&chain_id) + .ok_or(ProviderError::UnknownChain(chain_id))?; + let sub = provider + .subscribe_blocks() + .await + .map_err(|e| ProviderError::Rpc { + method: "eth_subscribe(newHeads)".into(), + detail: e.to_string(), + })?; + let stream = sub.into_stream().map(Ok::<_, ProviderError>); + Ok(Box::pin(stream)) + } + + /// Open an `eth_subscribe(logs, filter)` stream on `chain_id`. + pub async fn subscribe_logs( + &self, + chain_id: u64, + filter: Filter, + ) -> Result { + let provider = self + .providers + .get(&chain_id) + .ok_or(ProviderError::UnknownChain(chain_id))?; + let sub = provider + .subscribe_logs(&filter) + .await + .map_err(|e| ProviderError::Rpc { + method: "eth_subscribe(logs)".into(), + detail: e.to_string(), + })?; + let stream = sub.into_stream().map(Ok::<_, ProviderError>); + Ok(Box::pin(stream)) + } + /// Raw JSON-RPC dispatch. `params_json` must be the JSON encoding /// of the params array (e.g. `"[\"0x...\",\"latest\"]"`), as /// produced by the SDK's `chain::request` glue. @@ -104,6 +148,11 @@ impl ProviderPool { } } +/// Boxed stream of `newHeads`-style block headers. +pub type BlockStream = Pin> + Send>>; +/// Boxed stream of `logs`-filtered log events. +pub type LogStream = Pin> + Send>>; + /// Errors surfaced by [`ProviderPool`]. #[derive(Debug, Error)] pub enum ProviderError { diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 57be5a4..56e7be7 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -1,17 +1,20 @@ mod engine_config; mod host; mod manifest; +mod supervisor; use std::path::PathBuf; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use clap::Parser; +use futures::StreamExt; +use futures::stream::{FuturesUnordered, select_all}; use tracing::{info, warn}; use tracing_subscriber::EnvFilter; -use wasmtime::component::{Component, Linker, ResourceTable}; +use wasmtime::Engine; +use wasmtime::component::{Linker, ResourceTable}; use wasmtime::error::Context as _; -use wasmtime::{Engine, Store}; -use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; +use wasmtime_wasi::{WasiCtx, WasiCtxView, WasiView}; /// Reference CLI for the 0.2 `nexum-engine` runtime. /// @@ -29,19 +32,23 @@ use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; version, )] struct Cli { - /// Path to the Wasm Component file to load. - wasm_path: PathBuf, + /// Optional path to a Wasm Component file. Backwards-compat + /// shortcut that synthesises a one-module engine config when no + /// `--engine-config` is supplied. Production deployments declare + /// modules in TOML and omit this positional. + wasm: Option, /// Optional explicit path to the module's `nexum.toml` manifest. - /// When omitted, the engine looks for `nexum.toml` next to the - /// component file and falls back to a permissive default (with - /// a deprecation warning) when none is found. - manifest_path: Option, + /// Only consulted alongside the positional `wasm` shortcut; when + /// the engine config drives the module set, each module brings + /// its own manifest via the engine-config entries. + manifest: Option, /// Optional explicit path to the engine-wide `engine.toml` config. /// When omitted, the engine resolves the default search path /// documented in `engine_config::load_or_default`. - engine_config_path: Option, + #[arg(long = "engine-config")] + engine_config: Option, } // Both packages are listed explicitly so wit-parser can resolve the @@ -448,15 +455,16 @@ fn hex_encode(bytes: &[u8]) -> String { #[tokio::main] async fn main() -> anyhow::Result<()> { + // CLI surface (clap-derived, see `Cli`): + // nexum-engine [ []] [--engine-config ] + // + // Positional `` is a backwards-compat shortcut that + // synthesises a one-module engine config. Production deployments + // pass `--engine-config` and declare modules in TOML. let cli = Cli::parse(); - let wasm_path = cli.wasm_path; - let explicit_manifest = cli.manifest_path; - let explicit_engine_config = cli.engine_config_path; - // -- 1. Load engine config (optional). -- - let engine_cfg = engine_config::load_or_default(explicit_engine_config.as_deref())?; + let engine_cfg = engine_config::load_or_default(cli.engine_config.as_deref())?; - // -- 2. Install tracing subscriber. -- let env_filter = EnvFilter::try_from_default_env() .or_else(|_| EnvFilter::try_new(&engine_cfg.engine.log_level)) .unwrap_or_else(|_| EnvFilter::new("info")); @@ -466,20 +474,8 @@ async fn main() -> anyhow::Result<()> { .init(); info!("nexum-engine starting"); - info!(wasm = %wasm_path.display(), "loading component"); - - // -- 3. Load the module manifest. -- - let manifest_path = - explicit_manifest.or_else(|| wasm_path.parent().map(|p| p.join("nexum.toml"))); - let loaded = match manifest_path.as_deref() { - Some(p) if p.exists() => { - info!(manifest = %p.display(), "loading nexum.toml"); - manifest::load(p)? - } - _ => manifest::fallback_manifest(), - }; - // -- 4. Bring up the host backends. -- + // Bring up shared host backends. std::fs::create_dir_all(&engine_cfg.engine.state_dir).with_context(|| { format!( "create state directory {}", @@ -494,20 +490,11 @@ async fn main() -> anyhow::Result<()> { .await .context("open chain providers")?; - // -- 5. Build the wasmtime engine + component. -- + // wasmtime engine + linker — one of each, shared across modules. let mut config = wasmtime::Config::new(); config.wasm_component_model(true); - // `async_support` was deprecated in wasmtime 45 — the engine - // resolves async on its own. Keeping the call out of the Config - // chain silences the `deprecated` warning under - // `RUSTFLAGS=-D warnings`. let engine = Engine::new(&config)?; - let load_start = Instant::now(); - let component = - Component::from_file(&engine, &wasm_path).context("failed to load component")?; - tracing::debug!(elapsed_ms = ?load_start.elapsed(), "component load"); - let mut linker = Linker::::new(&engine); Shepherd::add_to_linker::>( &mut linker, @@ -515,72 +502,200 @@ async fn main() -> anyhow::Result<()> { )?; wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; - let wasi = WasiCtxBuilder::new().inherit_stdio().build(); - let module_namespace = if loaded.manifest.module.name.is_empty() { - "module".to_owned() + // Boot supervisor — `engine.toml.[[modules]]` first, CLI + // positional second. + let mut supervisor = if let Some(wasm) = cli.wasm.as_deref() { + if !engine_cfg.modules.is_empty() { + warn!("ignoring engine.toml [[modules]] because a positional was given"); + } + supervisor::Supervisor::boot_single( + &engine, + &linker, + wasm, + cli.manifest.as_deref(), + &cow_pool, + &provider_pool, + &local_store, + ) + .await? + } else if !engine_cfg.modules.is_empty() { + supervisor::Supervisor::boot( + &engine, + &linker, + &engine_cfg, + &cow_pool, + &provider_pool, + &local_store, + ) + .await? } else { - loaded.manifest.module.name.clone() + anyhow::bail!( + "no modules to run — either pass a positional or declare \ + [[modules]] entries in engine.toml" + ); }; - let mut store = Store::new( - &engine, - HostState { - wasi, - table: ResourceTable::new(), - monotonic_baseline: Instant::now(), - http_allowlist: loaded.http_allowlist, - module_namespace, - cow: cow_pool, - chain: provider_pool, - store: local_store, - }, + info!( + modules = supervisor.module_count(), + chains = supervisor.block_chains().len(), + "supervisor ready" ); - let inst_start = Instant::now(); - let bindings = Shepherd::instantiate_async(&mut store, &component, &linker) - .await - .context("failed to instantiate component")?; - tracing::debug!(elapsed_ms = ?inst_start.elapsed(), "component instantiate"); + // Open per-chain block subscriptions + per-module log + // subscriptions, merge, dispatch until shutdown. + let block_chains = supervisor.block_chains(); + let log_subs = supervisor.log_subscriptions(); - info!("calling init"); - let config_entries: Config = if loaded.config.is_empty() { - vec![("name".into(), loaded.manifest.module.name.clone())] - } else { - loaded.config - }; - let init_start = Instant::now(); - match bindings.call_init(&mut store, &config_entries).await? { - Ok(()) => info!(elapsed_ms = ?init_start.elapsed(), "init succeeded"), - Err(e) => warn!( - domain = %e.domain, - kind = ?e.kind, - code = e.code, - message = %e.message, - "init failed", - ), - } - - // Dispatch a test block event (timestamps are ms since Unix epoch, UTC). - info!("dispatching test block event"); - let block = nexum::host::types::Block { - chain_id: 1, - number: 19_000_000, - hash: vec![0xab; 32], - timestamp: 1_700_000_000_000, - }; - let event = nexum::host::types::Event::Block(block); - let evt_start = Instant::now(); - match bindings.call_on_event(&mut store, &event).await? { - Ok(()) => info!(elapsed_ms = ?evt_start.elapsed(), "on-event succeeded"), - Err(e) => warn!( - domain = %e.domain, - kind = ?e.kind, - code = e.code, - message = %e.message, - "on-event failed", - ), + if block_chains.is_empty() && log_subs.is_empty() { + info!("no [[subscription]] entries — engine has nothing to run; exiting"); + return Ok(()); } + let block_streams = open_block_streams(&provider_pool, &block_chains).await; + let log_streams = open_log_streams(&provider_pool, log_subs).await; + + let shutdown = async { + match wait_for_shutdown_signal().await { + Ok(name) => info!(signal = %name, "shutdown signal received"), + Err(err) => warn!(error = %err, "signal handler failed — using ctrl-c"), + } + }; + + run_event_loop(&mut supervisor, block_streams, log_streams, shutdown).await; info!("done"); Ok(()) } + +/// Per-chain block subscriptions, one shared stream per chain id. +async fn open_block_streams( + pool: &host::provider_pool::ProviderPool, + chains: &std::collections::BTreeSet, +) -> Vec { + let mut openings: FuturesUnordered<_> = chains + .iter() + .copied() + .map(|chain_id| async move { (chain_id, pool.subscribe_blocks(chain_id).await) }) + .collect(); + + let mut streams = Vec::new(); + while let Some((chain_id, result)) = openings.next().await { + match result { + Ok(stream) => { + info!(chain_id, "block subscription open"); + let tagged: TaggedBlockStream = Box::pin(stream.map(move |item| { + item.map(|header| (chain_id, header)) + .map_err(anyhow::Error::from) + })); + streams.push(tagged); + } + Err(err) => { + warn!(chain_id, error = %err, "block subscription failed"); + } + } + } + streams +} + +/// Per-module log subscriptions. Each entry is a stream tagged with +/// the owning module name + chain id. +async fn open_log_streams( + pool: &host::provider_pool::ProviderPool, + subs: Vec<(String, u64, alloy_rpc_types_eth::Filter)>, +) -> Vec { + let mut openings: FuturesUnordered<_> = subs + .into_iter() + .map(|(module, chain_id, filter)| async move { + let stream = pool.subscribe_logs(chain_id, filter).await; + (module, chain_id, stream) + }) + .collect(); + + let mut streams = Vec::new(); + while let Some((module, chain_id, result)) = openings.next().await { + match result { + Ok(stream) => { + info!(module = %module, chain_id, "log subscription open"); + let module_name = module.clone(); + let tagged: TaggedLogStream = Box::pin(stream.map(move |item| { + item.map(|log| (module_name.clone(), chain_id, log)) + .map_err(anyhow::Error::from) + })); + streams.push(tagged); + } + Err(err) => { + warn!(module = %module, chain_id, error = %err, "log subscription failed"); + } + } + } + streams +} + +type TaggedBlockStream = std::pin::Pin< + Box< + dyn futures::Stream> + + Send, + >, +>; +type TaggedLogStream = std::pin::Pin< + Box< + dyn futures::Stream> + + Send, + >, +>; + +/// Drive the supervisor with events until `shutdown` resolves. +async fn run_event_loop( + supervisor: &mut supervisor::Supervisor, + block_streams: Vec, + log_streams: Vec, + shutdown: impl std::future::Future + Send, +) { + let mut blocks = select_all(block_streams); + let mut logs = select_all(log_streams); + let mut shutdown = Box::pin(shutdown); + loop { + tokio::select! { + biased; + () = &mut shutdown => return, + next = blocks.next() => match next { + Some(Ok((chain_id, header))) => { + let block = nexum::host::types::Block { + chain_id, + number: header.number, + hash: header.hash.as_slice().to_vec(), + timestamp: header.timestamp.saturating_mul(1000), + }; + supervisor.dispatch_block(block).await; + } + Some(Err(err)) => warn!(error = %err, "block stream error — continuing"), + None => {} + }, + next = logs.next() => match next { + Some(Ok((module, chain_id, log))) => { + supervisor.dispatch_log(&module, chain_id, log).await; + } + Some(Err(err)) => warn!(error = %err, "log stream error — continuing"), + None => {} + }, + } + } +} + +/// Wait for SIGINT or (on Unix) SIGTERM, whichever arrives first. +async fn wait_for_shutdown_signal() -> anyhow::Result<&'static str> { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut sigterm = signal(SignalKind::terminate())?; + let mut sigint = signal(SignalKind::interrupt())?; + tokio::select! { + _ = sigterm.recv() => Ok("SIGTERM"), + _ = sigint.recv() => Ok("SIGINT"), + } + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await?; + Ok("ctrl-c") + } +} diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs index 522e168..2ce576a 100644 --- a/crates/nexum-engine/src/manifest.rs +++ b/crates/nexum-engine/src/manifest.rs @@ -47,6 +47,54 @@ pub struct Manifest { pub capabilities: Option, #[serde(default)] pub config: toml::Table, + /// Event subscriptions the runtime wires before calling + /// `_init`. See `docs/02-modules-events-packaging.md` for the + /// schema; 0.2 implements `block` and `log` kinds, `cron` is + /// parsed and ignored (deferred to 0.3). + #[serde(default, rename = "subscription")] + pub subscriptions: Vec, +} + +/// One `[[subscription]]` table in `nexum.toml`. +/// +/// The discriminator is the `kind` field; remaining fields are +/// validated per-kind by the supervisor. Unknown kinds are surfaced +/// at load time so a typo does not silently disable an event source. +#[derive(Debug, Deserialize, Clone)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Subscription { + /// New-block events. Fan-out is shared per chain — the + /// supervisor opens one subscription per chain id and routes to + /// every module that asked for blocks on that chain. + Block { + /// EVM chain id. + chain_id: u64, + }, + /// Log events matching `address` + topic-0. Fan-out is + /// per-module — the supervisor opens one subscription per + /// `[[subscription]]` entry and tags emitted events with the + /// owning module. + Log { + /// EVM chain id. + chain_id: u64, + /// Contract address as `0x`-prefixed 20-byte hex. Optional. + #[serde(default)] + address: Option, + /// Topic-0 of the event the module wants to consume. `0x`- + /// prefixed 32-byte hex. Optional — when absent the + /// subscription matches every event from the address(es). + #[serde(default)] + event_signature: Option, + }, + /// Cron-scheduled tick. 0.2 parses but does not dispatch; the + /// supervisor emits a warning so the operator knows the + /// declaration is currently inert. `schedule` is preserved so a + /// 0.3 dispatcher can pick it up without re-parsing the manifest. + Cron { + /// Standard 5-field cron expression. + #[allow(dead_code)] + schedule: String, + }, } #[derive(Debug, Deserialize, Default)] diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs new file mode 100644 index 0000000..18c2fa4 --- /dev/null +++ b/crates/nexum-engine/src/supervisor.rs @@ -0,0 +1,387 @@ +//! Multi-module supervisor. +//! +//! Loads every `[[modules]]` entry from `engine.toml`, instantiates +//! each as a `Shepherd` bindings against a dedicated wasmtime +//! `Store`, and routes the event types declared in each manifest's +//! `[[subscription]]` table. +//! +//! 0.2 dispatches a block event to every module that subscribed to +//! that chain's blocks, and a log event only to the module that +//! opened the subscription. Restart + poison-pill bookkeeping ships +//! in a follow-up — for now a failing `_on_event` is logged via +//! `tracing::error!` and the module continues to receive subsequent +//! events. Lifecycle states `Restart` / `Dead` from +//! `docs/02-modules-events-packaging.md` land alongside the +//! `[module.restart]` schema in 0.3. + +use std::collections::BTreeSet; +use std::path::Path; + +use anyhow::{Context, Error, Result, anyhow}; +use tracing::{error, info, warn}; +use wasmtime::component::{Component, Linker, ResourceTable}; +use wasmtime::{Engine, Store}; +use wasmtime_wasi::WasiCtxBuilder; + +use crate::engine_config::{EngineConfig, ModuleEntry}; +use crate::host::cow_orderbook::OrderBookPool; +use crate::host::local_store_redb::LocalStore; +use crate::host::provider_pool::ProviderPool; +use crate::manifest::{self, LoadedManifest, Subscription}; +use crate::{HostState, Shepherd}; + +/// Owns every loaded module and exposes the dispatch surface the +/// event loop needs. +pub struct Supervisor { + modules: Vec, +} + +struct LoadedModule { + name: String, + bindings: Shepherd, + store: Store, + /// Subscriptions copied from `nexum.toml`. The supervisor reads + /// these on every event to decide whether to dispatch. + subscriptions: Vec, +} + +impl Supervisor { + /// Compile + instantiate every module declared in + /// `engine_cfg.modules`. The wasmtime `Engine` + `Linker` are + /// passed in so `main.rs` can build them once (the bindgen + /// `Shepherd::add_to_linker` call binds them to `HostState`, + /// which the supervisor does not re-derive). + pub async fn boot( + engine: &Engine, + linker: &Linker, + engine_cfg: &EngineConfig, + cow_pool: &OrderBookPool, + provider_pool: &ProviderPool, + local_store: &LocalStore, + ) -> Result { + let mut modules = Vec::with_capacity(engine_cfg.modules.len()); + for entry in &engine_cfg.modules { + let loaded = + Self::load_one(engine, linker, entry, cow_pool, provider_pool, local_store) + .await + .with_context(|| format!("load module {}", entry.path.display()))?; + modules.push(loaded); + } + info!(count = modules.len(), "supervisor up"); + Ok(Self { modules }) + } + + /// One-shot construction from a single ad-hoc `(component, manifest)` + /// pair. Used by the CLI-positional invocation so `just run` + /// against the example module keeps working without an + /// `engine.toml`. + pub async fn boot_single( + engine: &Engine, + linker: &Linker, + wasm: &Path, + manifest: Option<&Path>, + cow_pool: &OrderBookPool, + provider_pool: &ProviderPool, + local_store: &LocalStore, + ) -> Result { + let entry = ModuleEntry { + path: wasm.to_path_buf(), + manifest: manifest.map(Path::to_path_buf), + }; + let loaded = + Self::load_one(engine, linker, &entry, cow_pool, provider_pool, local_store).await?; + Ok(Self { + modules: vec![loaded], + }) + } + + async fn load_one( + engine: &Engine, + linker: &Linker, + entry: &ModuleEntry, + cow_pool: &OrderBookPool, + provider_pool: &ProviderPool, + local_store: &LocalStore, + ) -> Result { + let manifest_path = entry + .manifest + .clone() + .or_else(|| entry.path.parent().map(|p| p.join("nexum.toml"))); + let loaded_manifest: LoadedManifest = match manifest_path.as_deref() { + Some(p) if p.exists() => { + info!(manifest = %p.display(), "loading nexum.toml"); + manifest::load(p)? + } + _ => { + warn!( + component = %entry.path.display(), + "no nexum.toml — falling back to anonymous module" + ); + manifest::fallback_manifest() + } + }; + + // Compile + instantiate. + info!(component = %entry.path.display(), "compiling component"); + let component = Component::from_file(engine, &entry.path) + .map_err(Error::from) + .with_context(|| format!("compile {}", entry.path.display()))?; + let wasi = WasiCtxBuilder::new().inherit_stdio().build(); + let module_namespace = if loaded_manifest.manifest.module.name.is_empty() { + "module".to_owned() + } else { + loaded_manifest.manifest.module.name.clone() + }; + let mut store = Store::new( + engine, + HostState { + wasi, + table: ResourceTable::new(), + monotonic_baseline: std::time::Instant::now(), + http_allowlist: loaded_manifest.http_allowlist.clone(), + module_namespace: module_namespace.clone(), + cow: cow_pool.clone(), + chain: provider_pool.clone(), + store: local_store.clone(), + }, + ); + let bindings = Shepherd::instantiate_async(&mut store, &component, linker) + .await + .map_err(Error::from) + .with_context(|| format!("instantiate {}", entry.path.display()))?; + + // Call `init` with the manifest's `[config]`. + let config: crate::Config = if loaded_manifest.config.is_empty() { + vec![("name".into(), module_namespace.clone())] + } else { + loaded_manifest.config.clone() + }; + match bindings + .call_init(&mut store, &config) + .await + .map_err(Error::from)? + { + Ok(()) => info!(module = %module_namespace, "init succeeded"), + Err(e) => warn!( + module = %module_namespace, + domain = %e.domain, + kind = ?e.kind, + code = e.code, + message = %e.message, + "init failed", + ), + } + + // Surface any `[[subscription]]` entries the host cannot + // service yet, so an operator running 0.2 against a 0.3 + // manifest does not silently drop events. + for sub in &loaded_manifest.manifest.subscriptions { + if matches!(sub, Subscription::Cron { .. }) { + warn!( + module = %module_namespace, + "cron subscriptions are declared but inert in 0.2 (lands in 0.3)", + ); + } + } + + Ok(LoadedModule { + name: module_namespace, + bindings, + store, + subscriptions: loaded_manifest.manifest.subscriptions.clone(), + }) + } + + /// Number of modules currently loaded. + pub fn module_count(&self) -> usize { + self.modules.len() + } + + /// Set of chain ids any module asked for block events on. The + /// caller opens one shared block subscription per chain id and + /// routes through `dispatch_block`. + pub fn block_chains(&self) -> BTreeSet { + let mut out = BTreeSet::new(); + for module in &self.modules { + for sub in &module.subscriptions { + if let Subscription::Block { chain_id } = sub { + out.insert(*chain_id); + } + } + } + out + } + + /// Per-module log subscriptions. Each entry is a `(module_name, + /// chain_id, filter)` triple the event loop opens against the + /// matching alloy provider; the resulting stream tags every log + /// with `module_name` so `dispatch_log` routes correctly. + pub fn log_subscriptions(&self) -> Vec<(String, u64, alloy_rpc_types_eth::Filter)> { + let mut out = Vec::new(); + for module in &self.modules { + for sub in &module.subscriptions { + if let Subscription::Log { + chain_id, + address, + event_signature, + } = sub + { + match build_alloy_filter(address.as_deref(), event_signature.as_deref()) { + Ok(filter) => out.push((module.name.clone(), *chain_id, filter)), + Err(err) => warn!( + module = %module.name, + chain_id, + error = %err, + "invalid log subscription — skipping", + ), + } + } + } + } + out + } + + /// Dispatch a block event to every module subscribed to + /// `block.chain_id`. Returns the number of modules invoked. + pub async fn dispatch_block(&mut self, block: crate::nexum::host::types::Block) -> usize { + let event = crate::nexum::host::types::Event::Block(block); + let chain_id = match &event { + crate::nexum::host::types::Event::Block(b) => b.chain_id, + _ => unreachable!(), + }; + let mut dispatched = 0; + for module in &mut self.modules { + let subscribed = module + .subscriptions + .iter() + .any(|s| matches!(s, Subscription::Block { chain_id: cid } if *cid == chain_id)); + if !subscribed { + continue; + } + match module + .bindings + .call_on_event(&mut module.store, &event) + .await + { + Ok(Ok(())) => dispatched += 1, + Ok(Err(host_err)) => warn!( + module = %module.name, + chain_id, + domain = %host_err.domain, + kind = ?host_err.kind, + message = %host_err.message, + "on-event returned host-error", + ), + Err(trap) => error!( + module = %module.name, + chain_id, + error = %trap, + "on-event trapped", + ), + } + } + dispatched + } + + /// Dispatch a log event to the specific module that opened the + /// subscription. Returns `true` when the module accepted the + /// dispatch; `false` when the module was not found or its + /// callback failed. + pub async fn dispatch_log( + &mut self, + module_name: &str, + chain_id: u64, + log: alloy_rpc_types_eth::Log, + ) -> bool { + let target = match self.modules.iter_mut().find(|m| m.name == module_name) { + Some(m) => m, + None => { + warn!(module = %module_name, "no such module — dropping log"); + return false; + } + }; + let event = crate::nexum::host::types::Event::Logs(vec![project_log(chain_id, &log)]); + match target + .bindings + .call_on_event(&mut target.store, &event) + .await + { + Ok(Ok(())) => true, + Ok(Err(host_err)) => { + warn!( + module = %module_name, + chain_id, + domain = %host_err.domain, + kind = ?host_err.kind, + message = %host_err.message, + "on-event returned host-error", + ); + false + } + Err(trap) => { + error!( + module = %module_name, + chain_id, + error = %trap, + "on-event trapped", + ); + false + } + } + } +} + +/// Project an alloy `Log` onto the WIT `log` record. The chain id +/// is not on the alloy log (the subscription context carries it), +/// so we receive it alongside. +fn project_log(chain_id: u64, log: &alloy_rpc_types_eth::Log) -> crate::nexum::host::types::Log { + crate::nexum::host::types::Log { + chain_id, + address: log.address().as_slice().to_vec(), + topics: log.topics().iter().map(|t| t.as_slice().to_vec()).collect(), + data: log.inner.data.data.to_vec(), + block_number: log.block_number.unwrap_or(0), + transaction_hash: log + .transaction_hash + .map(|h| h.as_slice().to_vec()) + .unwrap_or_default(), + log_index: log.log_index.unwrap_or(0) as u32, + } +} + +/// Translate a `[[subscription]]` log entry into an alloy `Filter`. +fn build_alloy_filter( + address: Option<&str>, + event_signature: Option<&str>, +) -> Result { + use alloy_primitives::{Address, B256}; + let mut filter = alloy_rpc_types_eth::Filter::new(); + if let Some(addr_hex) = address { + let addr: Address = addr_hex + .parse() + .map_err(|e| anyhow!("invalid log address {addr_hex:?}: {e}"))?; + filter = filter.address(addr); + } + if let Some(topic_hex) = event_signature { + let topic: B256 = topic_hex + .parse() + .map_err(|e| anyhow!("invalid topic {topic_hex:?}: {e}"))?; + filter = filter.event_signature(topic); + } + Ok(filter) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_supervisor_returns_no_subscriptions() { + let sup = Supervisor { + modules: Vec::new(), + }; + assert!(sup.block_chains().is_empty()); + assert!(sup.log_subscriptions().is_empty()); + assert_eq!(sup.module_count(), 0); + } +} From bfe74016f1432fb93e5c83e81f8de7cb70fd5c62 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 9 Jun 2026 17:59:38 -0300 Subject: [PATCH 07/29] feat(supervisor): apply ADR-0001/0003/0005/0016 and trap-based module death (BLEU-813-817) --- crates/nexum-engine/src/engine_config.rs | 8 +- crates/nexum-engine/src/host/cow_orderbook.rs | 21 +- .../nexum-engine/src/host/local_store_redb.rs | 83 +++++--- crates/nexum-engine/src/main.rs | 6 +- crates/nexum-engine/src/manifest.rs | 190 ++++++++++++++++-- crates/nexum-engine/src/supervisor.rs | 90 ++++++--- 6 files changed, 311 insertions(+), 87 deletions(-) diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs index 595a688..1246850 100644 --- a/crates/nexum-engine/src/engine_config.rs +++ b/crates/nexum-engine/src/engine_config.rs @@ -1,6 +1,6 @@ //! Engine-side runtime configuration. //! -//! Distinct from `nexum.toml` (module manifest): this file describes +//! Distinct from `module.toml` (module manifest): this file describes //! the *engine*'s I/O wiring — chain RPC endpoints and the on-disk //! location of the `local-store` database. Both are required for the //! 0.2 reference engine to do anything other than print stubs. @@ -33,7 +33,7 @@ pub struct EngineConfig { #[serde(default)] pub chains: BTreeMap, /// Modules the supervisor should boot. Each entry resolves a - /// `(component.wasm, nexum.toml)` pair on the local filesystem + /// `(component.wasm, module.toml)` pair on the local filesystem /// for 0.2 — content-addressed resolution (Swarm / OCI / /// `[[content.sources]]`) lands in 0.3 per /// `docs/03-module-discovery.md`. @@ -44,13 +44,13 @@ pub struct EngineConfig { /// One `[[modules]]` table from `engine.toml`. /// /// Both fields are filesystem paths in 0.2. `manifest` defaults to -/// `nexum.toml` next to `path` if omitted, matching the bundle layout +/// `module.toml` next to `path` if omitted, matching the bundle layout /// in `docs/02-modules-events-packaging.md`. #[derive(Debug, Deserialize)] pub struct ModuleEntry { /// Path to the compiled `.wasm` component. pub path: std::path::PathBuf, - /// Path to the module's `nexum.toml`. Defaults to `/nexum.toml`. + /// Path to the module's `module.toml`. Defaults to `/module.toml`. #[serde(default)] pub manifest: Option, } diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index cd11173..227efd1 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -28,12 +28,12 @@ pub struct OrderBookPool { http: reqwest::Client, } -impl OrderBookPool { - /// Build a pool covering every `cowprotocol::Chain` variant. The - /// default `OrderBookApi::new(chain)` constructor uses the canonical - /// `api.cow.fi/{slug}/api/v1` base URL from the SDK; callers that - /// need barn or a custom staging URL override per chain. - pub fn with_default_chains() -> Self { +impl Default for OrderBookPool { + /// Build a pool covering every `cowprotocol::Chain` variant. Each entry + /// uses the canonical `api.cow.fi/{slug}/api/v1` base URL from the SDK. + /// Override individual entries via `OrderBookApi::new_with_base_url` for + /// barn or staging targets. + fn default() -> Self { let http = reqwest::Client::new(); let chains = [ Chain::Mainnet, @@ -48,6 +48,9 @@ impl OrderBookPool { .collect(); Self { clients, http } } +} + +impl OrderBookPool { /// Look up the client for a chain. pub fn get(&self, chain_id: u64) -> Result<&OrderBookApi, CowApiError> { @@ -145,7 +148,7 @@ mod tests { #[test] fn pool_indexes_default_chains() { - let pool = OrderBookPool::with_default_chains(); + let pool = OrderBookPool::default(); assert!(pool.get(1).is_ok(), "mainnet present"); assert!(pool.get(100).is_ok(), "gnosis present"); assert!(pool.get(11_155_111).is_ok(), "sepolia present"); @@ -155,7 +158,7 @@ mod tests { #[test] fn unknown_chain_surfaces_typed_error() { - let pool = OrderBookPool::with_default_chains(); + let pool = OrderBookPool::default(); assert!(matches!( pool.get(99_999), Err(CowApiError::UnknownChain(99_999)) @@ -224,7 +227,7 @@ mod tests { #[tokio::test] async fn request_rejects_unknown_method() { - let pool = OrderBookPool::with_default_chains(); + let pool = OrderBookPool::default(); let err = pool .request(Chain::Mainnet.id(), "PATCH", "/x", None) .await diff --git a/crates/nexum-engine/src/host/local_store_redb.rs b/crates/nexum-engine/src/host/local_store_redb.rs index 4e535e0..46d980e 100644 --- a/crates/nexum-engine/src/host/local_store_redb.rs +++ b/crates/nexum-engine/src/host/local_store_redb.rs @@ -1,41 +1,43 @@ //! `nexum:host/local-store` backend. //! //! Single redb file under `EngineConfig.engine.state_dir`. Per-module -//! namespacing is enforced host-side via a `[len:u8][module_name][raw_key]` -//! prefix on every redb key. Two modules using the same key string see -//! disjoint data. +//! namespacing is enforced host-side via a fixed-length 32-byte prefix: +//! `keccak256(module_name) ++ raw_key`. Two modules using the same key +//! string see disjoint data regardless of how similar their names are. //! -//! The runtime supplies the namespace; modules see plain key strings. -//! Module names longer than 255 bytes are rejected at construction -//! (matches the one-byte length prefix). - -// The redb error enum is large by construction (Txn / Storage / -// Commit each carry a redb backtrace ≈ 160 bytes). Allowing the -// cap-on-Result-size lint here is the lesser evil: boxing every -// variant pushes the error path to the heap just to humour the lint. +//! The 32-byte hash prefix has two properties that the old +//! `[len:u8][name][key]` scheme lacked: +//! +//! - **Fixed width** — no length field to forge; a module cannot craft a +//! key that bleeds into another module's prefix range. +//! - **ENS-compatible** — keccak256 is the same hash used by ENS node +//! derivation, so module identities can be derived from ENS names +//! without extra hashing in the future (ADR-0003). + #![allow(clippy::result_large_err)] use std::path::Path; use std::sync::Arc; +use alloy_primitives::keccak256; use redb::{Database, ReadableTable, TableDefinition}; use thiserror::Error; const TABLE: TableDefinition<'static, &[u8], &[u8]> = TableDefinition::new("nexum:local-store"); -const MAX_NAMESPACE_LEN: usize = u8::MAX as usize; +#[cfg(test)] +const PREFIX_LEN: usize = 32; /// Process-wide handle to the local-store redb database. Cheap to -/// clone; the per-module view is constructed by setting the -/// namespace prefix at call time. +/// clone; the per-module view is constructed by setting the namespace +/// prefix at call time. #[derive(Debug, Clone)] pub struct LocalStore { db: Arc, } impl LocalStore { - /// Open (or create) the redb file at `path`. Materialises the - /// shared table so subsequent read transactions never hit - /// `TableDoesNotExist`. + /// Open (or create) the redb file at `path`. Materialises the shared + /// table so subsequent read transactions never hit `TableDoesNotExist`. pub fn open(path: impl AsRef) -> Result { let db = Database::create(path).map_err(StorageError::Open)?; { @@ -47,7 +49,7 @@ impl LocalStore { } /// Fetch a value for `(namespace, key)`. Returns `Ok(None)` when - /// no entry exists; module never observes the prefix. + /// no entry exists; the module never observes the prefix. pub fn get(&self, namespace: &str, key: &str) -> Result>, StorageError> { let full = build_key(namespace, key)?; let txn = self.db.begin_read().map_err(StorageError::Txn)?; @@ -87,9 +89,9 @@ impl LocalStore { Ok(()) } - /// Enumerate keys in `namespace` whose raw key (post-prefix) - /// starts with `prefix`. Returns only the module-visible key - /// strings — the host strips the namespace prefix. + /// Enumerate keys in `namespace` whose raw key (post-prefix) starts + /// with `prefix`. Returns only the module-visible key strings — the + /// host strips the namespace prefix. pub fn list_keys(&self, namespace: &str, prefix: &str) -> Result, StorageError> { let ns_prefix = namespace_prefix(namespace)?; let full_prefix = build_key(namespace, prefix)?; @@ -109,23 +111,15 @@ impl LocalStore { } } +/// Returns the 32-byte keccak256 hash of `namespace` as a `Vec`. +/// Rejects the empty string so callers can rely on a non-trivial prefix. fn namespace_prefix(namespace: &str) -> Result, StorageError> { if namespace.is_empty() { return Err(StorageError::InvalidNamespace( "module namespace must not be empty".into(), )); } - let bytes = namespace.as_bytes(); - if bytes.len() > MAX_NAMESPACE_LEN { - return Err(StorageError::InvalidNamespace(format!( - "namespace `{namespace}` is {} bytes; max is {MAX_NAMESPACE_LEN}", - bytes.len() - ))); - } - let mut out = Vec::with_capacity(1 + bytes.len()); - out.push(bytes.len() as u8); - out.extend_from_slice(bytes); - Ok(out) + Ok(keccak256(namespace.as_bytes()).to_vec()) } fn build_key(namespace: &str, key: &str) -> Result, StorageError> { @@ -208,4 +202,29 @@ mod tests { let err = store.set("", "k", b"v").unwrap_err(); assert!(matches!(err, StorageError::InvalidNamespace(_))); } + + #[test] + fn prefix_is_fixed_32_bytes() { + let short = namespace_prefix("a").unwrap(); + let long = namespace_prefix(&"a".repeat(300)).unwrap(); + assert_eq!(short.len(), PREFIX_LEN); + assert_eq!(long.len(), PREFIX_LEN); + // Different inputs produce different prefixes. + assert_ne!(short, long); + } + + #[test] + fn prefix_is_deterministic() { + let p1 = namespace_prefix("twap-monitor").unwrap(); + let p2 = namespace_prefix("twap-monitor").unwrap(); + assert_eq!(p1, p2); + } + + #[test] + fn similar_names_differ() { + // Verify that names that share a common prefix don't collide. + let pa = namespace_prefix("module-a").unwrap(); + let pb = namespace_prefix("module-b").unwrap(); + assert_ne!(pa, pb); + } } diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 56e7be7..6db49da 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -69,7 +69,7 @@ struct HostState { /// Origin for `clock::monotonic-ns`. Differences between successive /// readings are the only meaningful values. monotonic_baseline: Instant, - /// Per-module `[capabilities.http].allow` allowlist (from nexum.toml). + /// Per-module `[capabilities.http].allow` allowlist (from module.toml). /// Consulted by `http::fetch` before any outbound call. http_allowlist: Vec, /// Namespace for the running module's `local-store` rows. Set from @@ -429,7 +429,7 @@ impl nexum::host::http::Host for HostState { code: 0, message: format!( "host {host} not in [capabilities.http].allow; \ - add it to nexum.toml to permit" + add it to module.toml to permit" ), data: None, }); @@ -485,7 +485,7 @@ async fn main() -> anyhow::Result<()> { let store_path = engine_cfg.engine.state_dir.join("local-store.redb"); let local_store = host::local_store_redb::LocalStore::open(&store_path) .with_context(|| format!("open local-store at {}", store_path.display()))?; - let cow_pool = host::cow_orderbook::OrderBookPool::with_default_chains(); + let cow_pool = host::cow_orderbook::OrderBookPool::default(); let provider_pool = host::provider_pool::ProviderPool::from_config(&engine_cfg) .await .context("open chain providers")?; diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs index 2ce576a..d1ce6bc 100644 --- a/crates/nexum-engine/src/manifest.rs +++ b/crates/nexum-engine/src/manifest.rs @@ -1,7 +1,6 @@ -//! Minimal `nexum.toml` parser and capability-enforcement helpers (0.2 scope). +//! `module.toml` parser and capability-enforcement helpers (0.2 scope). //! -//! 0.2 intentionally ships a slim subset of the manifest spec described in -//! the migration guide §3: +//! 0.2 intentionally ships a slim subset of the manifest spec: //! //! - `[capabilities].required` is parsed and validated (names must be in //! the known capability set; the 0.2 reference engine always provides @@ -14,9 +13,9 @@ //! module's `init`. Typed `config-value` variant is deferred to 0.3. //! //! When the manifest file is missing or has no `[capabilities]` section, -//! a deprecation warning is emitted on stderr and the engine falls back -//! to 0.1 behaviour (treat every linked capability as required). This -//! fallback will be removed in 0.3. +//! a deprecation warning is emitted and the engine falls back to 0.1 +//! behaviour (treat every linked capability as required). This fallback +//! will be removed in 0.3. use std::collections::HashSet; use std::path::Path; @@ -55,7 +54,7 @@ pub struct Manifest { pub subscriptions: Vec, } -/// One `[[subscription]]` table in `nexum.toml`. +/// One `[[subscription]]` table in `module.toml`. /// /// The discriminator is the `kind` field; remaining fields are /// validated per-kind by the supervisor. Unknown kinds are surfaced @@ -162,7 +161,88 @@ pub struct LoadedManifest { pub config: Vec<(String, String)>, } -/// Read `nexum.toml` from `path`, parse, validate, and emit a deprecation +/// Error returned when a component's WIT imports exceed its declared capabilities. +#[derive(Debug)] +pub struct CapabilityViolation { + /// Capability name (e.g. `"remote-store"`). + pub capability: String, + /// Full WIT import name as it appeared in the component (e.g. + /// `"nexum:host/remote-store@0.2.0"`). + pub wit_import: String, +} + +impl std::fmt::Display for CapabilityViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "component imports `{}` ({}) but it is not listed in \ + [capabilities].required or [capabilities].optional", + self.capability, self.wit_import + ) + } +} + +impl std::error::Error for CapabilityViolation {} + +/// Check that every capability-bearing WIT import of the component is covered +/// by the module's manifest declarations. Call this after loading the +/// component but before instantiation. +/// +/// When `[capabilities]` is absent the manifest is in 0.1-fallback mode and +/// all imports are allowed; the caller is expected to have already emitted +/// a deprecation warning. +/// +/// `component_imports` should be the iterator returned by +/// `component.component_type().imports(&engine)` — pass the **name** part +/// (`&str`) of each `(&str, ComponentItem)` tuple. +pub fn enforce_capabilities<'a>( + loaded: &LoadedManifest, + component_imports: impl Iterator, +) -> Result<(), CapabilityViolation> { + let caps = match loaded.manifest.capabilities.as_ref() { + None => return Ok(()), // 0.1-fallback: no enforcement + Some(c) => c, + }; + + let declared: HashSet<&str> = caps + .required + .iter() + .chain(caps.optional.iter()) + .map(String::as_str) + .collect(); + + for import_name in component_imports { + if let Some(cap) = wit_import_to_cap(import_name) { + if !declared.contains(cap) { + return Err(CapabilityViolation { + capability: cap.to_owned(), + wit_import: import_name.to_owned(), + }); + } + } + } + Ok(()) +} + +/// Map a WIT import name to a capability name, or `None` for non-capability +/// imports (wasi:*, wasi:io, wasi:cli, etc.). +/// +/// Examples: +/// - `"nexum:host/chain@0.2.0"` → `Some("chain")` +/// - `"shepherd:cow/cow-api@0.2.0"` → `Some("cow-api")` +/// - `"wasi:io/streams@0.2.0"` → `None` +fn wit_import_to_cap(import_name: &str) -> Option<&str> { + let without_version = import_name.split('@').next().unwrap_or(import_name); + if let Some(iface) = without_version.strip_prefix("nexum:host/") { + Some(iface) + } else if let Some(iface) = without_version.strip_prefix("shepherd:cow/") { + Some(iface) + } else { + None + } +} + +/// Read `module.toml` from `path`, parse, validate, and emit a deprecation /// warning if `[capabilities]` is absent (0.1-compat fallback). pub fn load(path: &Path) -> Result { let raw = std::fs::read_to_string(path).map_err(ParseError::Io)?; @@ -171,10 +251,9 @@ pub fn load(path: &Path) -> Result { let caps = manifest.capabilities.as_ref(); if caps.is_none() { eprintln!( - "[deprecation] no [capabilities] section in nexum.toml — \ + "[deprecation] no [capabilities] section in module.toml — \ defaulting to all-required (0.1 behaviour). This default \ - will be removed in 0.3; add an explicit [capabilities] block \ - now." + will be removed in 0.3; add an explicit [capabilities] block." ); } @@ -221,13 +300,13 @@ pub fn load(path: &Path) -> Result { }) } -/// Synthesise a "0.1 fallback" manifest for when no `nexum.toml` is found. +/// Synthesise a "0.1 fallback" manifest for when no `module.toml` is found. /// Emits the same deprecation warning as a missing-section manifest. pub fn fallback_manifest() -> LoadedManifest { eprintln!( - "[deprecation] no nexum.toml found — defaulting to all-required \ + "[deprecation] no module.toml found — defaulting to all-required \ (0.1 behaviour). This default will be removed in 0.3; ship a \ - nexum.toml alongside your component." + module.toml alongside your component." ); LoadedManifest { manifest: Manifest::default(), @@ -310,4 +389,87 @@ mod tests { assert!(!host_allowed("discord.com", &allow)); assert!(!host_allowed("nope.example", &allow)); } + + // ── capability enforcement ──────────────────────────────────────────── + + #[test] + fn wit_import_to_cap_nexum_host() { + assert_eq!(wit_import_to_cap("nexum:host/chain@0.2.0"), Some("chain")); + assert_eq!( + wit_import_to_cap("nexum:host/local-store@0.2.0"), + Some("local-store") + ); + assert_eq!(wit_import_to_cap("nexum:host/http@0.2.0"), Some("http")); + } + + #[test] + fn wit_import_to_cap_shepherd_cow() { + assert_eq!( + wit_import_to_cap("shepherd:cow/cow-api@0.2.0"), + Some("cow-api") + ); + } + + #[test] + fn wit_import_to_cap_wasi_is_none() { + assert_eq!(wit_import_to_cap("wasi:io/streams@0.2.0"), None); + assert_eq!(wit_import_to_cap("wasi:cli/stdin@0.2.0"), None); + } + + fn manifest_with_caps(required: &[&str], optional: &[&str]) -> LoadedManifest { + LoadedManifest { + manifest: Manifest { + capabilities: Some(CapabilitiesSection { + required: required.iter().map(|s| s.to_string()).collect(), + optional: optional.iter().map(|s| s.to_string()).collect(), + http: None, + }), + ..Default::default() + }, + http_allowlist: vec![], + config: vec![], + } + } + + fn manifest_no_caps() -> LoadedManifest { + LoadedManifest { + manifest: Manifest::default(), + http_allowlist: vec![], + config: vec![], + } + } + + #[test] + fn enforce_passes_when_caps_absent() { + let loaded = manifest_no_caps(); + let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; + assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); + } + + #[test] + fn enforce_passes_when_all_imports_declared() { + let loaded = manifest_with_caps(&["chain", "cow-api"], &["http"]); + let imports = [ + "nexum:host/chain@0.2.0", + "shepherd:cow/cow-api@0.2.0", + "nexum:host/http@0.2.0", + "wasi:io/streams@0.2.0", + ]; + assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); + } + + #[test] + fn enforce_rejects_undeclared_import() { + let loaded = manifest_with_caps(&["chain"], &[]); + let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; + let err = enforce_capabilities(&loaded, imports.into_iter()).unwrap_err(); + assert_eq!(err.capability, "remote-store"); + } + + #[test] + fn enforce_optional_caps_are_also_allowed() { + let loaded = manifest_with_caps(&["chain"], &["remote-store"]); + let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; + assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); + } } diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 18c2fa4..89e02f2 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -5,14 +5,11 @@ //! `Store`, and routes the event types declared in each manifest's //! `[[subscription]]` table. //! -//! 0.2 dispatches a block event to every module that subscribed to -//! that chain's blocks, and a log event only to the module that -//! opened the subscription. Restart + poison-pill bookkeeping ships -//! in a follow-up — for now a failing `_on_event` is logged via -//! `tracing::error!` and the module continues to receive subsequent -//! events. Lifecycle states `Restart` / `Dead` from -//! `docs/02-modules-events-packaging.md` land alongside the -//! `[module.restart]` schema in 0.3. +//! Trap handling (BLEU-817): a wasmtime trap in `on_event` marks the +//! module as `alive = false` and removes it from all future dispatch. +//! The module's subscriptions remain registered (the event-loop +//! streams are not closed) but the dispatcher skips dead modules. +//! Full restart-with-backoff lands in 0.3. use std::collections::BTreeSet; use std::path::Path; @@ -40,9 +37,13 @@ struct LoadedModule { name: String, bindings: Shepherd, store: Store, - /// Subscriptions copied from `nexum.toml`. The supervisor reads + /// Subscriptions copied from `module.toml`. The supervisor reads /// these on every event to decide whether to dispatch. subscriptions: Vec, + /// Set to `false` when `on_event` traps. Dead modules are silently + /// skipped on every subsequent dispatch. Full restart-with-backoff + /// lands in 0.3. + alive: bool, } impl Supervisor { @@ -103,19 +104,33 @@ impl Supervisor { provider_pool: &ProviderPool, local_store: &LocalStore, ) -> Result { - let manifest_path = entry - .manifest - .clone() - .or_else(|| entry.path.parent().map(|p| p.join("nexum.toml"))); + // Canonical name is module.toml (ADR-0001). nexum.toml is accepted + // with a deprecation warning during the 0.1→0.2 transition. + let manifest_path = entry.manifest.clone().or_else(|| { + let dir = entry.path.parent()?.to_owned(); + let canonical = dir.join("module.toml"); + if canonical.exists() { + return Some(canonical); + } + let legacy = dir.join("nexum.toml"); + if legacy.exists() { + eprintln!( + "[deprecation] nexum.toml is deprecated; rename to module.toml \ + (ADR-0001). Support will be removed in 0.3." + ); + return Some(legacy); + } + None + }); let loaded_manifest: LoadedManifest = match manifest_path.as_deref() { Some(p) if p.exists() => { - info!(manifest = %p.display(), "loading nexum.toml"); + info!(manifest = %p.display(), "loading module manifest"); manifest::load(p)? } _ => { warn!( component = %entry.path.display(), - "no nexum.toml — falling back to anonymous module" + "no module.toml — falling back to anonymous module" ); manifest::fallback_manifest() } @@ -126,6 +141,14 @@ impl Supervisor { let component = Component::from_file(engine, &entry.path) .map_err(Error::from) .with_context(|| format!("compile {}", entry.path.display()))?; + + // Enforce capability declarations before spending time on instantiation. + manifest::enforce_capabilities( + &loaded_manifest, + component.component_type().imports(engine).map(|(n, _)| n), + ) + .map_err(|e| Error::msg(e.to_string())) + .with_context(|| format!("capability violation in {}", entry.path.display()))?; let wasi = WasiCtxBuilder::new().inherit_stdio().build(); let module_namespace = if loaded_manifest.manifest.module.name.is_empty() { "module".to_owned() @@ -189,6 +212,7 @@ impl Supervisor { bindings, store, subscriptions: loaded_manifest.manifest.subscriptions.clone(), + alive: true, }) } @@ -243,6 +267,7 @@ impl Supervisor { /// Dispatch a block event to every module subscribed to /// `block.chain_id`. Returns the number of modules invoked. + /// Modules that trap are marked dead and excluded from future dispatch. pub async fn dispatch_block(&mut self, block: crate::nexum::host::types::Block) -> usize { let event = crate::nexum::host::types::Event::Block(block); let chain_id = match &event { @@ -251,6 +276,9 @@ impl Supervisor { }; let mut dispatched = 0; for module in &mut self.modules { + if !module.alive { + continue; + } let subscribed = module .subscriptions .iter() @@ -272,21 +300,24 @@ impl Supervisor { message = %host_err.message, "on-event returned host-error", ), - Err(trap) => error!( - module = %module.name, - chain_id, - error = %trap, - "on-event trapped", - ), + Err(trap) => { + error!( + module = %module.name, + chain_id, + error = %trap, + "on-event trapped — module marked dead, removed from dispatch", + ); + module.alive = false; + } } } dispatched } /// Dispatch a log event to the specific module that opened the - /// subscription. Returns `true` when the module accepted the - /// dispatch; `false` when the module was not found or its - /// callback failed. + /// subscription. Returns `true` when the module accepted the dispatch; + /// `false` when the module is dead, not found, or its callback failed. + /// A trapping module is marked dead and excluded from future dispatch. pub async fn dispatch_log( &mut self, module_name: &str, @@ -300,6 +331,9 @@ impl Supervisor { return false; } }; + if !target.alive { + return false; + } let event = crate::nexum::host::types::Event::Logs(vec![project_log(chain_id, &log)]); match target .bindings @@ -323,12 +357,18 @@ impl Supervisor { module = %module_name, chain_id, error = %trap, - "on-event trapped", + "on-event trapped — module marked dead, removed from dispatch", ); + target.alive = false; false } } } + + /// Count of modules currently alive (not dead due to traps). + pub fn alive_count(&self) -> usize { + self.modules.iter().filter(|m| m.alive).count() + } } /// Project an alloy `Log` onto the WIT `log` record. The chain id From eb22f09ebfb8f8b5eab503274a30156c8fe9f6d8 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 9 Jun 2026 18:04:18 -0300 Subject: [PATCH 08/29] feat(supervisor): add fuel + memory limits per module store (BLEU-818) --- crates/nexum-engine/src/main.rs | 12 ++++++++++++ crates/nexum-engine/src/supervisor.rs | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 6db49da..37d892e 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -63,9 +63,20 @@ wasmtime::component::bindgen!({ use nexum::host::types::HostErrorKind; +/// Default fuel budget granted per `on_event` invocation (≈ 1 billion WASM +/// instructions). Modules that exceed this budget trap with `OutOfFuel`. +/// Configurable per-module via `engine.toml` in 0.3. +pub const DEFAULT_FUEL_PER_EVENT: u64 = 1_000_000_000; + +/// Default linear-memory cap per module store (64 MiB). Prevents a single +/// runaway module from exhausting process memory. Configurable in 0.3. +pub const DEFAULT_MEMORY_LIMIT: usize = 64 * 1024 * 1024; + struct HostState { wasi: WasiCtx, table: ResourceTable, + /// Wasmtime memory/table/instance resource limits for this store. + limits: wasmtime::StoreLimits, /// Origin for `clock::monotonic-ns`. Differences between successive /// readings are the only meaningful values. monotonic_baseline: Instant, @@ -493,6 +504,7 @@ async fn main() -> anyhow::Result<()> { // wasmtime engine + linker — one of each, shared across modules. let mut config = wasmtime::Config::new(); config.wasm_component_model(true); + config.consume_fuel(true); let engine = Engine::new(&config)?; let mut linker = Linker::::new(&engine); diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 89e02f2..9bb4cb3 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -155,11 +155,15 @@ impl Supervisor { } else { loaded_manifest.manifest.module.name.clone() }; + let limits = wasmtime::StoreLimitsBuilder::new() + .memory_size(crate::DEFAULT_MEMORY_LIMIT) + .build(); let mut store = Store::new( engine, HostState { wasi, table: ResourceTable::new(), + limits, monotonic_baseline: std::time::Instant::now(), http_allowlist: loaded_manifest.http_allowlist.clone(), module_namespace: module_namespace.clone(), @@ -168,6 +172,8 @@ impl Supervisor { store: local_store.clone(), }, ); + store.limiter(|state| &mut state.limits); + store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT)?; let bindings = Shepherd::instantiate_async(&mut store, &component, linker) .await .map_err(Error::from) @@ -194,6 +200,8 @@ impl Supervisor { "init failed", ), } + // Refuel after init so the first on_event starts with a full budget. + store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT)?; // Surface any `[[subscription]]` entries the host cannot // service yet, so an operator running 0.2 against a 0.3 @@ -286,6 +294,11 @@ impl Supervisor { if !subscribed { continue; } + // Refuel before each invocation so each event gets a fresh budget. + if let Err(e) = module.store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT) { + error!(module = %module.name, error = %e, "set_fuel failed — skipping"); + continue; + } match module .bindings .call_on_event(&mut module.store, &event) @@ -334,6 +347,10 @@ impl Supervisor { if !target.alive { return false; } + if let Err(e) = target.store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT) { + error!(module = %module_name, error = %e, "set_fuel failed — skipping"); + return false; + } let event = crate::nexum::host::types::Event::Logs(vec![project_log(chain_id, &log)]); match target .bindings From efbe70b2c939657c3efd359071085623907c8fa8 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 9 Jun 2026 21:50:36 -0300 Subject: [PATCH 09/29] docs: rename nexum.toml -> module.toml in example, justfile, and README (BLEU-820) --- README.md | 62 +++++++++++++++++++++++++++++++++++++ justfile | 8 +++-- modules/example/module.toml | 27 ++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 modules/example/module.toml diff --git a/README.md b/README.md index d146082..f6be460 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,72 @@ just build # Run the runtime against the example module just run + +# Run unit tests +just test ``` Without Nix, you need: Rust (edition 2024, see `rust-toolchain.toml` if present), the `wasm32-wasip2` target, and `wasm-tools`. +## Running + +### Single-module (development) + +```sh +nexum-engine [] +``` + +The `module.toml` is optional; without it the engine prints a deprecation warning and loads the module with empty capabilities and config (0.1 fallback). + +### Multi-module (production) + +```sh +nexum-engine --engine-config engine.toml +``` + +`engine.toml` declares RPC endpoints, the state directory, and a `[[modules]]` list: + +```toml +[engine] +state_dir = "/var/lib/shepherd" +log_level = "info" + +[[chains]] +id = 1 +url = "wss://mainnet.infura.io/ws/v3/..." + +[[modules]] +path = "modules/twap-monitor/twap-monitor.wasm" +manifest = "modules/twap-monitor/module.toml" + +[[modules]] +path = "modules/ethflow-watcher/ethflow-watcher.wasm" +``` + +### Module manifest (`module.toml`) + +```toml +[module] +name = "twap-monitor" +version = "0.1.0" + +[capabilities] +required = ["chain", "local-store", "cow-api"] +optional = ["http"] + +[capabilities.http] +allow = ["api.cow.fi"] + +[[subscription]] +kind = "log" +chain_id = 1 +address = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110" # ComposableCoW + +[[subscription]] +kind = "block" +chain_id = 1 +``` + ## Documentation The `docs/` directory contains the design corpus: diff --git a/justfile b/justfile index 3e01212..e3ebd15 100644 --- a/justfile +++ b/justfile @@ -10,10 +10,14 @@ build-module: build: build-engine build-module # Build the module then run the engine with it. The second argument is the -# module's nexum.toml — without it the engine prints the 0.1-compat +# module's module.toml — without it the engine prints the 0.1-compat # deprecation warning and proceeds with empty capabilities/config. run: build-module build-engine - cargo run -p nexum-engine -- target/wasm32-wasip2/release/example.wasm modules/example/nexum.toml + cargo run -p nexum-engine -- target/wasm32-wasip2/release/example.wasm modules/example/module.toml + +# Run host engine unit tests +test: + cargo test -p nexum-engine # Check the entire workspace check: diff --git a/modules/example/module.toml b/modules/example/module.toml new file mode 100644 index 0000000..e17a547 --- /dev/null +++ b/modules/example/module.toml @@ -0,0 +1,27 @@ +# Example module manifest — exercises the 0.2 manifest schema end-to-end. + +[module] +name = "example" +version = "0.1.0" +# Placeholder content hash. 0.2 parses but does not verify this; 0.3 will +# compare it against the sha256 of the loaded component bytes. +component = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + +[capabilities] +# 0.2 reference engine provides all listed capabilities; this list is a +# sanity check + future-proofing. +required = ["logging"] + +# Capabilities the module would use opportunistically. In 0.2 these are +# parsed and logged; trap-stub fallback for absent optionals ships in 0.3. +optional = [] + +[capabilities.http] +# Per-module HTTP allowlist. Empty list = no outbound HTTP permitted. +# Entries are exact hostnames or *.domain wildcards. +allow = [] + +[config] +# Stringly-typed in 0.2 (typed variant on 0.3 roadmap). Numbers and +# booleans are flattened to their text form by the host on the way through. +name = "example" From 771f2841439b28204760a29a38c0c2505f8dc32b Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 9 Jun 2026 22:02:40 -0300 Subject: [PATCH 10/29] =?UTF-8?q?test:=20fill=20host=20backend=20test=20ga?= =?UTF-8?q?ps=20=E2=80=94=20manifest=20parsing,=20cow-api,=20provider-pool?= =?UTF-8?q?,=20supervisor=20(BLEU-821)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/nexum-engine/src/host/cow_orderbook.rs | 58 ++++++++++++ crates/nexum-engine/src/host/provider_pool.rs | 29 ++++++ crates/nexum-engine/src/manifest.rs | 92 +++++++++++++++++++ crates/nexum-engine/src/supervisor.rs | 39 ++++++++ 4 files changed, 218 insertions(+) diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index 227efd1..61375fd 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -235,6 +235,64 @@ mod tests { assert!(matches!(err, CowApiError::BadMethod(_))); } + #[tokio::test] + async fn request_post_with_body_is_forwarded() { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v1/quote")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"quote":"ok"}"#)) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request( + Chain::Mainnet.id(), + "POST", + "/api/v1/quote", + Some(r#"{"sellToken":"0x01"}"#), + ) + .await + .expect("post with body succeeds"); + assert_eq!(body, r#"{"quote":"ok"}"#); + } + + #[tokio::test] + async fn request_4xx_response_is_returned_verbatim() { + // The host must NOT surface a 4xx as an error — the module + // needs the structured JSON body to decode `OrderPostError`. + let mock = MockServer::start().await; + let error_body = r#"{"errorType":"InsufficientFee","description":"fee too low"}"#; + Mock::given(method("POST")) + .and(path("/api/v1/orders")) + .respond_with( + ResponseTemplate::new(400).set_body_string(error_body), + ) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request( + Chain::Mainnet.id(), + "POST", + "/api/v1/orders", + Some(r#"{"test":true}"#), + ) + .await + .expect("4xx body is returned, not an Err"); + assert_eq!(body, error_body); + } + + #[tokio::test] + async fn request_rejects_unknown_chain() { + let pool = OrderBookPool::default(); + let err = pool.request(99_999, "GET", "/x", None).await.unwrap_err(); + assert!(matches!(err, CowApiError::UnknownChain(99_999))); + } + #[tokio::test] async fn submit_order_propagates_orderbook_response() { let mock = MockServer::start().await; diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index adf589f..e6c3e42 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -198,4 +198,33 @@ mod tests { .unwrap_err(); assert!(matches!(err, ProviderError::UnknownChain(1))); } + + #[tokio::test] + async fn empty_pool_rejects_block_subscribe() { + let pool = ProviderPool::empty(); + // Can't use .unwrap_err() because BlockStream doesn't impl Debug. + assert!(matches!( + pool.subscribe_blocks(1).await, + Err(ProviderError::UnknownChain(1)) + )); + } + + #[tokio::test] + async fn empty_pool_rejects_log_subscribe() { + let pool = ProviderPool::empty(); + let filter = alloy_rpc_types_eth::Filter::new(); + assert!(matches!( + pool.subscribe_logs(1, filter).await, + Err(ProviderError::UnknownChain(1)) + )); + } + + #[tokio::test] + async fn invalid_params_json_is_rejected_before_network() { + // RawValue::from_string rejects non-JSON; verify the parse layer + // we rely on before forwarding to alloy. + let bad = "not json at all {{{"; + let result = RawValue::from_string(bad.to_owned()); + assert!(result.is_err(), "invalid JSON should fail RawValue parse"); + } } diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs index d1ce6bc..8eacffb 100644 --- a/crates/nexum-engine/src/manifest.rs +++ b/crates/nexum-engine/src/manifest.rs @@ -149,6 +149,7 @@ impl std::fmt::Display for ParseError { impl std::error::Error for ParseError {} /// Loaded + validated manifest, plus its source path for diagnostics. +#[derive(Debug)] pub struct LoadedManifest { pub manifest: Manifest, /// Hosts to allow for `http::fetch`. Each entry is either an exact @@ -472,4 +473,95 @@ mod tests { let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); } + + // ── manifest parsing ────────────────────────────────────────────────── + + #[test] + fn load_parses_block_and_log_subscriptions() { + let toml = r#" +[module] +name = "twap-monitor" + +[capabilities] +required = ["chain", "local-store"] + +[[subscription]] +kind = "block" +chain_id = 1 + +[[subscription]] +kind = "log" +chain_id = 1 +address = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110" +event_signature = "0x00000000000000000000000000000000000000000000000000000000deadbeef" +"#; + let manifest: Manifest = toml::from_str(toml).expect("parse"); + assert_eq!(manifest.module.name, "twap-monitor"); + assert_eq!(manifest.subscriptions.len(), 2); + assert!(matches!( + &manifest.subscriptions[0], + Subscription::Block { chain_id: 1 } + )); + if let Subscription::Log { chain_id, address, .. } = &manifest.subscriptions[1] { + assert_eq!(*chain_id, 1); + assert!(address.is_some()); + } else { + panic!("expected Log subscription"); + } + } + + #[test] + fn load_parses_cron_subscription() { + let toml = r#" +[module] +name = "scheduler" + +[[subscription]] +kind = "cron" +schedule = "*/5 * * * *" +"#; + let manifest: Manifest = toml::from_str(toml).expect("parse"); + assert!(matches!( + &manifest.subscriptions[0], + Subscription::Cron { .. } + )); + } + + #[test] + fn load_rejects_unknown_capability() { + let toml = r#" +[module] +name = "bad" + +[capabilities] +required = ["chain", "not-a-real-cap"] +"#; + // Write to a temp file so load() can read it. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("module.toml"); + std::fs::write(&path, toml).unwrap(); + let err = load(&path).unwrap_err(); + assert!(matches!(err, ParseError::UnknownCapability(ref name) if name == "not-a-real-cap")); + } + + #[test] + fn load_parses_config_table() { + let toml = r#" +[module] +name = "example" + +[config] +chain_id = 1 +label = "mainnet" +enabled = true +"#; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("module.toml"); + std::fs::write(&path, toml).unwrap(); + let loaded = load(&path).unwrap(); + let config: std::collections::HashMap<_, _> = loaded.config.into_iter().collect(); + assert_eq!(config.get("chain_id").map(String::as_str), Some("1")); + assert_eq!(config.get("label").map(String::as_str), Some("mainnet")); + assert_eq!(config.get("enabled").map(String::as_str), Some("true")); + } } diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 9bb4cb3..0067e1b 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -441,4 +441,43 @@ mod tests { assert!(sup.log_subscriptions().is_empty()); assert_eq!(sup.module_count(), 0); } + + // ── build_alloy_filter ──────────────────────────────────────────────── + + #[test] + fn alloy_filter_with_address_and_topic() { + let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; + let topic = "0x237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c00"; + let filter = build_alloy_filter(Some(addr), Some(topic)).unwrap(); + // Check address is set (alloy Filter doesn't expose a simple getter, + // but we can verify the filter serialises the address field). + let serialised = serde_json::to_value(&filter).unwrap(); + let addr_field = serialised.get("address").unwrap().to_string().to_lowercase(); + assert!(addr_field.contains(&addr.to_lowercase()[2..])); // strip 0x + } + + #[test] + fn alloy_filter_no_address_no_topic() { + let filter = build_alloy_filter(None, None).unwrap(); + let serialised = serde_json::to_value(&filter).unwrap(); + // Address and topics should be absent or null. + assert!( + serialised.get("address").is_none() + || serialised["address"].is_null() + || serialised["address"] == serde_json::json!([]) + ); + } + + #[test] + fn alloy_filter_rejects_bad_address() { + let err = build_alloy_filter(Some("not-an-address"), None); + assert!(err.is_err()); + } + + #[test] + fn alloy_filter_rejects_bad_topic() { + let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; + let err = build_alloy_filter(Some(addr), Some("not-a-topic")); + assert!(err.is_err()); + } } From cf565296eb90f0d63df743c1eb3a9cf4c698059f Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 9 Jun 2026 22:07:11 -0300 Subject: [PATCH 11/29] test: E2E supervisor tests + fix wit_import_to_cap to skip type-only interfaces (BLEU-819) --- crates/nexum-engine/src/manifest.rs | 28 +++-- crates/nexum-engine/src/supervisor.rs | 150 ++++++++++++++++++++++++++ justfile | 4 + 3 files changed, 172 insertions(+), 10 deletions(-) diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs index 8eacffb..9c885ed 100644 --- a/crates/nexum-engine/src/manifest.rs +++ b/crates/nexum-engine/src/manifest.rs @@ -226,21 +226,29 @@ pub fn enforce_capabilities<'a>( } /// Map a WIT import name to a capability name, or `None` for non-capability -/// imports (wasi:*, wasi:io, wasi:cli, etc.). +/// imports. +/// +/// Returns `Some` only for functional interfaces that appear in +/// `KNOWN_CAPABILITIES`. Type-only packages (e.g. `nexum:host/types`) and +/// WASI system interfaces are treated as non-capability and ignored. /// /// Examples: -/// - `"nexum:host/chain@0.2.0"` → `Some("chain")` -/// - `"shepherd:cow/cow-api@0.2.0"` → `Some("cow-api")` -/// - `"wasi:io/streams@0.2.0"` → `None` +/// - `"nexum:host/chain@0.2.0"` → `Some("chain")` +/// - `"shepherd:cow/cow-api@0.2.0"` → `Some("cow-api")` +/// - `"nexum:host/types@0.2.0"` → `None` (type-only, not a capability) +/// - `"wasi:io/streams@0.2.0"` → `None` fn wit_import_to_cap(import_name: &str) -> Option<&str> { let without_version = import_name.split('@').next().unwrap_or(import_name); - if let Some(iface) = without_version.strip_prefix("nexum:host/") { - Some(iface) - } else if let Some(iface) = without_version.strip_prefix("shepherd:cow/") { - Some(iface) + let iface = if let Some(i) = without_version.strip_prefix("nexum:host/") { + i + } else if let Some(i) = without_version.strip_prefix("shepherd:cow/") { + i } else { - None - } + return None; + }; + // Only return Some for functional capabilities. Type-only packages + // (like nexum:host/types) are shared data definitions, not capabilities. + if KNOWN_CAPABILITIES.contains(&iface) { Some(iface) } else { None } } /// Read `module.toml` from `path`, parse, validate, and emit a deprecation diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 0067e1b..c4f9f4e 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -383,6 +383,7 @@ impl Supervisor { } /// Count of modules currently alive (not dead due to traps). + #[cfg_attr(not(test), allow(dead_code))] pub fn alive_count(&self) -> usize { self.modules.iter().filter(|m| m.alive).count() } @@ -430,6 +431,8 @@ fn build_alloy_filter( #[cfg(test)] mod tests { + use std::path::{Path, PathBuf}; + use super::*; #[test] @@ -442,6 +445,153 @@ mod tests { assert_eq!(sup.module_count(), 0); } + // ── E2E helpers ─────────────────────────────────────────────────────── + + /// Path to the pre-built example WASM component. Tests that need it + /// call `example_wasm_or_skip()` which skips gracefully if absent. + fn example_wasm() -> PathBuf { + // CARGO_MANIFEST_DIR → crates/nexum-engine + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("target/wasm32-wasip2/release/example.wasm") + } + + fn example_module_toml() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("modules/example/module.toml") + } + + /// Returns `None` and prints a skip message if the fixture isn't built. + fn example_wasm_or_skip() -> Option { + let p = example_wasm(); + if p.exists() { + Some(p) + } else { + eprintln!( + "SKIP: {} not found — run `just build-module` to enable E2E tests", + p.display() + ); + None + } + } + + fn make_wasmtime_engine() -> wasmtime::Engine { + let mut config = wasmtime::Config::new(); + config.wasm_component_model(true); + config.consume_fuel(true); + wasmtime::Engine::new(&config).expect("wasmtime engine") + } + + fn make_linker(engine: &wasmtime::Engine) -> Linker { + let mut linker = Linker::::new(engine); + crate::Shepherd::add_to_linker::>( + &mut linker, + |s| s, + ) + .expect("add_to_linker"); + wasmtime_wasi::p2::add_to_linker_async(&mut linker).expect("add_wasi"); + linker + } + + fn temp_local_store() -> crate::host::local_store_redb::LocalStore { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("ls.redb"); + // Leak the dir so the file stays alive for the duration of the test. + let _ = std::mem::ManuallyDrop::new(dir); + crate::host::local_store_redb::LocalStore::open(path).expect("local store") + } + + // ── E2E tests ───────────────────────────────────────────────────────── + + /// Boot supervisor with the example module; verify it starts alive. + #[tokio::test] + async fn e2e_supervisor_boots_example_module() { + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let local_store = temp_local_store(); + + let supervisor = Supervisor::boot_single( + &engine, + &linker, + &wasm, + Some(example_module_toml()).as_deref(), + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot_single"); + + assert_eq!(supervisor.module_count(), 1); + assert_eq!(supervisor.alive_count(), 1); + } + + /// Boot with a manifest that subscribes to block events; dispatch one + /// block event and verify the module was invoked and stayed alive. + #[tokio::test] + async fn e2e_block_subscription_dispatched() { + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let manifest = dir.path().join("module.toml"); + std::fs::write( + &manifest, + r#" +[module] +name = "example" + +[capabilities] +required = ["logging"] + +[[subscription]] +kind = "block" +chain_id = 1 +"#, + ) + .unwrap(); + + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let local_store = temp_local_store(); + + let mut supervisor = Supervisor::boot_single( + &engine, + &linker, + &wasm, + Some(&manifest), + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot_single"); + + let block = crate::nexum::host::types::Block { + chain_id: 1, + number: 19_000_000, + hash: vec![0xab; 32], + timestamp: 1_700_000_000_000, + }; + let dispatched = supervisor.dispatch_block(block).await; + assert_eq!(dispatched, 1, "one module subscribed to chain 1 blocks"); + assert_eq!(supervisor.alive_count(), 1, "module must remain alive"); + } + // ── build_alloy_filter ──────────────────────────────────────────────── #[test] diff --git a/justfile b/justfile index e3ebd15..4230a80 100644 --- a/justfile +++ b/justfile @@ -19,6 +19,10 @@ run: build-module build-engine test: cargo test -p nexum-engine +# Build module + engine, then run E2E integration tests +test-e2e: build-module build-engine + cargo test -p nexum-engine supervisor::tests::e2e + # Check the entire workspace check: cargo check --target wasm32-wasip2 -p example From 85f059b52e5c86e158f86d8663afdd49bbb4dc17 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 9 Jun 2026 22:26:54 -0300 Subject: [PATCH 12/29] style: apply rust-idiomatic rules (em-dashes, #[from] Orderbook, unused_crate_dependencies, drop redundant map_err) --- crates/nexum-engine/src/engine_config.rs | 8 ++--- crates/nexum-engine/src/host/cow_orderbook.rs | 19 +++++----- .../nexum-engine/src/host/local_store_redb.rs | 8 ++--- crates/nexum-engine/src/host/provider_pool.rs | 8 ++--- crates/nexum-engine/src/main.rs | 36 ++++++++++--------- crates/nexum-engine/src/manifest.rs | 14 ++++---- crates/nexum-engine/src/supervisor.rs | 17 +++++---- 7 files changed, 54 insertions(+), 56 deletions(-) diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs index 1246850..9637981 100644 --- a/crates/nexum-engine/src/engine_config.rs +++ b/crates/nexum-engine/src/engine_config.rs @@ -1,7 +1,7 @@ //! Engine-side runtime configuration. //! //! Distinct from `module.toml` (module manifest): this file describes -//! the *engine*'s I/O wiring — chain RPC endpoints and the on-disk +//! the *engine*'s I/O wiring - chain RPC endpoints and the on-disk //! location of the `local-store` database. Both are required for the //! 0.2 reference engine to do anything other than print stubs. //! @@ -10,7 +10,7 @@ //! 1. `--engine-config ` CLI flag (future), or third positional //! argument today; //! 2. `engine.toml` in the current working directory; -//! 3. defaults — no chains configured, `state_dir = ./data`. +//! 3. defaults - no chains configured, `state_dir = ./data`. //! //! A missing config is OK for the example module (it only logs); for //! the cow-api / chain backends it surfaces as `HostError { @@ -34,7 +34,7 @@ pub struct EngineConfig { pub chains: BTreeMap, /// Modules the supervisor should boot. Each entry resolves a /// `(component.wasm, module.toml)` pair on the local filesystem - /// for 0.2 — content-addressed resolution (Swarm / OCI / + /// for 0.2 - content-addressed resolution (Swarm / OCI / /// `[[content.sources]]`) lands in 0.3 per /// `docs/03-module-discovery.md`. #[serde(default)] @@ -101,7 +101,7 @@ pub fn load_or_default(path: Option<&Path>) -> anyhow::Result { if !path.exists() { warn!( path = %path.display(), - "engine.toml not found — running with defaults (no chain RPC endpoints; \ + "engine.toml not found - running with defaults (no chain RPC endpoints; \ chain::request and cow_api::submit_order will return Unsupported)" ); return Ok(EngineConfig::default()); diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index 61375fd..e95df9c 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -2,11 +2,11 @@ //! //! Two responsibilities: //! -//! 1. `request` — generic REST passthrough. Module gives the HTTP +//! 1. `request` - generic REST passthrough. Module gives the HTTP //! method, path (relative to the chain's orderbook base URL), and //! optional JSON body. We dispatch via `reqwest`, return the //! response body verbatim. -//! 2. `submit_order` — typed submission. Module gives a JSON-encoded +//! 2. `submit_order` - typed submission. Module gives a JSON-encoded //! `cowprotocol::OrderCreation`; we parse, dispatch via //! `cowprotocol::OrderBookApi::post_order`, return the assigned //! `OrderUid` as a `0x`-prefixed hex string. @@ -60,7 +60,7 @@ impl OrderBookPool { } /// REST passthrough. The base URL is whichever URL the pool's - /// `OrderBookApi` client carries — overrides set via + /// `OrderBookApi` client carries - overrides set via /// `OrderBookApi::new_with_base_url` (staging, wiremock) flow /// through here too, which keeps the passthrough and the typed /// `submit_order_json` path aimed at the same orderbook. @@ -99,7 +99,7 @@ impl OrderBookPool { let response = request.send().await.map_err(CowApiError::Network)?; // Surface the orderbook's structured 4xx / 5xx bodies verbatim // so the guest can decode `{"errorType": "...", "description": - // "..."}` — projecting them into HostError here loses the + // "..."}` - projecting them into HostError here loses the // detail the guest needs to recover. let text = response.text().await.map_err(CowApiError::Network)?; Ok(text) @@ -116,10 +116,7 @@ impl OrderBookPool { ) -> Result { let creation: OrderCreation = serde_json::from_slice(body).map_err(CowApiError::Decode)?; let api = self.get(chain_id)?; - let uid = api - .post_order(&creation) - .await - .map_err(|e| CowApiError::Orderbook(e.to_string()))?; + let uid = api.post_order(&creation).await?; Ok(uid) } } @@ -136,8 +133,8 @@ pub enum CowApiError { Network(#[from] reqwest::Error), #[error("decode OrderCreation JSON: {0}")] Decode(#[from] serde_json::Error), - #[error("orderbook rejected: {0}")] - Orderbook(String), + #[error("orderbook: {0}")] + Orderbook(#[from] cowprotocol::Error), } #[cfg(test)] @@ -260,7 +257,7 @@ mod tests { #[tokio::test] async fn request_4xx_response_is_returned_verbatim() { - // The host must NOT surface a 4xx as an error — the module + // The host must NOT surface a 4xx as an error - the module // needs the structured JSON body to decode `OrderPostError`. let mock = MockServer::start().await; let error_body = r#"{"errorType":"InsufficientFee","description":"fee too low"}"#; diff --git a/crates/nexum-engine/src/host/local_store_redb.rs b/crates/nexum-engine/src/host/local_store_redb.rs index 46d980e..4822e3e 100644 --- a/crates/nexum-engine/src/host/local_store_redb.rs +++ b/crates/nexum-engine/src/host/local_store_redb.rs @@ -8,9 +8,9 @@ //! The 32-byte hash prefix has two properties that the old //! `[len:u8][name][key]` scheme lacked: //! -//! - **Fixed width** — no length field to forge; a module cannot craft a +//! - **Fixed width** - no length field to forge; a module cannot craft a //! key that bleeds into another module's prefix range. -//! - **ENS-compatible** — keccak256 is the same hash used by ENS node +//! - **ENS-compatible** - keccak256 is the same hash used by ENS node //! derivation, so module identities can be derived from ENS names //! without extra hashing in the future (ADR-0003). @@ -75,7 +75,7 @@ impl LocalStore { Ok(()) } - /// Delete. Idempotent — deleting a missing key is a no-op. + /// Delete. Idempotent - deleting a missing key is a no-op. pub fn delete(&self, namespace: &str, key: &str) -> Result<(), StorageError> { let full = build_key(namespace, key)?; let txn = self.db.begin_write().map_err(StorageError::Txn)?; @@ -90,7 +90,7 @@ impl LocalStore { } /// Enumerate keys in `namespace` whose raw key (post-prefix) starts - /// with `prefix`. Returns only the module-visible key strings — the + /// with `prefix`. Returns only the module-visible key strings - the /// host strips the namespace prefix. pub fn list_keys(&self, namespace: &str, prefix: &str) -> Result, StorageError> { let ns_prefix = namespace_prefix(namespace)?; diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index e6c3e42..4928982 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -3,13 +3,13 @@ //! Per-chain alloy provider, opened from the engine config at boot. //! `request` is a raw JSON-RPC dispatch: the host hands `(method, //! params)` straight to alloy's transport and returns the result body -//! verbatim. No method allowlist, no re-encoding of params — the +//! verbatim. No method allowlist, no re-encoding of params - the //! contract is "give us a JSON-RPC pair, we'll return what the node //! returns". //! //! Transports: -//! - `ws://` / `wss://` — `WsConnect`; required for `eth_subscribe`. -//! - `http://` / `https://` — alloy's HTTP transport; request/response only. +//! - `ws://` / `wss://` - `WsConnect`; required for `eth_subscribe`. +//! - `http://` / `https://` - alloy's HTTP transport; request/response only. use std::collections::BTreeMap; use std::pin::Pin; @@ -66,7 +66,7 @@ impl ProviderPool { }) } - /// Empty pool — used by tests and as a default when no + /// Empty pool - used by tests and as a default when no /// `engine.toml` is found. Every `request` call returns /// `UnknownChain`. #[cfg_attr(not(test), allow(dead_code))] diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 37d892e..3b38cde 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -1,3 +1,5 @@ +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + mod engine_config; mod host; mod manifest; @@ -52,7 +54,7 @@ struct Cli { } // Both packages are listed explicitly so wit-parser can resolve the -// cross-package reference natively — no vendored deps/ tree needed. +// cross-package reference natively - no vendored deps/ tree needed. // World name is fully qualified. wasmtime::component::bindgen!({ path: ["../../wit/nexum-host", "../../wit/shepherd-cow"], @@ -86,11 +88,11 @@ struct HostState { /// Namespace for the running module's `local-store` rows. Set from /// `manifest.module.name` at instantiation. module_namespace: String, - /// `cow-api` backend — per-chain `OrderBookApi` clients + reqwest. + /// `cow-api` backend - per-chain `OrderBookApi` clients + reqwest. cow: host::cow_orderbook::OrderBookPool, - /// `chain` backend — per-chain alloy `DynProvider` pool. + /// `chain` backend - per-chain alloy `DynProvider` pool. chain: host::provider_pool::ProviderPool, - /// `local-store` backend — redb file with host-side namespacing. + /// `local-store` backend - redb file with host-side namespacing. store: host::local_store_redb::LocalStore, } @@ -189,11 +191,11 @@ impl shepherd::cow::cow_api::Host for HostState { message: format!("invalid OrderCreation JSON: {err}"), data: None, }), - Err(host::cow_orderbook::CowApiError::Orderbook(msg)) => Err(HostError { + Err(host::cow_orderbook::CowApiError::Orderbook(err)) => Err(HostError { domain: "cow-api".into(), kind: HostErrorKind::Denied, code: 0, - message: msg, + message: err.to_string(), data: None, }), Err(err) => Err(internal_error("cow-api", err.to_string())), @@ -268,7 +270,7 @@ impl nexum::host::chain::Host for HostState { impl nexum::host::identity::Host for HostState { async fn accounts(&mut self) -> Result>, HostError> { - // No keystore wired yet — return an empty roster so guests can + // No keystore wired yet - return an empty roster so guests can // probe-then-skip without erroring. Real keystore lands in 0.3. Ok(vec![]) } @@ -367,7 +369,7 @@ impl nexum::host::messaging::Host for HostState { _end_time: Option, _limit: Option, ) -> Result, HostError> { - // Empty result — same posture as `identity::accounts`. + // Empty result - same posture as `identity::accounts`. Ok(vec![]) } } @@ -405,7 +407,7 @@ impl nexum::host::random::Host for HostState { let mut buf = vec![0u8; len as usize]; // getrandom 0.4: fill() returns Result<(), Error>. CSPRNG failures // are exceptionally rare on supported platforms; on failure we - // return zero-filled bytes — guests that need a strong-failure + // return zero-filled bytes - guests that need a strong-failure // signal should use identity or chain primitives instead. let _ = getrandom::fill(&mut buf); buf @@ -419,7 +421,7 @@ impl nexum::host::http::Host for HostState { ) -> Result { // Manifest allowlist enforcement runs before any I/O. Hosts that // never link a manifest leave `http_allowlist` empty, which denies - // every request — matching the "no implicit network" stance. + // every request - matching the "no implicit network" stance. let host = match manifest::extract_host(&req.url) { Some(h) => h, None => { @@ -501,7 +503,7 @@ async fn main() -> anyhow::Result<()> { .await .context("open chain providers")?; - // wasmtime engine + linker — one of each, shared across modules. + // wasmtime engine + linker - one of each, shared across modules. let mut config = wasmtime::Config::new(); config.wasm_component_model(true); config.consume_fuel(true); @@ -514,7 +516,7 @@ async fn main() -> anyhow::Result<()> { )?; wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; - // Boot supervisor — `engine.toml.[[modules]]` first, CLI + // Boot supervisor - `engine.toml.[[modules]]` first, CLI // positional second. let mut supervisor = if let Some(wasm) = cli.wasm.as_deref() { if !engine_cfg.modules.is_empty() { @@ -542,7 +544,7 @@ async fn main() -> anyhow::Result<()> { .await? } else { anyhow::bail!( - "no modules to run — either pass a positional or declare \ + "no modules to run - either pass a positional or declare \ [[modules]] entries in engine.toml" ); }; @@ -559,7 +561,7 @@ async fn main() -> anyhow::Result<()> { let log_subs = supervisor.log_subscriptions(); if block_chains.is_empty() && log_subs.is_empty() { - info!("no [[subscription]] entries — engine has nothing to run; exiting"); + info!("no [[subscription]] entries - engine has nothing to run; exiting"); return Ok(()); } @@ -569,7 +571,7 @@ async fn main() -> anyhow::Result<()> { let shutdown = async { match wait_for_shutdown_signal().await { Ok(name) => info!(signal = %name, "shutdown signal received"), - Err(err) => warn!(error = %err, "signal handler failed — using ctrl-c"), + Err(err) => warn!(error = %err, "signal handler failed - using ctrl-c"), } }; @@ -679,14 +681,14 @@ async fn run_event_loop( }; supervisor.dispatch_block(block).await; } - Some(Err(err)) => warn!(error = %err, "block stream error — continuing"), + Some(Err(err)) => warn!(error = %err, "block stream error - continuing"), None => {} }, next = logs.next() => match next { Some(Ok((module, chain_id, log))) => { supervisor.dispatch_log(&module, chain_id, log).await; } - Some(Err(err)) => warn!(error = %err, "log stream error — continuing"), + Some(Err(err)) => warn!(error = %err, "log stream error - continuing"), None => {} }, } diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs index 9c885ed..c659d9c 100644 --- a/crates/nexum-engine/src/manifest.rs +++ b/crates/nexum-engine/src/manifest.rs @@ -62,7 +62,7 @@ pub struct Manifest { #[derive(Debug, Deserialize, Clone)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum Subscription { - /// New-block events. Fan-out is shared per chain — the + /// New-block events. Fan-out is shared per chain - the /// supervisor opens one subscription per chain id and routes to /// every module that asked for blocks on that chain. Block { @@ -70,7 +70,7 @@ pub enum Subscription { chain_id: u64, }, /// Log events matching `address` + topic-0. Fan-out is - /// per-module — the supervisor opens one subscription per + /// per-module - the supervisor opens one subscription per /// `[[subscription]]` entry and tags emitted events with the /// owning module. Log { @@ -80,7 +80,7 @@ pub enum Subscription { #[serde(default)] address: Option, /// Topic-0 of the event the module wants to consume. `0x`- - /// prefixed 32-byte hex. Optional — when absent the + /// prefixed 32-byte hex. Optional - when absent the /// subscription matches every event from the address(es). #[serde(default)] event_signature: Option, @@ -194,7 +194,7 @@ impl std::error::Error for CapabilityViolation {} /// a deprecation warning. /// /// `component_imports` should be the iterator returned by -/// `component.component_type().imports(&engine)` — pass the **name** part +/// `component.component_type().imports(&engine)` - pass the **name** part /// (`&str`) of each `(&str, ComponentItem)` tuple. pub fn enforce_capabilities<'a>( loaded: &LoadedManifest, @@ -260,7 +260,7 @@ pub fn load(path: &Path) -> Result { let caps = manifest.capabilities.as_ref(); if caps.is_none() { eprintln!( - "[deprecation] no [capabilities] section in module.toml — \ + "[deprecation] no [capabilities] section in module.toml - \ defaulting to all-required (0.1 behaviour). This default \ will be removed in 0.3; add an explicit [capabilities] block." ); @@ -313,7 +313,7 @@ pub fn load(path: &Path) -> Result { /// Emits the same deprecation warning as a missing-section manifest. pub fn fallback_manifest() -> LoadedManifest { eprintln!( - "[deprecation] no module.toml found — defaulting to all-required \ + "[deprecation] no module.toml found - defaulting to all-required \ (0.1 behaviour). This default will be removed in 0.3; ship a \ module.toml alongside your component." ); @@ -340,7 +340,7 @@ pub fn host_allowed(host: &str, allowlist: &[String]) -> bool { } /// Extract the host component from a URL. Returns `None` for non-http(s) -/// schemes or malformed input. Intentionally simple — adds no `url` +/// schemes or malformed input. Intentionally simple - adds no `url` /// crate dependency. pub fn extract_host(url: &str) -> Option<&str> { let after_scheme = url diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index c4f9f4e..4b7823d 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -130,7 +130,7 @@ impl Supervisor { _ => { warn!( component = %entry.path.display(), - "no module.toml — falling back to anonymous module" + "no module.toml - falling back to anonymous module" ); manifest::fallback_manifest() } @@ -147,7 +147,6 @@ impl Supervisor { &loaded_manifest, component.component_type().imports(engine).map(|(n, _)| n), ) - .map_err(|e| Error::msg(e.to_string())) .with_context(|| format!("capability violation in {}", entry.path.display()))?; let wasi = WasiCtxBuilder::new().inherit_stdio().build(); let module_namespace = if loaded_manifest.manifest.module.name.is_empty() { @@ -264,7 +263,7 @@ impl Supervisor { module = %module.name, chain_id, error = %err, - "invalid log subscription — skipping", + "invalid log subscription - skipping", ), } } @@ -296,7 +295,7 @@ impl Supervisor { } // Refuel before each invocation so each event gets a fresh budget. if let Err(e) = module.store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT) { - error!(module = %module.name, error = %e, "set_fuel failed — skipping"); + error!(module = %module.name, error = %e, "set_fuel failed - skipping"); continue; } match module @@ -318,7 +317,7 @@ impl Supervisor { module = %module.name, chain_id, error = %trap, - "on-event trapped — module marked dead, removed from dispatch", + "on-event trapped - module marked dead, removed from dispatch", ); module.alive = false; } @@ -340,7 +339,7 @@ impl Supervisor { let target = match self.modules.iter_mut().find(|m| m.name == module_name) { Some(m) => m, None => { - warn!(module = %module_name, "no such module — dropping log"); + warn!(module = %module_name, "no such module - dropping log"); return false; } }; @@ -348,7 +347,7 @@ impl Supervisor { return false; } if let Err(e) = target.store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT) { - error!(module = %module_name, error = %e, "set_fuel failed — skipping"); + error!(module = %module_name, error = %e, "set_fuel failed - skipping"); return false; } let event = crate::nexum::host::types::Event::Logs(vec![project_log(chain_id, &log)]); @@ -374,7 +373,7 @@ impl Supervisor { module = %module_name, chain_id, error = %trap, - "on-event trapped — module marked dead, removed from dispatch", + "on-event trapped - module marked dead, removed from dispatch", ); target.alive = false; false @@ -475,7 +474,7 @@ mod tests { Some(p) } else { eprintln!( - "SKIP: {} not found — run `just build-module` to enable E2E tests", + "SKIP: {} not found - run `just build-module` to enable E2E tests", p.display() ); None From 6f991f11d98a28aed5f35c63e9cee997db9aa05d Mon Sep 17 00:00:00 2001 From: brunota20 Date: Fri, 12 Jun 2026 10:06:10 -0300 Subject: [PATCH 13/29] review: apply lgahdl feedback on PR #9 (+ rebase PR #8 fixes) PR #9 specific: - main: warn + return when block/log streams end (WebSocket dropped) - supervisor: simplify dispatch_block by extracting chain_id before move - supervisor: temp_local_store returns (TempDir, LocalStore) instead of leaking - README: correct engine.toml chain syntax to [chains.] with rpc_url Rebased from PR #8: - local_store_redb: table.range() instead of iter() for O(matching) keys - provider_pool: dedupe method clone on the success path - main: hex_encode writes into the pre-allocated buffer - cow_orderbook: drop blank line nit - manifest: collapse nested if and use ? operator (clippy) - alloy_rpc_client / alloy_transport(_ws) imports as _ to satisfy unused_crate_dependencies. --- README.md | 5 ++- crates/nexum-engine/src/host/cow_orderbook.rs | 1 - .../nexum-engine/src/host/local_store_redb.rs | 19 ++++++++--- crates/nexum-engine/src/host/provider_pool.rs | 24 ++++++++------ crates/nexum-engine/src/main.rs | 32 ++++++++++++++++--- crates/nexum-engine/src/manifest.rs | 30 ++++++++--------- crates/nexum-engine/src/supervisor.rs | 20 ++++++------ 7 files changed, 83 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index f6be460..e44e9d4 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,8 @@ nexum-engine --engine-config engine.toml state_dir = "/var/lib/shepherd" log_level = "info" -[[chains]] -id = 1 -url = "wss://mainnet.infura.io/ws/v3/..." +[chains.1] +rpc_url = "wss://mainnet.infura.io/ws/v3/..." [[modules]] path = "modules/twap-monitor/twap-monitor.wasm" diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index e95df9c..bac376d 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -51,7 +51,6 @@ impl Default for OrderBookPool { } impl OrderBookPool { - /// Look up the client for a chain. pub fn get(&self, chain_id: u64) -> Result<&OrderBookApi, CowApiError> { self.clients diff --git a/crates/nexum-engine/src/host/local_store_redb.rs b/crates/nexum-engine/src/host/local_store_redb.rs index 4822e3e..96f02eb 100644 --- a/crates/nexum-engine/src/host/local_store_redb.rs +++ b/crates/nexum-engine/src/host/local_store_redb.rs @@ -20,7 +20,7 @@ use std::path::Path; use std::sync::Arc; use alloy_primitives::keccak256; -use redb::{Database, ReadableTable, TableDefinition}; +use redb::{Database, TableDefinition}; use thiserror::Error; const TABLE: TableDefinition<'static, &[u8], &[u8]> = TableDefinition::new("nexum:local-store"); @@ -98,12 +98,21 @@ impl LocalStore { let txn = self.db.begin_read().map_err(StorageError::Txn)?; let table = txn.open_table(TABLE).map_err(StorageError::Table)?; let mut out = Vec::new(); - for entry in table.iter().map_err(StorageError::Storage)? { + // redb's B-tree iterates keys in sorted order, so a range + // starting at `full_prefix` only touches matching entries (and + // the first key past the prefix range). Breaking on the first + // non-matching key keeps this O(matching entries) instead of + // the O(total DB entries) `table.iter()` would do. + for entry in table + .range(full_prefix.as_slice()..) + .map_err(StorageError::Storage)? + { let (k, _v) = entry.map_err(StorageError::Storage)?; let key_bytes = k.value(); - if key_bytes.starts_with(&full_prefix) - && let Ok(s) = std::str::from_utf8(&key_bytes[ns_prefix.len()..]) - { + if !key_bytes.starts_with(&full_prefix) { + break; + } + if let Ok(s) = std::str::from_utf8(&key_bytes[ns_prefix.len()..]) { out.push(s.to_owned()); } } diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index 4928982..d65e391 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -131,19 +131,23 @@ impl ProviderPool { .ok_or(ProviderError::UnknownChain(chain_id))?; // Pass the params through as a raw JSON value so alloy does // not re-encode them on the way to the node. - let params: Box = RawValue::from_string(params_json.clone()).map_err(|e| { - ProviderError::InvalidParams { + let params: Box = + RawValue::from_string(params_json).map_err(|e| ProviderError::InvalidParams { method: method.clone(), detail: e.to_string(), - } - })?; - let result: Box = provider - .raw_request(method.clone().into(), params) - .await - .map_err(|e| ProviderError::Rpc { - method, - detail: e.to_string(), })?; + // `raw_request` consumes the method name; clone once for the + // error branch so the success path moves the original string + // straight into alloy without an extra allocation. + let method_for_err = method.clone(); + let result: Box = + provider + .raw_request(method.into(), params) + .await + .map_err(|e| ProviderError::Rpc { + method: method_for_err, + detail: e.to_string(), + })?; Ok(result.get().to_owned()) } } diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 3b38cde..d05b4d6 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -1,5 +1,13 @@ #![cfg_attr(not(test), warn(unused_crate_dependencies))] +// alloy split its API across multiple crates; we depend on the +// transports directly so cargo resolves the right feature set, but +// the runtime code only names them through the `alloy_provider` +// re-exports. Silence `unused_crate_dependencies` with `as _`. +use alloy_rpc_client as _; +use alloy_transport as _; +use alloy_transport_ws as _; + mod engine_config; mod host; mod manifest; @@ -457,11 +465,14 @@ impl nexum::host::http::Host for HostState { } /// Lowercase hex encoder. Kept in the engine binary rather than -/// pulling a `hex` crate just for one call site. +/// pulling a `hex` crate just for one call site. Writes into the +/// pre-allocated buffer to avoid the per-byte `String` allocation +/// `format!("{b:02x}")` would do. fn hex_encode(bytes: &[u8]) -> String { + use std::fmt::Write as _; let mut s = String::with_capacity(bytes.len() * 2); for b in bytes { - s.push_str(&format!("{b:02x}")); + write!(s, "{b:02x}").expect("writing to String never fails"); } s } @@ -682,14 +693,27 @@ async fn run_event_loop( supervisor.dispatch_block(block).await; } Some(Err(err)) => warn!(error = %err, "block stream error - continuing"), - None => {} + None => { + // alloy ends the stream with None when the + // WebSocket drops. Without this branch the loop + // keeps polling a dead stream and the operator + // sees no events with no indication anything is + // wrong. Bail out so the supervisor (or whatever + // wraps the engine) restarts us; a reconnect- + // with-backoff is the 0.3 fix. + warn!("block stream ended (WebSocket dropped?) - shutting down for restart"); + return; + } }, next = logs.next() => match next { Some(Ok((module, chain_id, log))) => { supervisor.dispatch_log(&module, chain_id, log).await; } Some(Err(err)) => warn!(error = %err, "log stream error - continuing"), - None => {} + None => { + warn!("log stream ended (WebSocket dropped?) - shutting down for restart"); + return; + } }, } } diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs index c659d9c..a73d177 100644 --- a/crates/nexum-engine/src/manifest.rs +++ b/crates/nexum-engine/src/manifest.rs @@ -213,13 +213,13 @@ pub fn enforce_capabilities<'a>( .collect(); for import_name in component_imports { - if let Some(cap) = wit_import_to_cap(import_name) { - if !declared.contains(cap) { - return Err(CapabilityViolation { - capability: cap.to_owned(), - wit_import: import_name.to_owned(), - }); - } + if let Some(cap) = wit_import_to_cap(import_name) + && !declared.contains(cap) + { + return Err(CapabilityViolation { + capability: cap.to_owned(), + wit_import: import_name.to_owned(), + }); } } Ok(()) @@ -239,16 +239,16 @@ pub fn enforce_capabilities<'a>( /// - `"wasi:io/streams@0.2.0"` → `None` fn wit_import_to_cap(import_name: &str) -> Option<&str> { let without_version = import_name.split('@').next().unwrap_or(import_name); - let iface = if let Some(i) = without_version.strip_prefix("nexum:host/") { - i - } else if let Some(i) = without_version.strip_prefix("shepherd:cow/") { - i - } else { - return None; - }; + let iface = without_version + .strip_prefix("nexum:host/") + .or_else(|| without_version.strip_prefix("shepherd:cow/"))?; // Only return Some for functional capabilities. Type-only packages // (like nexum:host/types) are shared data definitions, not capabilities. - if KNOWN_CAPABILITIES.contains(&iface) { Some(iface) } else { None } + if KNOWN_CAPABILITIES.contains(&iface) { + Some(iface) + } else { + None + } } /// Read `module.toml` from `path`, parse, validate, and emit a deprecation diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 4b7823d..631696e 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -276,11 +276,8 @@ impl Supervisor { /// `block.chain_id`. Returns the number of modules invoked. /// Modules that trap are marked dead and excluded from future dispatch. pub async fn dispatch_block(&mut self, block: crate::nexum::host::types::Block) -> usize { + let chain_id = block.chain_id; let event = crate::nexum::host::types::Event::Block(block); - let chain_id = match &event { - crate::nexum::host::types::Event::Block(b) => b.chain_id, - _ => unreachable!(), - }; let mut dispatched = 0; for module in &mut self.modules { if !module.alive { @@ -499,12 +496,15 @@ mod tests { linker } - fn temp_local_store() -> crate::host::local_store_redb::LocalStore { + /// Return `(dir, store)` so the test holds the `TempDir` for the + /// duration of the test scope and cleans it up on drop. Forgetting + /// the dir (the old `ManuallyDrop` approach) leaks it for the + /// entire process lifetime. + fn temp_local_store() -> (tempfile::TempDir, crate::host::local_store_redb::LocalStore) { let dir = tempfile::tempdir().expect("tempdir"); let path = dir.path().join("ls.redb"); - // Leak the dir so the file stays alive for the duration of the test. - let _ = std::mem::ManuallyDrop::new(dir); - crate::host::local_store_redb::LocalStore::open(path).expect("local store") + let store = crate::host::local_store_redb::LocalStore::open(path).expect("local store"); + (dir, store) } // ── E2E tests ───────────────────────────────────────────────────────── @@ -519,7 +519,7 @@ mod tests { let linker = make_linker(&engine); let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); let provider_pool = crate::host::provider_pool::ProviderPool::empty(); - let local_store = temp_local_store(); + let (_dir, local_store) = temp_local_store(); let supervisor = Supervisor::boot_single( &engine, @@ -566,7 +566,7 @@ chain_id = 1 let linker = make_linker(&engine); let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); let provider_pool = crate::host::provider_pool::ProviderPool::empty(); - let local_store = temp_local_store(); + let (_dir, local_store) = temp_local_store(); let mut supervisor = Supervisor::boot_single( &engine, From 6f279089ddbaf604087d5a9b1e63761201ec5bf9 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Sat, 13 Jun 2026 09:29:31 -0300 Subject: [PATCH 14/29] refactor(manifest): split into types/load/capabilities/error submodules Move the manifest.rs monolith into a directory module with four focused submodules (types, load, capabilities, error). Includes the Subscription enum and the four PR #9 tests for subscription parsing. Behaviour unchanged - pure code motion. --- crates/nexum-engine/src/manifest.rs | 575 ------------------ .../nexum-engine/src/manifest/capabilities.rs | 162 +++++ crates/nexum-engine/src/manifest/error.rs | 51 ++ crates/nexum-engine/src/manifest/load.rs | 253 ++++++++ crates/nexum-engine/src/manifest/mod.rs | 40 ++ crates/nexum-engine/src/manifest/types.rs | 123 ++++ 6 files changed, 629 insertions(+), 575 deletions(-) delete mode 100644 crates/nexum-engine/src/manifest.rs create mode 100644 crates/nexum-engine/src/manifest/capabilities.rs create mode 100644 crates/nexum-engine/src/manifest/error.rs create mode 100644 crates/nexum-engine/src/manifest/load.rs create mode 100644 crates/nexum-engine/src/manifest/mod.rs create mode 100644 crates/nexum-engine/src/manifest/types.rs diff --git a/crates/nexum-engine/src/manifest.rs b/crates/nexum-engine/src/manifest.rs deleted file mode 100644 index a73d177..0000000 --- a/crates/nexum-engine/src/manifest.rs +++ /dev/null @@ -1,575 +0,0 @@ -//! `module.toml` parser and capability-enforcement helpers (0.2 scope). -//! -//! 0.2 intentionally ships a slim subset of the manifest spec: -//! -//! - `[capabilities].required` is parsed and validated (names must be in -//! the known capability set; the 0.2 reference engine always provides -//! all of them, so this is a sanity check + future-proofing). -//! - `[capabilities].optional` is parsed and logged; trap-stub fallback -//! for absent optionals is deferred to 0.3. -//! - `[capabilities.http].allow` is parsed and consulted by the `http` -//! host impl before any outbound call. -//! - `[config]` is flattened to `Vec<(String, String)>` and passed to the -//! module's `init`. Typed `config-value` variant is deferred to 0.3. -//! -//! When the manifest file is missing or has no `[capabilities]` section, -//! a deprecation warning is emitted and the engine falls back to 0.1 -//! behaviour (treat every linked capability as required). This fallback -//! will be removed in 0.3. - -use std::collections::HashSet; -use std::path::Path; - -use serde::Deserialize; - -/// Capability names recognised by the 0.2 reference engine. Matches the -/// interfaces the `shepherd` world links into the linker. -pub const KNOWN_CAPABILITIES: &[&str] = &[ - "chain", - "identity", - "local-store", - "remote-store", - "messaging", - "logging", - "clock", - "random", - "http", - // Domain-extension caps (provided by the shepherd world only): - "cow-api", -]; - -#[derive(Debug, Deserialize, Default)] -pub struct Manifest { - #[serde(default)] - pub module: ModuleSection, - #[serde(default)] - pub capabilities: Option, - #[serde(default)] - pub config: toml::Table, - /// Event subscriptions the runtime wires before calling - /// `_init`. See `docs/02-modules-events-packaging.md` for the - /// schema; 0.2 implements `block` and `log` kinds, `cron` is - /// parsed and ignored (deferred to 0.3). - #[serde(default, rename = "subscription")] - pub subscriptions: Vec, -} - -/// One `[[subscription]]` table in `module.toml`. -/// -/// The discriminator is the `kind` field; remaining fields are -/// validated per-kind by the supervisor. Unknown kinds are surfaced -/// at load time so a typo does not silently disable an event source. -#[derive(Debug, Deserialize, Clone)] -#[serde(tag = "kind", rename_all = "lowercase")] -pub enum Subscription { - /// New-block events. Fan-out is shared per chain - the - /// supervisor opens one subscription per chain id and routes to - /// every module that asked for blocks on that chain. - Block { - /// EVM chain id. - chain_id: u64, - }, - /// Log events matching `address` + topic-0. Fan-out is - /// per-module - the supervisor opens one subscription per - /// `[[subscription]]` entry and tags emitted events with the - /// owning module. - Log { - /// EVM chain id. - chain_id: u64, - /// Contract address as `0x`-prefixed 20-byte hex. Optional. - #[serde(default)] - address: Option, - /// Topic-0 of the event the module wants to consume. `0x`- - /// prefixed 32-byte hex. Optional - when absent the - /// subscription matches every event from the address(es). - #[serde(default)] - event_signature: Option, - }, - /// Cron-scheduled tick. 0.2 parses but does not dispatch; the - /// supervisor emits a warning so the operator knows the - /// declaration is currently inert. `schedule` is preserved so a - /// 0.3 dispatcher can pick it up without re-parsing the manifest. - Cron { - /// Standard 5-field cron expression. - #[allow(dead_code)] - schedule: String, - }, -} - -#[derive(Debug, Deserialize, Default)] -#[allow(dead_code)] // version + component parsed for future 0.3 hash-verification. -pub struct ModuleSection { - #[serde(default)] - pub name: String, - #[serde(default)] - pub version: String, - #[serde(default)] - pub component: String, -} - -#[derive(Debug, Deserialize, Default)] -pub struct CapabilitiesSection { - #[serde(default)] - pub required: Vec, - #[serde(default)] - pub optional: Vec, - #[serde(default)] - pub http: Option, -} - -#[derive(Debug, Deserialize, Default)] -pub struct HttpSection { - #[serde(default)] - pub allow: Vec, -} - -/// Errors returned while loading or validating a manifest. -#[derive(Debug)] -pub enum ParseError { - Io(std::io::Error), - Toml(toml::de::Error), - UnknownCapability(String), -} - -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Io(e) => write!(f, "manifest: i/o: {e}"), - Self::Toml(e) => write!(f, "manifest: parse: {e}"), - Self::UnknownCapability(name) => write!( - f, - "manifest: unknown capability {:?} in [capabilities].required (known: {})", - name, - KNOWN_CAPABILITIES.join(", ") - ), - } - } -} - -impl std::error::Error for ParseError {} - -/// Loaded + validated manifest, plus its source path for diagnostics. -#[derive(Debug)] -pub struct LoadedManifest { - pub manifest: Manifest, - /// Hosts to allow for `http::fetch`. Each entry is either an exact - /// hostname or a `*.suffix` wildcard. - pub http_allowlist: Vec, - /// `[config]` flattened to `(key, stringified-value)` pairs ready to - /// hand to a module's `init`. TOML scalars (string, integer, float, - /// boolean) become their text form. Arrays and tables are rendered as - /// their TOML representation. - pub config: Vec<(String, String)>, -} - -/// Error returned when a component's WIT imports exceed its declared capabilities. -#[derive(Debug)] -pub struct CapabilityViolation { - /// Capability name (e.g. `"remote-store"`). - pub capability: String, - /// Full WIT import name as it appeared in the component (e.g. - /// `"nexum:host/remote-store@0.2.0"`). - pub wit_import: String, -} - -impl std::fmt::Display for CapabilityViolation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "component imports `{}` ({}) but it is not listed in \ - [capabilities].required or [capabilities].optional", - self.capability, self.wit_import - ) - } -} - -impl std::error::Error for CapabilityViolation {} - -/// Check that every capability-bearing WIT import of the component is covered -/// by the module's manifest declarations. Call this after loading the -/// component but before instantiation. -/// -/// When `[capabilities]` is absent the manifest is in 0.1-fallback mode and -/// all imports are allowed; the caller is expected to have already emitted -/// a deprecation warning. -/// -/// `component_imports` should be the iterator returned by -/// `component.component_type().imports(&engine)` - pass the **name** part -/// (`&str`) of each `(&str, ComponentItem)` tuple. -pub fn enforce_capabilities<'a>( - loaded: &LoadedManifest, - component_imports: impl Iterator, -) -> Result<(), CapabilityViolation> { - let caps = match loaded.manifest.capabilities.as_ref() { - None => return Ok(()), // 0.1-fallback: no enforcement - Some(c) => c, - }; - - let declared: HashSet<&str> = caps - .required - .iter() - .chain(caps.optional.iter()) - .map(String::as_str) - .collect(); - - for import_name in component_imports { - if let Some(cap) = wit_import_to_cap(import_name) - && !declared.contains(cap) - { - return Err(CapabilityViolation { - capability: cap.to_owned(), - wit_import: import_name.to_owned(), - }); - } - } - Ok(()) -} - -/// Map a WIT import name to a capability name, or `None` for non-capability -/// imports. -/// -/// Returns `Some` only for functional interfaces that appear in -/// `KNOWN_CAPABILITIES`. Type-only packages (e.g. `nexum:host/types`) and -/// WASI system interfaces are treated as non-capability and ignored. -/// -/// Examples: -/// - `"nexum:host/chain@0.2.0"` → `Some("chain")` -/// - `"shepherd:cow/cow-api@0.2.0"` → `Some("cow-api")` -/// - `"nexum:host/types@0.2.0"` → `None` (type-only, not a capability) -/// - `"wasi:io/streams@0.2.0"` → `None` -fn wit_import_to_cap(import_name: &str) -> Option<&str> { - let without_version = import_name.split('@').next().unwrap_or(import_name); - let iface = without_version - .strip_prefix("nexum:host/") - .or_else(|| without_version.strip_prefix("shepherd:cow/"))?; - // Only return Some for functional capabilities. Type-only packages - // (like nexum:host/types) are shared data definitions, not capabilities. - if KNOWN_CAPABILITIES.contains(&iface) { - Some(iface) - } else { - None - } -} - -/// Read `module.toml` from `path`, parse, validate, and emit a deprecation -/// warning if `[capabilities]` is absent (0.1-compat fallback). -pub fn load(path: &Path) -> Result { - let raw = std::fs::read_to_string(path).map_err(ParseError::Io)?; - let manifest: Manifest = toml::from_str(&raw).map_err(ParseError::Toml)?; - - let caps = manifest.capabilities.as_ref(); - if caps.is_none() { - eprintln!( - "[deprecation] no [capabilities] section in module.toml - \ - defaulting to all-required (0.1 behaviour). This default \ - will be removed in 0.3; add an explicit [capabilities] block." - ); - } - - if let Some(c) = caps { - let known: HashSet<&str> = KNOWN_CAPABILITIES.iter().copied().collect(); - for name in c.required.iter().chain(c.optional.iter()) { - if !known.contains(name.as_str()) { - return Err(ParseError::UnknownCapability(name.clone())); - } - } - if !c.required.is_empty() { - eprintln!( - "[manifest] required capabilities: {}", - c.required.join(", ") - ); - } - if !c.optional.is_empty() { - eprintln!( - "[manifest] optional capabilities (advisory in 0.2; trap-stub fallback \ - ships in 0.3): {}", - c.optional.join(", ") - ); - } - } - - let http_allowlist = caps - .and_then(|c| c.http.as_ref()) - .map(|h| h.allow.clone()) - .unwrap_or_default(); - if !http_allowlist.is_empty() { - eprintln!("[manifest] http allowlist: {}", http_allowlist.join(", ")); - } - - let config = manifest - .config - .iter() - .map(|(k, v)| (k.clone(), stringify_toml_value(v))) - .collect(); - - Ok(LoadedManifest { - manifest, - http_allowlist, - config, - }) -} - -/// Synthesise a "0.1 fallback" manifest for when no `module.toml` is found. -/// Emits the same deprecation warning as a missing-section manifest. -pub fn fallback_manifest() -> LoadedManifest { - eprintln!( - "[deprecation] no module.toml found - defaulting to all-required \ - (0.1 behaviour). This default will be removed in 0.3; ship a \ - module.toml alongside your component." - ); - LoadedManifest { - manifest: Manifest::default(), - http_allowlist: Vec::new(), - config: Vec::new(), - } -} - -/// Check whether `host` matches any pattern in the allowlist. Patterns are -/// either exact (`api.example.com`) or `*.suffix` wildcards which match -/// any subdomain of `suffix` (but not `suffix` itself). -pub fn host_allowed(host: &str, allowlist: &[String]) -> bool { - let host = host.to_ascii_lowercase(); - allowlist.iter().any(|pat| { - let pat = pat.to_ascii_lowercase(); - if let Some(suffix) = pat.strip_prefix("*.") { - host.ends_with(&format!(".{suffix}")) - } else { - host == pat - } - }) -} - -/// Extract the host component from a URL. Returns `None` for non-http(s) -/// schemes or malformed input. Intentionally simple - adds no `url` -/// crate dependency. -pub fn extract_host(url: &str) -> Option<&str> { - let after_scheme = url - .strip_prefix("https://") - .or_else(|| url.strip_prefix("http://"))?; - let host_end = after_scheme - .find('/') - .or_else(|| after_scheme.find('?')) - .unwrap_or(after_scheme.len()); - let host = &after_scheme[..host_end]; - // strip optional user-info and port. - let host = host.rsplit('@').next().unwrap_or(host); - let host = host.split(':').next().unwrap_or(host); - if host.is_empty() { None } else { Some(host) } -} - -fn stringify_toml_value(v: &toml::Value) -> String { - match v { - toml::Value::String(s) => s.clone(), - toml::Value::Integer(i) => i.to_string(), - toml::Value::Float(f) => f.to_string(), - toml::Value::Boolean(b) => b.to_string(), - toml::Value::Datetime(d) => d.to_string(), - toml::Value::Array(_) | toml::Value::Table(_) => v.to_string(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn extract_host_handles_common_shapes() { - assert_eq!( - extract_host("https://api.example.com/v1/x"), - Some("api.example.com") - ); - assert_eq!(extract_host("http://example.com"), Some("example.com")); - assert_eq!( - extract_host("https://user:pw@host.example.com:8443/x"), - Some("host.example.com") - ); - assert_eq!(extract_host("https://example.com?q=1"), Some("example.com")); - assert_eq!(extract_host("ftp://example.com"), None); - assert_eq!(extract_host("not a url"), None); - } - - #[test] - fn host_allowed_exact_and_wildcard() { - let allow = vec!["api.cow.fi".to_string(), "*.discord.com".to_string()]; - assert!(host_allowed("api.cow.fi", &allow)); - assert!(!host_allowed("evil.api.cow.fi", &allow)); - assert!(host_allowed("foo.discord.com", &allow)); - assert!(host_allowed("a.b.discord.com", &allow)); - assert!(!host_allowed("discord.com", &allow)); - assert!(!host_allowed("nope.example", &allow)); - } - - // ── capability enforcement ──────────────────────────────────────────── - - #[test] - fn wit_import_to_cap_nexum_host() { - assert_eq!(wit_import_to_cap("nexum:host/chain@0.2.0"), Some("chain")); - assert_eq!( - wit_import_to_cap("nexum:host/local-store@0.2.0"), - Some("local-store") - ); - assert_eq!(wit_import_to_cap("nexum:host/http@0.2.0"), Some("http")); - } - - #[test] - fn wit_import_to_cap_shepherd_cow() { - assert_eq!( - wit_import_to_cap("shepherd:cow/cow-api@0.2.0"), - Some("cow-api") - ); - } - - #[test] - fn wit_import_to_cap_wasi_is_none() { - assert_eq!(wit_import_to_cap("wasi:io/streams@0.2.0"), None); - assert_eq!(wit_import_to_cap("wasi:cli/stdin@0.2.0"), None); - } - - fn manifest_with_caps(required: &[&str], optional: &[&str]) -> LoadedManifest { - LoadedManifest { - manifest: Manifest { - capabilities: Some(CapabilitiesSection { - required: required.iter().map(|s| s.to_string()).collect(), - optional: optional.iter().map(|s| s.to_string()).collect(), - http: None, - }), - ..Default::default() - }, - http_allowlist: vec![], - config: vec![], - } - } - - fn manifest_no_caps() -> LoadedManifest { - LoadedManifest { - manifest: Manifest::default(), - http_allowlist: vec![], - config: vec![], - } - } - - #[test] - fn enforce_passes_when_caps_absent() { - let loaded = manifest_no_caps(); - let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; - assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); - } - - #[test] - fn enforce_passes_when_all_imports_declared() { - let loaded = manifest_with_caps(&["chain", "cow-api"], &["http"]); - let imports = [ - "nexum:host/chain@0.2.0", - "shepherd:cow/cow-api@0.2.0", - "nexum:host/http@0.2.0", - "wasi:io/streams@0.2.0", - ]; - assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); - } - - #[test] - fn enforce_rejects_undeclared_import() { - let loaded = manifest_with_caps(&["chain"], &[]); - let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; - let err = enforce_capabilities(&loaded, imports.into_iter()).unwrap_err(); - assert_eq!(err.capability, "remote-store"); - } - - #[test] - fn enforce_optional_caps_are_also_allowed() { - let loaded = manifest_with_caps(&["chain"], &["remote-store"]); - let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; - assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); - } - - // ── manifest parsing ────────────────────────────────────────────────── - - #[test] - fn load_parses_block_and_log_subscriptions() { - let toml = r#" -[module] -name = "twap-monitor" - -[capabilities] -required = ["chain", "local-store"] - -[[subscription]] -kind = "block" -chain_id = 1 - -[[subscription]] -kind = "log" -chain_id = 1 -address = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110" -event_signature = "0x00000000000000000000000000000000000000000000000000000000deadbeef" -"#; - let manifest: Manifest = toml::from_str(toml).expect("parse"); - assert_eq!(manifest.module.name, "twap-monitor"); - assert_eq!(manifest.subscriptions.len(), 2); - assert!(matches!( - &manifest.subscriptions[0], - Subscription::Block { chain_id: 1 } - )); - if let Subscription::Log { chain_id, address, .. } = &manifest.subscriptions[1] { - assert_eq!(*chain_id, 1); - assert!(address.is_some()); - } else { - panic!("expected Log subscription"); - } - } - - #[test] - fn load_parses_cron_subscription() { - let toml = r#" -[module] -name = "scheduler" - -[[subscription]] -kind = "cron" -schedule = "*/5 * * * *" -"#; - let manifest: Manifest = toml::from_str(toml).expect("parse"); - assert!(matches!( - &manifest.subscriptions[0], - Subscription::Cron { .. } - )); - } - - #[test] - fn load_rejects_unknown_capability() { - let toml = r#" -[module] -name = "bad" - -[capabilities] -required = ["chain", "not-a-real-cap"] -"#; - // Write to a temp file so load() can read it. - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("module.toml"); - std::fs::write(&path, toml).unwrap(); - let err = load(&path).unwrap_err(); - assert!(matches!(err, ParseError::UnknownCapability(ref name) if name == "not-a-real-cap")); - } - - #[test] - fn load_parses_config_table() { - let toml = r#" -[module] -name = "example" - -[config] -chain_id = 1 -label = "mainnet" -enabled = true -"#; - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("module.toml"); - std::fs::write(&path, toml).unwrap(); - let loaded = load(&path).unwrap(); - let config: std::collections::HashMap<_, _> = loaded.config.into_iter().collect(); - assert_eq!(config.get("chain_id").map(String::as_str), Some("1")); - assert_eq!(config.get("label").map(String::as_str), Some("mainnet")); - assert_eq!(config.get("enabled").map(String::as_str), Some("true")); - } -} diff --git a/crates/nexum-engine/src/manifest/capabilities.rs b/crates/nexum-engine/src/manifest/capabilities.rs new file mode 100644 index 0000000..fddb788 --- /dev/null +++ b/crates/nexum-engine/src/manifest/capabilities.rs @@ -0,0 +1,162 @@ +//! Capability enforcement: cross-checks the component's WIT imports +//! against the `[capabilities]` block declared in `module.toml`. + +use std::collections::HashSet; + +use super::error::CapabilityViolation; +use super::types::{KNOWN_CAPABILITIES, LoadedManifest}; + +/// Check that every capability-bearing WIT import of the component is covered +/// by the module's manifest declarations. Call this after loading the +/// component but before instantiation. +/// +/// When `[capabilities]` is absent the manifest is in 0.1-fallback mode and +/// all imports are allowed; the caller is expected to have already emitted +/// a deprecation warning. +/// +/// `component_imports` should be the iterator returned by +/// `component.component_type().imports(&engine)` - pass the **name** part +/// (`&str`) of each `(&str, ComponentItem)` tuple. +pub fn enforce_capabilities<'a>( + loaded: &LoadedManifest, + component_imports: impl Iterator, +) -> Result<(), CapabilityViolation> { + let caps = match loaded.manifest.capabilities.as_ref() { + None => return Ok(()), // 0.1-fallback: no enforcement + Some(c) => c, + }; + + let declared: HashSet<&str> = caps + .required + .iter() + .chain(caps.optional.iter()) + .map(String::as_str) + .collect(); + + for import_name in component_imports { + if let Some(cap) = wit_import_to_cap(import_name) + && !declared.contains(cap) + { + return Err(CapabilityViolation { + capability: cap.to_owned(), + wit_import: import_name.to_owned(), + }); + } + } + Ok(()) +} + +/// Map a WIT import name to a capability name, or `None` for non-capability +/// imports. +/// +/// Returns `Some(iface)` only for interfaces in [`KNOWN_CAPABILITIES`]; +/// type-only packages like `nexum:host/types` and unrelated namespaces +/// (`wasi:*`) fall through to `None` so they do not need a manifest +/// declaration. +/// +/// Examples: +/// - `"nexum:host/chain@0.2.0"` -> `Some("chain")` +/// - `"shepherd:cow/cow-api@0.2.0"` -> `Some("cow-api")` +/// - `"nexum:host/types@0.2.0"` -> `None` (type-only, not a capability) +/// - `"wasi:io/streams@0.2.0"` -> `None` +pub(super) fn wit_import_to_cap(import_name: &str) -> Option<&str> { + let without_version = import_name.split('@').next().unwrap_or(import_name); + let iface = without_version + .strip_prefix("nexum:host/") + .or_else(|| without_version.strip_prefix("shepherd:cow/"))?; + if KNOWN_CAPABILITIES.contains(&iface) { + Some(iface) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::types::{CapabilitiesSection, Manifest}; + + #[test] + fn wit_import_to_cap_nexum_host() { + assert_eq!(wit_import_to_cap("nexum:host/chain@0.2.0"), Some("chain")); + assert_eq!( + wit_import_to_cap("nexum:host/local-store@0.2.0"), + Some("local-store") + ); + assert_eq!(wit_import_to_cap("nexum:host/http@0.2.0"), Some("http")); + } + + #[test] + fn wit_import_to_cap_shepherd_cow() { + assert_eq!( + wit_import_to_cap("shepherd:cow/cow-api@0.2.0"), + Some("cow-api") + ); + } + + #[test] + fn wit_import_to_cap_wasi_is_none() { + assert_eq!(wit_import_to_cap("wasi:io/streams@0.2.0"), None); + assert_eq!(wit_import_to_cap("wasi:cli/stdin@0.2.0"), None); + assert_eq!(wit_import_to_cap("wasi:sockets/tcp@0.2.0"), None); + } + + fn manifest_with_caps(required: &[&str], optional: &[&str]) -> LoadedManifest { + LoadedManifest { + manifest: Manifest { + capabilities: Some(CapabilitiesSection { + required: required.iter().map(|s| s.to_string()).collect(), + optional: optional.iter().map(|s| s.to_string()).collect(), + http: None, + }), + ..Default::default() + }, + http_allowlist: vec![], + config: vec![], + } + } + + fn manifest_no_caps() -> LoadedManifest { + LoadedManifest { + manifest: Manifest::default(), + http_allowlist: vec![], + config: vec![], + } + } + + #[test] + fn enforce_passes_when_caps_absent() { + // 0.1-fallback: no capabilities section -> all imports allowed + let loaded = manifest_no_caps(); + let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; + assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); + } + + #[test] + fn enforce_passes_when_all_imports_declared() { + let loaded = manifest_with_caps(&["chain", "cow-api"], &["http"]); + let imports = [ + "nexum:host/chain@0.2.0", + "shepherd:cow/cow-api@0.2.0", + "nexum:host/http@0.2.0", + "wasi:io/streams@0.2.0", // wasi is always skipped + ]; + assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); + } + + #[test] + fn enforce_rejects_undeclared_import() { + let loaded = manifest_with_caps(&["chain"], &[]); + // module imports remote-store but didn't declare it + let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; + let err = enforce_capabilities(&loaded, imports.into_iter()).unwrap_err(); + assert_eq!(err.capability, "remote-store"); + } + + #[test] + fn enforce_optional_caps_are_also_allowed() { + let loaded = manifest_with_caps(&["chain"], &["remote-store"]); + let imports = ["nexum:host/chain@0.2.0", "nexum:host/remote-store@0.2.0"]; + assert!(enforce_capabilities(&loaded, imports.into_iter()).is_ok()); + } +} diff --git a/crates/nexum-engine/src/manifest/error.rs b/crates/nexum-engine/src/manifest/error.rs new file mode 100644 index 0000000..98bf7d7 --- /dev/null +++ b/crates/nexum-engine/src/manifest/error.rs @@ -0,0 +1,51 @@ +//! Error types for manifest parsing and capability enforcement. + +use super::types::KNOWN_CAPABILITIES; + +/// Errors returned while loading or validating a manifest. +#[derive(Debug)] +pub enum ParseError { + Io(std::io::Error), + Toml(toml::de::Error), + UnknownCapability(String), +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => write!(f, "manifest: i/o: {e}"), + Self::Toml(e) => write!(f, "manifest: parse: {e}"), + Self::UnknownCapability(name) => write!( + f, + "manifest: unknown capability {:?} in [capabilities].required (known: {})", + name, + KNOWN_CAPABILITIES.join(", ") + ), + } + } +} + +impl std::error::Error for ParseError {} + +/// Error returned when a component's WIT imports exceed its declared capabilities. +#[derive(Debug)] +pub struct CapabilityViolation { + /// Capability name (e.g. `"remote-store"`). + pub capability: String, + /// Full WIT import name as it appeared in the component (e.g. + /// `"nexum:host/remote-store@0.2.0"`). + pub wit_import: String, +} + +impl std::fmt::Display for CapabilityViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "component imports `{}` ({}) but it is not listed in \ + [capabilities].required or [capabilities].optional", + self.capability, self.wit_import + ) + } +} + +impl std::error::Error for CapabilityViolation {} diff --git a/crates/nexum-engine/src/manifest/load.rs b/crates/nexum-engine/src/manifest/load.rs new file mode 100644 index 0000000..b857a76 --- /dev/null +++ b/crates/nexum-engine/src/manifest/load.rs @@ -0,0 +1,253 @@ +//! Parse `module.toml` from disk, validate, and emit operator-visible +//! warnings. +//! +//! Also exposes the small URL/host helpers the `http` host backend +//! uses to enforce the manifest's `[capabilities.http].allow` list at +//! request time. + +use std::collections::HashSet; +use std::path::Path; + +use super::error::ParseError; +use super::types::{KNOWN_CAPABILITIES, LoadedManifest, Manifest}; + +/// Read `module.toml` from `path`, parse, validate, and emit a deprecation +/// warning if `[capabilities]` is absent (0.1-compat fallback). +pub fn load(path: &Path) -> Result { + let raw = std::fs::read_to_string(path).map_err(ParseError::Io)?; + let manifest: Manifest = toml::from_str(&raw).map_err(ParseError::Toml)?; + + let caps = manifest.capabilities.as_ref(); + if caps.is_none() { + eprintln!( + "[deprecation] no [capabilities] section in module.toml - \ + defaulting to all-required (0.1 behaviour). This default \ + will be removed in 0.3; add an explicit [capabilities] block." + ); + } + + if let Some(c) = caps { + let known: HashSet<&str> = KNOWN_CAPABILITIES.iter().copied().collect(); + for name in c.required.iter().chain(c.optional.iter()) { + if !known.contains(name.as_str()) { + return Err(ParseError::UnknownCapability(name.clone())); + } + } + if !c.required.is_empty() { + eprintln!( + "[manifest] required capabilities: {}", + c.required.join(", ") + ); + } + if !c.optional.is_empty() { + eprintln!( + "[manifest] optional capabilities (advisory in 0.2; trap-stub fallback \ + ships in 0.3): {}", + c.optional.join(", ") + ); + } + } + + let http_allowlist = caps + .and_then(|c| c.http.as_ref()) + .map(|h| h.allow.clone()) + .unwrap_or_default(); + if !http_allowlist.is_empty() { + eprintln!("[manifest] http allowlist: {}", http_allowlist.join(", ")); + } + + let config = manifest + .config + .iter() + .map(|(k, v)| (k.clone(), stringify_toml_value(v))) + .collect(); + + Ok(LoadedManifest { + manifest, + http_allowlist, + config, + }) +} + +/// Synthesise a "0.1 fallback" manifest for when no `module.toml` is found. +/// Emits the same deprecation warning as a missing-section manifest. +pub fn fallback_manifest() -> LoadedManifest { + eprintln!( + "[deprecation] no module.toml found - defaulting to all-required \ + (0.1 behaviour). This default will be removed in 0.3; ship a \ + module.toml alongside your component." + ); + LoadedManifest { + manifest: Manifest::default(), + http_allowlist: Vec::new(), + config: Vec::new(), + } +} + +/// Check whether `host` matches any pattern in the allowlist. Patterns are +/// either exact (`api.example.com`) or `*.suffix` wildcards which match +/// any subdomain of `suffix` (but not `suffix` itself). +pub fn host_allowed(host: &str, allowlist: &[String]) -> bool { + let host = host.to_ascii_lowercase(); + allowlist.iter().any(|pat| { + let pat = pat.to_ascii_lowercase(); + if let Some(suffix) = pat.strip_prefix("*.") { + host.ends_with(&format!(".{suffix}")) + } else { + host == pat + } + }) +} + +/// Extract the host component from a URL. Returns `None` for non-http(s) +/// schemes or malformed input. Intentionally simple - adds no `url` +/// crate dependency. +pub fn extract_host(url: &str) -> Option<&str> { + let after_scheme = url + .strip_prefix("https://") + .or_else(|| url.strip_prefix("http://"))?; + let host_end = after_scheme + .find('/') + .or_else(|| after_scheme.find('?')) + .unwrap_or(after_scheme.len()); + let host = &after_scheme[..host_end]; + // strip optional user-info and port. + let host = host.rsplit('@').next().unwrap_or(host); + let host = host.split(':').next().unwrap_or(host); + if host.is_empty() { None } else { Some(host) } +} + +fn stringify_toml_value(v: &toml::Value) -> String { + match v { + toml::Value::String(s) => s.clone(), + toml::Value::Integer(i) => i.to_string(), + toml::Value::Float(f) => f.to_string(), + toml::Value::Boolean(b) => b.to_string(), + toml::Value::Datetime(d) => d.to_string(), + toml::Value::Array(_) | toml::Value::Table(_) => v.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::types::Subscription; + + #[test] + fn load_parses_block_and_log_subscriptions() { + let toml = r#" +[module] +name = "twap-monitor" + +[capabilities] +required = ["chain", "local-store"] + +[[subscription]] +kind = "block" +chain_id = 1 + +[[subscription]] +kind = "log" +chain_id = 1 +address = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110" +event_signature = "0x00000000000000000000000000000000000000000000000000000000deadbeef" +"#; + let manifest: Manifest = toml::from_str(toml).expect("parse"); + assert_eq!(manifest.module.name, "twap-monitor"); + assert_eq!(manifest.subscriptions.len(), 2); + assert!(matches!( + &manifest.subscriptions[0], + Subscription::Block { chain_id: 1 } + )); + if let Subscription::Log { + chain_id, address, .. + } = &manifest.subscriptions[1] + { + assert_eq!(*chain_id, 1); + assert!(address.is_some()); + } else { + panic!("expected Log subscription"); + } + } + + #[test] + fn load_parses_cron_subscription() { + let toml = r#" +[module] +name = "scheduler" + +[[subscription]] +kind = "cron" +schedule = "*/5 * * * *" +"#; + let manifest: Manifest = toml::from_str(toml).expect("parse"); + assert!(matches!( + &manifest.subscriptions[0], + Subscription::Cron { .. } + )); + } + + #[test] + fn load_rejects_unknown_capability() { + let toml = r#" +[module] +name = "bad" + +[capabilities] +required = ["chain", "not-a-real-cap"] +"#; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("module.toml"); + std::fs::write(&path, toml).unwrap(); + let err = load(&path).unwrap_err(); + assert!(matches!(err, ParseError::UnknownCapability(ref name) if name == "not-a-real-cap")); + } + + #[test] + fn load_parses_config_table() { + let toml = r#" +[module] +name = "example" + +[config] +chain_id = 1 +label = "mainnet" +enabled = true +"#; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("module.toml"); + std::fs::write(&path, toml).unwrap(); + let loaded = load(&path).unwrap(); + let config: std::collections::HashMap<_, _> = loaded.config.into_iter().collect(); + assert_eq!(config.get("chain_id").map(String::as_str), Some("1")); + assert_eq!(config.get("label").map(String::as_str), Some("mainnet")); + assert_eq!(config.get("enabled").map(String::as_str), Some("true")); + } + + #[test] + fn extract_host_handles_common_shapes() { + assert_eq!( + extract_host("https://api.example.com/v1/x"), + Some("api.example.com") + ); + assert_eq!(extract_host("http://example.com"), Some("example.com")); + assert_eq!( + extract_host("https://user:pw@host.example.com:8443/x"), + Some("host.example.com") + ); + assert_eq!(extract_host("https://example.com?q=1"), Some("example.com")); + assert_eq!(extract_host("ftp://example.com"), None); + assert_eq!(extract_host("not a url"), None); + } + + #[test] + fn host_allowed_exact_and_wildcard() { + let allow = vec!["api.cow.fi".to_string(), "*.discord.com".to_string()]; + assert!(host_allowed("api.cow.fi", &allow)); + assert!(!host_allowed("evil.api.cow.fi", &allow)); + assert!(host_allowed("foo.discord.com", &allow)); + assert!(host_allowed("a.b.discord.com", &allow)); + assert!(!host_allowed("discord.com", &allow)); + assert!(!host_allowed("nope.example", &allow)); + } +} diff --git a/crates/nexum-engine/src/manifest/mod.rs b/crates/nexum-engine/src/manifest/mod.rs new file mode 100644 index 0000000..9cd00b6 --- /dev/null +++ b/crates/nexum-engine/src/manifest/mod.rs @@ -0,0 +1,40 @@ +//! `module.toml` parser and capability-enforcement helpers (0.2 scope). +//! +//! 0.2 intentionally ships a slim subset of the manifest spec: +//! +//! - `[capabilities].required` is parsed and validated (names must be in +//! the known capability set; the 0.2 reference engine always provides +//! all of them, so this is a sanity check + future-proofing). +//! - `[capabilities].optional` is parsed and logged; trap-stub fallback +//! for absent optionals is deferred to 0.3. +//! - `[capabilities.http].allow` is parsed and consulted by the `http` +//! host impl before any outbound call. +//! - `[config]` is flattened to `Vec<(String, String)>` and passed to the +//! module's `init`. Typed `config-value` variant is deferred to 0.3. +//! +//! When the manifest file is missing or has no `[capabilities]` section, +//! a deprecation warning is emitted and the engine falls back to 0.1 +//! behaviour (treat every linked capability as required). This fallback +//! will be removed in 0.3. +//! +//! ## Layout +//! +//! - [`types`]: the serde `Manifest` shape + `LoadedManifest` the engine +//! actually consumes, plus the `KNOWN_CAPABILITIES` registry. +//! - [`load`]: `module.toml` -> `LoadedManifest`, plus the host/URL +//! helpers the `http` backend uses at request time. +//! - [`capabilities`]: WIT-import vs declared-capabilities cross-check. +//! - [`error`]: `ParseError`, `CapabilityViolation`. + +mod capabilities; +mod error; +mod load; +mod types; + +pub use capabilities::enforce_capabilities; +pub use load::{extract_host, fallback_manifest, host_allowed, load}; +pub use types::{LoadedManifest, Subscription}; +// CapabilityViolation, ParseError, and the *Section structs are +// reachable through these functions' return / argument types; +// consumers that need to name them directly do so via +// `crate::manifest::error::*` or `::types::*`. diff --git a/crates/nexum-engine/src/manifest/types.rs b/crates/nexum-engine/src/manifest/types.rs new file mode 100644 index 0000000..403a201 --- /dev/null +++ b/crates/nexum-engine/src/manifest/types.rs @@ -0,0 +1,123 @@ +//! Data structures: `Manifest`, sections, and `LoadedManifest`. +//! +//! Plain serde shapes plus the `KNOWN_CAPABILITIES` registry. The parsing +//! and validation logic lives in [`super::load`]; capability enforcement +//! in [`super::capabilities`]. + +use serde::Deserialize; + +/// Capability names recognised by the 0.2 reference engine. Matches the +/// interfaces the `shepherd` world links into the linker. +pub const KNOWN_CAPABILITIES: &[&str] = &[ + "chain", + "identity", + "local-store", + "remote-store", + "messaging", + "logging", + "clock", + "random", + "http", + // Domain-extension caps (provided by the shepherd world only): + "cow-api", +]; + +#[derive(Debug, Deserialize, Default)] +pub struct Manifest { + #[serde(default)] + pub module: ModuleSection, + #[serde(default)] + pub capabilities: Option, + #[serde(default)] + pub config: toml::Table, + /// Event subscriptions the runtime wires before calling + /// `_init`. See `docs/02-modules-events-packaging.md` for the + /// schema; 0.2 implements `block` and `log` kinds, `cron` is + /// parsed and ignored (deferred to 0.3). + #[serde(default, rename = "subscription")] + pub subscriptions: Vec, +} + +/// One `[[subscription]]` table in `module.toml`. +/// +/// The discriminator is the `kind` field; remaining fields are +/// validated per-kind by the supervisor. Unknown kinds are surfaced +/// at load time so a typo does not silently disable an event source. +#[derive(Debug, Deserialize, Clone)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Subscription { + /// New-block events. Fan-out is shared per chain - the + /// supervisor opens one subscription per chain id and routes to + /// every module that asked for blocks on that chain. + Block { + /// EVM chain id. + chain_id: u64, + }, + /// Log events matching `address` + topic-0. Fan-out is + /// per-module - the supervisor opens one subscription per + /// `[[subscription]]` entry and tags emitted events with the + /// owning module. + Log { + /// EVM chain id. + chain_id: u64, + /// Contract address as `0x`-prefixed 20-byte hex. Optional. + #[serde(default)] + address: Option, + /// Topic-0 of the event the module wants to consume. `0x`- + /// prefixed 32-byte hex. Optional - when absent the + /// subscription matches every event from the address(es). + #[serde(default)] + event_signature: Option, + }, + /// Cron-scheduled tick. 0.2 parses but does not dispatch; the + /// supervisor emits a warning so the operator knows the + /// declaration is currently inert. `schedule` is preserved so a + /// 0.3 dispatcher can pick it up without re-parsing the manifest. + Cron { + /// Standard 5-field cron expression. + #[allow(dead_code)] + schedule: String, + }, +} + +#[derive(Debug, Deserialize, Default)] +#[allow(dead_code)] // version + component parsed for future 0.3 hash-verification. +pub struct ModuleSection { + #[serde(default)] + pub name: String, + #[serde(default)] + pub version: String, + #[serde(default)] + pub component: String, +} + +#[derive(Debug, Deserialize, Default)] +pub struct CapabilitiesSection { + #[serde(default)] + pub required: Vec, + #[serde(default)] + pub optional: Vec, + #[serde(default)] + pub http: Option, +} + +#[derive(Debug, Deserialize, Default)] +pub struct HttpSection { + #[serde(default)] + pub allow: Vec, +} + +/// Loaded + validated manifest, plus the data the engine needs to +/// instantiate a module. +#[derive(Debug)] +pub struct LoadedManifest { + pub manifest: Manifest, + /// Hosts to allow for `http::fetch`. Each entry is either an exact + /// hostname or a `*.suffix` wildcard. + pub http_allowlist: Vec, + /// `[config]` flattened to `(key, stringified-value)` pairs ready to + /// hand to a module's `init`. TOML scalars (string, integer, float, + /// boolean) become their text form. Arrays and tables are rendered as + /// their TOML representation. + pub config: Vec<(String, String)>, +} From fe0c528d2b86a8d4ca5fbfc6670ea0dc8e6a6d74 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Sat, 13 Jun 2026 09:35:22 -0300 Subject: [PATCH 15/29] refactor(main): extract host impls + CLI + event loop + limits main.rs went from 739 lines of mixed bootstrap + 8 Host trait impls + CLI parser + event loop to ~125 lines of pure orchestration. New layout: - bindings.rs: wasmtime::component::bindgen!() moved out so other modules can name the generated types. - cli.rs: Cli struct + manual parser. - host/state.rs: HostState + WasiView impl. - host/error.rs: unimplemented / internal_error / hex_encode helpers. - host/impls/{chain,cow_api,identity,local_store,remote_store,messaging, logging,clock,random,http,types}.rs: one Host trait impl per file. - runtime/limits.rs: DEFAULT_FUEL_PER_EVENT + DEFAULT_MEMORY_LIMIT. - runtime/event_loop.rs: open_block_streams, open_log_streams, run, wait_for_shutdown_signal, TaggedBlockStream, TaggedLogStream. Adding a new capability is now a single new file under host/impls/ rather than a 60-80 line diff in main.rs. --- crates/nexum-engine/src/bindings.rs | 16 + crates/nexum-engine/src/cli.rs | 38 ++ crates/nexum-engine/src/host/error.rs | 42 ++ crates/nexum-engine/src/host/impls/chain.rs | 67 ++ crates/nexum-engine/src/host/impls/clock.rs | 19 + crates/nexum-engine/src/host/impls/cow_api.rs | 85 +++ crates/nexum-engine/src/host/impls/http.rs | 51 ++ .../nexum-engine/src/host/impls/identity.rs | 29 + .../src/host/impls/local_store.rs | 32 + crates/nexum-engine/src/host/impls/logging.rs | 18 + .../nexum-engine/src/host/impls/messaging.rs | 27 + crates/nexum-engine/src/host/impls/mod.rs | 19 + crates/nexum-engine/src/host/impls/random.rs | 16 + .../src/host/impls/remote_store.rs | 40 ++ crates/nexum-engine/src/host/impls/types.rs | 7 + crates/nexum-engine/src/host/mod.rs | 23 +- crates/nexum-engine/src/host/state.rs | 45 ++ crates/nexum-engine/src/main.rs | 641 +----------------- crates/nexum-engine/src/runtime/event_loop.rs | 157 +++++ crates/nexum-engine/src/runtime/limits.rs | 14 + crates/nexum-engine/src/runtime/mod.rs | 5 + crates/nexum-engine/src/supervisor.rs | 28 +- 22 files changed, 775 insertions(+), 644 deletions(-) create mode 100644 crates/nexum-engine/src/bindings.rs create mode 100644 crates/nexum-engine/src/cli.rs create mode 100644 crates/nexum-engine/src/host/error.rs create mode 100644 crates/nexum-engine/src/host/impls/chain.rs create mode 100644 crates/nexum-engine/src/host/impls/clock.rs create mode 100644 crates/nexum-engine/src/host/impls/cow_api.rs create mode 100644 crates/nexum-engine/src/host/impls/http.rs create mode 100644 crates/nexum-engine/src/host/impls/identity.rs create mode 100644 crates/nexum-engine/src/host/impls/local_store.rs create mode 100644 crates/nexum-engine/src/host/impls/logging.rs create mode 100644 crates/nexum-engine/src/host/impls/messaging.rs create mode 100644 crates/nexum-engine/src/host/impls/mod.rs create mode 100644 crates/nexum-engine/src/host/impls/random.rs create mode 100644 crates/nexum-engine/src/host/impls/remote_store.rs create mode 100644 crates/nexum-engine/src/host/impls/types.rs create mode 100644 crates/nexum-engine/src/host/state.rs create mode 100644 crates/nexum-engine/src/runtime/event_loop.rs create mode 100644 crates/nexum-engine/src/runtime/limits.rs create mode 100644 crates/nexum-engine/src/runtime/mod.rs diff --git a/crates/nexum-engine/src/bindings.rs b/crates/nexum-engine/src/bindings.rs new file mode 100644 index 0000000..d0f57dd --- /dev/null +++ b/crates/nexum-engine/src/bindings.rs @@ -0,0 +1,16 @@ +//! WIT bindings generated by `wasmtime::component::bindgen!`. +//! +//! Both `wit/nexum-host` and `wit/shepherd-cow` packages are listed +//! explicitly so wit-parser can resolve the cross-package reference +//! natively - no vendored `deps/` tree needed. The world name is fully +//! qualified. +//! +//! Every `Host` trait impl in [`crate::host::impls`] consumes types +//! generated here. + +wasmtime::component::bindgen!({ + path: ["../../wit/nexum-host", "../../wit/shepherd-cow"], + world: "shepherd:cow/shepherd", + imports: { default: async }, + exports: { default: async }, +}); diff --git a/crates/nexum-engine/src/cli.rs b/crates/nexum-engine/src/cli.rs new file mode 100644 index 0000000..80c86ba --- /dev/null +++ b/crates/nexum-engine/src/cli.rs @@ -0,0 +1,38 @@ +//! CLI surface for the `nexum-engine` binary, derived via clap. +//! +//! The 0.2 binary accepts either a positional ` []` +//! shortcut that synthesises a one-module engine config, or a +//! `--engine-config ` flag that points at a TOML declaring +//! multiple modules. Production deployments use the second form; the +//! positional shortcut stays for parity with the M1 reference CLI and +//! for smoke tests. + +use std::path::PathBuf; + +use clap::Parser; + +/// Parsed CLI surface. +/// +/// `nexum-engine [ []] [--engine-config ]` +#[derive(Parser, Debug, Default)] +#[command( + name = "nexum-engine", + about = "Run one or more Wasm Component modules under the Shepherd supervisor", + long_about = None, + version, +)] +pub struct Cli { + /// Optional positional path to a Wasm Component file. Synthesises + /// a one-module engine config when no `--engine-config` is given. + pub wasm: Option, + + /// Optional positional path to the module's `nexum.toml` manifest. + /// Only consulted alongside the positional `wasm` shortcut. + pub manifest: Option, + + /// Optional explicit path to the engine-wide `engine.toml` config. + /// When omitted, the engine resolves the default search path + /// documented in `engine_config::load_or_default`. + #[arg(long = "engine-config")] + pub engine_config: Option, +} diff --git a/crates/nexum-engine/src/host/error.rs b/crates/nexum-engine/src/host/error.rs new file mode 100644 index 0000000..15b7255 --- /dev/null +++ b/crates/nexum-engine/src/host/error.rs @@ -0,0 +1,42 @@ +//! Small constructors that wrap the WIT `HostError` shape, used by +//! every `Host` trait impl, plus the lowercase hex encoder shared by +//! the `cow-api` submission path. + +use crate::bindings::HostError; +use crate::bindings::nexum::host::types::HostErrorKind; + +/// `Unsupported` (HTTP 501-style) error for capabilities the engine +/// reference build does not implement yet. +pub(crate) fn unimplemented(domain: &str, detail: impl Into) -> HostError { + HostError { + domain: domain.into(), + kind: HostErrorKind::Unsupported, + code: 501, + message: detail.into(), + data: None, + } +} + +/// `Internal` (HTTP 500-style) error for unexpected backend failures. +pub(crate) fn internal_error(domain: &str, detail: impl Into) -> HostError { + HostError { + domain: domain.into(), + kind: HostErrorKind::Internal, + code: 0, + message: detail.into(), + data: None, + } +} + +/// Lowercase hex encoder. Kept in the engine binary rather than +/// pulling a `hex` crate just for one call site. Writes into the +/// pre-allocated buffer to avoid the per-byte `String` allocation +/// `format!("{b:02x}")` would do. +pub(crate) fn hex_encode(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + write!(s, "{b:02x}").expect("writing to String never fails"); + } + s +} diff --git a/crates/nexum-engine/src/host/impls/chain.rs b/crates/nexum-engine/src/host/impls/chain.rs new file mode 100644 index 0000000..e0e30db --- /dev/null +++ b/crates/nexum-engine/src/host/impls/chain.rs @@ -0,0 +1,67 @@ +//! `nexum:host/chain`: raw JSON-RPC dispatch over alloy. + +use std::time::Instant; + +use crate::bindings::HostError; +use crate::bindings::nexum; +use crate::bindings::nexum::host::types::HostErrorKind; +use crate::host::error::internal_error; +use crate::host::provider_pool::ProviderError; +use crate::host::state::HostState; + +impl nexum::host::chain::Host for HostState { + async fn request( + &mut self, + chain_id: u64, + method: String, + params: String, + ) -> Result { + let start = Instant::now(); + tracing::debug!(chain_id, %method, "chain::request"); + let result = match self.chain.request(chain_id, method, params).await { + Ok(body) => Ok(body), + Err(ProviderError::UnknownChain(id)) => Err(HostError { + domain: "chain".into(), + kind: HostErrorKind::Unsupported, + code: 0, + message: format!("chain {id} has no engine.toml RPC entry"), + data: None, + }), + Err(ProviderError::InvalidParams { detail, .. }) => Err(HostError { + domain: "chain".into(), + kind: HostErrorKind::InvalidInput, + code: -32602, + message: detail, + data: None, + }), + Err(ProviderError::Rpc { detail, .. }) => Err(HostError { + domain: "chain".into(), + kind: HostErrorKind::Internal, + code: -32603, + message: detail, + data: None, + }), + Err(err) => Err(internal_error("chain", err.to_string())), + }; + tracing::trace!(elapsed_ms = ?start.elapsed(), "chain::request done"); + result + } + + async fn request_batch( + &mut self, + chain_id: u64, + requests: Vec, + ) -> Result, HostError> { + let start = Instant::now(); + tracing::debug!(chain_id, count = requests.len(), "chain::request-batch"); + let mut out = Vec::with_capacity(requests.len()); + for req in requests { + match nexum::host::chain::Host::request(self, chain_id, req.method, req.params).await { + Ok(s) => out.push(nexum::host::chain::RpcResult::Ok(s)), + Err(e) => out.push(nexum::host::chain::RpcResult::Err(e)), + } + } + tracing::trace!(elapsed_ms = ?start.elapsed(), "chain::request-batch done"); + Ok(out) + } +} diff --git a/crates/nexum-engine/src/host/impls/clock.rs b/crates/nexum-engine/src/host/impls/clock.rs new file mode 100644 index 0000000..f65b674 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/clock.rs @@ -0,0 +1,19 @@ +//! `nexum:host/clock`: wall-clock + monotonic time. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::bindings::nexum; +use crate::host::state::HostState; + +impl nexum::host::clock::Host for HostState { + async fn now_ms(&mut self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) + } + + async fn monotonic_ns(&mut self) -> u64 { + self.monotonic_baseline.elapsed().as_nanos() as u64 + } +} diff --git a/crates/nexum-engine/src/host/impls/cow_api.rs b/crates/nexum-engine/src/host/impls/cow_api.rs new file mode 100644 index 0000000..b3b51d5 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/cow_api.rs @@ -0,0 +1,85 @@ +//! `shepherd:cow/cow-api`: REST passthrough + typed `submit_order`. +//! Backend logic lives in [`crate::host::cow_orderbook`]; this is the +//! WIT-side error mapping. + +use std::time::Instant; + +use crate::bindings::nexum::host::types::HostErrorKind; +use crate::bindings::{HostError, shepherd}; +use crate::host::cow_orderbook::CowApiError; +use crate::host::error::{hex_encode, internal_error, unimplemented}; +use crate::host::state::HostState; + +impl shepherd::cow::cow_api::Host for HostState { + async fn request( + &mut self, + chain_id: u64, + method: String, + path: String, + body: Option, + ) -> Result { + let start = Instant::now(); + tracing::debug!(chain_id, %method, %path, "cow-api::request"); + let result = match self + .cow + .request(chain_id, &method, &path, body.as_deref()) + .await + { + Ok(body) => Ok(body), + Err(CowApiError::UnknownChain(id)) => Err(unimplemented( + "cow-api", + format!("chain {id} not in cowprotocol"), + )), + Err(CowApiError::BadMethod(m)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: format!("unsupported HTTP method: {m}"), + data: None, + }), + Err(CowApiError::BadPath(msg)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: msg, + data: None, + }), + Err(err) => Err(internal_error("cow-api", err.to_string())), + }; + tracing::trace!(elapsed_ms = ?start.elapsed(), "cow-api::request done"); + result + } + + async fn submit_order( + &mut self, + chain_id: u64, + order_data: Vec, + ) -> Result { + let start = Instant::now(); + tracing::debug!(chain_id, bytes = order_data.len(), "cow-api::submit-order"); + let result = match self.cow.submit_order_json(chain_id, &order_data).await { + Ok(uid) => Ok(format!("0x{}", hex_encode(uid.as_slice()))), + Err(CowApiError::UnknownChain(id)) => Err(unimplemented( + "cow-api", + format!("chain {id} not in cowprotocol"), + )), + Err(CowApiError::Decode(err)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: format!("invalid OrderCreation JSON: {err}"), + data: None, + }), + Err(CowApiError::Orderbook(err)) => Err(HostError { + domain: "cow-api".into(), + kind: HostErrorKind::Denied, + code: 0, + message: err.to_string(), + data: None, + }), + Err(err) => Err(internal_error("cow-api", err.to_string())), + }; + tracing::trace!(elapsed_ms = ?start.elapsed(), "cow-api::submit-order done"); + result + } +} diff --git a/crates/nexum-engine/src/host/impls/http.rs b/crates/nexum-engine/src/host/impls/http.rs new file mode 100644 index 0000000..2900d4d --- /dev/null +++ b/crates/nexum-engine/src/host/impls/http.rs @@ -0,0 +1,51 @@ +//! `nexum:host/http`: manifest allowlist check, then `Unsupported`. +//! +//! Real `fetch` lands in 0.3. The allowlist is enforced now so a +//! module that ships with an empty (or no) `[capabilities.http].allow` +//! gets denied loudly, matching the "no implicit network" stance. + +use tracing::warn; + +use crate::bindings::HostError; +use crate::bindings::nexum; +use crate::bindings::nexum::host::types::HostErrorKind; +use crate::host::error::unimplemented; +use crate::host::state::HostState; +use crate::manifest::{extract_host, host_allowed}; + +impl nexum::host::http::Host for HostState { + async fn fetch( + &mut self, + req: nexum::host::http::Request, + ) -> Result { + let host = match extract_host(&req.url) { + Some(h) => h, + None => { + return Err(HostError { + domain: "http".into(), + kind: HostErrorKind::InvalidInput, + code: 0, + message: format!("not an http(s) URL: {}", req.url), + data: None, + }); + } + }; + if !host_allowed(host, &self.http_allowlist) { + warn!(host, "[http] denied by allowlist"); + return Err(HostError { + domain: "http".into(), + kind: HostErrorKind::Denied, + code: 0, + message: format!( + "host {host} not in [capabilities.http].allow; \ + add it to module.toml to permit" + ), + data: None, + }); + } + Err(unimplemented( + "http", + "fetch not implemented in 0.2 reference runtime (allowlist passed)", + )) + } +} diff --git a/crates/nexum-engine/src/host/impls/identity.rs b/crates/nexum-engine/src/host/impls/identity.rs new file mode 100644 index 0000000..6a4a050 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/identity.rs @@ -0,0 +1,29 @@ +//! `nexum:host/identity`: deferred to 0.3 (keystore / KMS backend). +//! `accounts()` returns an empty roster so guests can probe-then-skip; +//! signing returns `Unsupported`. + +use crate::bindings::HostError; +use crate::bindings::nexum; +use crate::host::error::unimplemented; +use crate::host::state::HostState; + +impl nexum::host::identity::Host for HostState { + async fn accounts(&mut self) -> Result>, HostError> { + Ok(vec![]) + } + + async fn sign(&mut self, _account: Vec, _message: Vec) -> Result, HostError> { + Err(unimplemented("identity", "sign requires a keystore (0.3)")) + } + + async fn sign_typed_data( + &mut self, + _account: Vec, + _typed_data: String, + ) -> Result, HostError> { + Err(unimplemented( + "identity", + "sign-typed-data requires a keystore (0.3)", + )) + } +} diff --git a/crates/nexum-engine/src/host/impls/local_store.rs b/crates/nexum-engine/src/host/impls/local_store.rs new file mode 100644 index 0000000..66bcc52 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/local_store.rs @@ -0,0 +1,32 @@ +//! `nexum:host/local-store`: redb backend with host-side namespacing. + +use crate::bindings::HostError; +use crate::bindings::nexum; +use crate::host::error::internal_error; +use crate::host::state::HostState; + +impl nexum::host::local_store::Host for HostState { + async fn get(&mut self, key: String) -> Result>, HostError> { + self.store + .get(&self.module_namespace, &key) + .map_err(|err| internal_error("local-store", err.to_string())) + } + + async fn set(&mut self, key: String, value: Vec) -> Result<(), HostError> { + self.store + .set(&self.module_namespace, &key, &value) + .map_err(|err| internal_error("local-store", err.to_string())) + } + + async fn delete(&mut self, key: String) -> Result<(), HostError> { + self.store + .delete(&self.module_namespace, &key) + .map_err(|err| internal_error("local-store", err.to_string())) + } + + async fn list_keys(&mut self, prefix: String) -> Result, HostError> { + self.store + .list_keys(&self.module_namespace, &prefix) + .map_err(|err| internal_error("local-store", err.to_string())) + } +} diff --git a/crates/nexum-engine/src/host/impls/logging.rs b/crates/nexum-engine/src/host/impls/logging.rs new file mode 100644 index 0000000..b3a2a02 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/logging.rs @@ -0,0 +1,18 @@ +//! `nexum:host/logging`: routes guest log lines through the host's +//! `tracing` subscriber, tagged with the module namespace. + +use crate::bindings::nexum; +use crate::host::state::HostState; + +impl nexum::host::logging::Host for HostState { + async fn log(&mut self, level: nexum::host::logging::Level, message: String) { + let module = self.module_namespace.as_str(); + match level { + nexum::host::logging::Level::Trace => tracing::trace!(module, "{}", message), + nexum::host::logging::Level::Debug => tracing::debug!(module, "{}", message), + nexum::host::logging::Level::Info => tracing::info!(module, "{}", message), + nexum::host::logging::Level::Warn => tracing::warn!(module, "{}", message), + nexum::host::logging::Level::Error => tracing::error!(module, "{}", message), + } + } +} diff --git a/crates/nexum-engine/src/host/impls/messaging.rs b/crates/nexum-engine/src/host/impls/messaging.rs new file mode 100644 index 0000000..5582b00 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/messaging.rs @@ -0,0 +1,27 @@ +//! `nexum:host/messaging`: deferred to 0.3 (Waku backend). `query` +//! returns an empty result, same posture as `identity::accounts`. + +use crate::bindings::HostError; +use crate::bindings::nexum; +use crate::host::error::unimplemented; +use crate::host::state::HostState; + +impl nexum::host::messaging::Host for HostState { + async fn publish( + &mut self, + _content_topic: String, + _payload: Vec, + ) -> Result<(), HostError> { + Err(unimplemented("messaging", "Waku backend deferred to 0.3")) + } + + async fn query( + &mut self, + _content_topic: String, + _start_time: Option, + _end_time: Option, + _limit: Option, + ) -> Result, HostError> { + Ok(vec![]) + } +} diff --git a/crates/nexum-engine/src/host/impls/mod.rs b/crates/nexum-engine/src/host/impls/mod.rs new file mode 100644 index 0000000..8256af9 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/mod.rs @@ -0,0 +1,19 @@ +//! `Host` trait impls for [`crate::host::state::HostState`], one +//! file per WIT interface. +//! +//! The interfaces themselves (and their generated trait shapes) live +//! in [`crate::bindings`]; this module only contains the dispatch +//! glue between the WIT signature and the corresponding backend in +//! [`crate::host`]. + +mod chain; +mod clock; +mod cow_api; +mod http; +mod identity; +mod local_store; +mod logging; +mod messaging; +mod random; +mod remote_store; +mod types; diff --git a/crates/nexum-engine/src/host/impls/random.rs b/crates/nexum-engine/src/host/impls/random.rs new file mode 100644 index 0000000..88e2f8c --- /dev/null +++ b/crates/nexum-engine/src/host/impls/random.rs @@ -0,0 +1,16 @@ +//! `nexum:host/random`: fills `len` bytes from the OS CSPRNG. +//! Getrandom 0.4 failures are exceptionally rare on supported +//! platforms; on failure we return zero-filled bytes - guests that +//! need a strong-failure signal should use identity or chain primitives +//! instead. + +use crate::bindings::nexum; +use crate::host::state::HostState; + +impl nexum::host::random::Host for HostState { + async fn fill(&mut self, len: u32) -> Vec { + let mut buf = vec![0u8; len as usize]; + let _ = getrandom::fill(&mut buf); + buf + } +} diff --git a/crates/nexum-engine/src/host/impls/remote_store.rs b/crates/nexum-engine/src/host/impls/remote_store.rs new file mode 100644 index 0000000..9001d1f --- /dev/null +++ b/crates/nexum-engine/src/host/impls/remote_store.rs @@ -0,0 +1,40 @@ +//! `nexum:host/remote-store`: deferred to 0.3 (Swarm backend). + +use crate::bindings::HostError; +use crate::bindings::nexum; +use crate::host::error::unimplemented; +use crate::host::state::HostState; + +impl nexum::host::remote_store::Host for HostState { + async fn upload(&mut self, _data: Vec) -> Result, HostError> { + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) + } + + async fn download(&mut self, _reference: Vec) -> Result, HostError> { + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) + } + + async fn read_feed( + &mut self, + _owner: Vec, + _topic: Vec, + ) -> Result>, HostError> { + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) + } + + async fn write_feed(&mut self, _topic: Vec, _data: Vec) -> Result, HostError> { + Err(unimplemented( + "remote-store", + "Swarm backend deferred to 0.3", + )) + } +} diff --git a/crates/nexum-engine/src/host/impls/types.rs b/crates/nexum-engine/src/host/impls/types.rs new file mode 100644 index 0000000..c4a93e1 --- /dev/null +++ b/crates/nexum-engine/src/host/impls/types.rs @@ -0,0 +1,7 @@ +//! `nexum:host/types` is a type-only interface (no functions). The +//! generated trait is empty; we just provide the marker impl. + +use crate::bindings::nexum; +use crate::host::state::HostState; + +impl nexum::host::types::Host for HostState {} diff --git a/crates/nexum-engine/src/host/mod.rs b/crates/nexum-engine/src/host/mod.rs index b76fe99..20f2ec2 100644 --- a/crates/nexum-engine/src/host/mod.rs +++ b/crates/nexum-engine/src/host/mod.rs @@ -1,12 +1,21 @@ -//! Host-side backends for the `nexum:host` / `shepherd:cow` -//! interfaces. +//! Host-side backends for the `nexum:host` / `shepherd:cow` interfaces, +//! plus the per-module `HostState` and the WIT `Host` trait impls. //! -//! Each submodule owns one capability. The trait impls in `main.rs` -//! stay thin: they validate inputs, dispatch to the backend, and -//! project the backend's typed error onto the bindgen-generated -//! `HostError`. Keeping the backends pure (no bindgen types) means -//! each can be unit-tested without spinning up a wasmtime store. +//! Layout: +//! - [`state`]: `HostState` struct + `WasiView` impl, the receiver +//! every WIT `Host` trait is implemented for. +//! - [`error`]: small constructors that build the WIT `HostError` +//! shape (`unimplemented`, `internal_error`) plus the lowercase +//! `hex_encode` shared by the `cow-api` submission path. +//! - [`cow_orderbook`], [`provider_pool`], [`local_store_redb`]: +//! capability backends. Pure code with no bindgen types, so each +//! can be unit-tested without spinning up a wasmtime store. +//! - [`impls`] (private): the bindgen-side trait impls, one file per +//! WIT interface, that dispatch to the backends above. pub mod cow_orderbook; +pub mod error; +mod impls; pub mod local_store_redb; pub mod provider_pool; +pub mod state; diff --git a/crates/nexum-engine/src/host/state.rs b/crates/nexum-engine/src/host/state.rs new file mode 100644 index 0000000..ec50219 --- /dev/null +++ b/crates/nexum-engine/src/host/state.rs @@ -0,0 +1,45 @@ +//! Per-instance host state and its WASI view. +//! +//! One [`HostState`] is created per module, lives inside the wasmtime +//! `Store`, and is the receiver every `Host` trait impl in +//! [`super::impls`] is implemented for. + +use std::time::Instant; + +use wasmtime::component::ResourceTable; +use wasmtime_wasi::{WasiCtx, WasiCtxView, WasiView}; + +use super::cow_orderbook::OrderBookPool; +use super::local_store_redb::LocalStore; +use super::provider_pool::ProviderPool; + +pub(crate) struct HostState { + pub wasi: WasiCtx, + pub table: ResourceTable, + /// Wasmtime memory/table/instance resource limits for this store. + pub limits: wasmtime::StoreLimits, + /// Origin for `clock::monotonic-ns`. Differences between successive + /// readings are the only meaningful values. + pub monotonic_baseline: Instant, + /// Per-module `[capabilities.http].allow` allowlist (from module.toml). + /// Consulted by `http::fetch` before any outbound call. + pub http_allowlist: Vec, + /// Namespace for the running module's `local-store` rows. Set from + /// `manifest.module.name` at instantiation. + pub module_namespace: String, + /// `cow-api` backend - per-chain `OrderBookApi` clients + reqwest. + pub cow: OrderBookPool, + /// `chain` backend - per-chain alloy `DynProvider` pool. + pub chain: ProviderPool, + /// `local-store` backend - redb file with host-side namespacing. + pub store: LocalStore, +} + +impl WasiView for HostState { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index d05b4d6..8a1598f 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -8,483 +8,26 @@ use alloy_rpc_client as _; use alloy_transport as _; use alloy_transport_ws as _; +mod bindings; +mod cli; mod engine_config; mod host; mod manifest; +mod runtime; mod supervisor; -use std::path::PathBuf; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; - use clap::Parser; -use futures::StreamExt; -use futures::stream::{FuturesUnordered, select_all}; use tracing::{info, warn}; use tracing_subscriber::EnvFilter; use wasmtime::Engine; -use wasmtime::component::{Linker, ResourceTable}; -use wasmtime::error::Context as _; -use wasmtime_wasi::{WasiCtx, WasiCtxView, WasiView}; - -/// Reference CLI for the 0.2 `nexum-engine` runtime. -/// -/// Loads a Wasm Component, links the `shepherd:cow/shepherd` host -/// world plus the WASI p2 set, calls `init` once, then dispatches a -/// single synthetic block event so the host stubs exercise their -/// timing paths. Production deployments invoke the engine through -/// the supervisor entrypoint introduced in later milestones; this -/// CLI is the M1 smoke-test surface. -#[derive(Parser, Debug)] -#[command( - name = "nexum-engine", - about = "Load a Wasm Component and dispatch a synthetic block event", - long_about = None, - version, -)] -struct Cli { - /// Optional path to a Wasm Component file. Backwards-compat - /// shortcut that synthesises a one-module engine config when no - /// `--engine-config` is supplied. Production deployments declare - /// modules in TOML and omit this positional. - wasm: Option, - - /// Optional explicit path to the module's `nexum.toml` manifest. - /// Only consulted alongside the positional `wasm` shortcut; when - /// the engine config drives the module set, each module brings - /// its own manifest via the engine-config entries. - manifest: Option, - - /// Optional explicit path to the engine-wide `engine.toml` config. - /// When omitted, the engine resolves the default search path - /// documented in `engine_config::load_or_default`. - #[arg(long = "engine-config")] - engine_config: Option, -} - -// Both packages are listed explicitly so wit-parser can resolve the -// cross-package reference natively - no vendored deps/ tree needed. -// World name is fully qualified. -wasmtime::component::bindgen!({ - path: ["../../wit/nexum-host", "../../wit/shepherd-cow"], - world: "shepherd:cow/shepherd", - imports: { default: async }, - exports: { default: async }, -}); - -use nexum::host::types::HostErrorKind; - -/// Default fuel budget granted per `on_event` invocation (≈ 1 billion WASM -/// instructions). Modules that exceed this budget trap with `OutOfFuel`. -/// Configurable per-module via `engine.toml` in 0.3. -pub const DEFAULT_FUEL_PER_EVENT: u64 = 1_000_000_000; - -/// Default linear-memory cap per module store (64 MiB). Prevents a single -/// runaway module from exhausting process memory. Configurable in 0.3. -pub const DEFAULT_MEMORY_LIMIT: usize = 64 * 1024 * 1024; - -struct HostState { - wasi: WasiCtx, - table: ResourceTable, - /// Wasmtime memory/table/instance resource limits for this store. - limits: wasmtime::StoreLimits, - /// Origin for `clock::monotonic-ns`. Differences between successive - /// readings are the only meaningful values. - monotonic_baseline: Instant, - /// Per-module `[capabilities.http].allow` allowlist (from module.toml). - /// Consulted by `http::fetch` before any outbound call. - http_allowlist: Vec, - /// Namespace for the running module's `local-store` rows. Set from - /// `manifest.module.name` at instantiation. - module_namespace: String, - /// `cow-api` backend - per-chain `OrderBookApi` clients + reqwest. - cow: host::cow_orderbook::OrderBookPool, - /// `chain` backend - per-chain alloy `DynProvider` pool. - chain: host::provider_pool::ProviderPool, - /// `local-store` backend - redb file with host-side namespacing. - store: host::local_store_redb::LocalStore, -} - -impl WasiView for HostState { - fn ctx(&mut self) -> WasiCtxView<'_> { - WasiCtxView { - ctx: &mut self.wasi, - table: &mut self.table, - } - } -} - -fn unimplemented(domain: &str, detail: impl Into) -> HostError { - HostError { - domain: domain.into(), - kind: HostErrorKind::Unsupported, - code: 501, - message: detail.into(), - data: None, - } -} - -fn internal_error(domain: &str, detail: impl Into) -> HostError { - HostError { - domain: domain.into(), - kind: HostErrorKind::Internal, - code: 0, - message: detail.into(), - data: None, - } -} - -// -- nexum:host/types is empty (declarations only). -- - -impl nexum::host::types::Host for HostState {} - -// -- shepherd:cow/cow-api: REST passthrough + typed submission. -- - -impl shepherd::cow::cow_api::Host for HostState { - async fn request( - &mut self, - chain_id: u64, - method: String, - path: String, - body: Option, - ) -> Result { - let start = Instant::now(); - tracing::debug!(chain_id, %method, %path, "cow-api::request"); - let result = match self - .cow - .request(chain_id, &method, &path, body.as_deref()) - .await - { - Ok(body) => Ok(body), - Err(host::cow_orderbook::CowApiError::UnknownChain(id)) => Err(unimplemented( - "cow-api", - format!("chain {id} not in cowprotocol"), - )), - Err(host::cow_orderbook::CowApiError::BadMethod(m)) => Err(HostError { - domain: "cow-api".into(), - kind: HostErrorKind::InvalidInput, - code: 0, - message: format!("unsupported HTTP method: {m}"), - data: None, - }), - Err(host::cow_orderbook::CowApiError::BadPath(msg)) => Err(HostError { - domain: "cow-api".into(), - kind: HostErrorKind::InvalidInput, - code: 0, - message: msg, - data: None, - }), - Err(err) => Err(internal_error("cow-api", err.to_string())), - }; - tracing::trace!(elapsed_ms = ?start.elapsed(), "cow-api::request done"); - result - } - - async fn submit_order( - &mut self, - chain_id: u64, - order_data: Vec, - ) -> Result { - let start = Instant::now(); - tracing::debug!(chain_id, bytes = order_data.len(), "cow-api::submit-order"); - let result = match self.cow.submit_order_json(chain_id, &order_data).await { - Ok(uid) => Ok(format!("0x{}", hex_encode(uid.as_slice()))), - Err(host::cow_orderbook::CowApiError::UnknownChain(id)) => Err(unimplemented( - "cow-api", - format!("chain {id} not in cowprotocol"), - )), - Err(host::cow_orderbook::CowApiError::Decode(err)) => Err(HostError { - domain: "cow-api".into(), - kind: HostErrorKind::InvalidInput, - code: 0, - message: format!("invalid OrderCreation JSON: {err}"), - data: None, - }), - Err(host::cow_orderbook::CowApiError::Orderbook(err)) => Err(HostError { - domain: "cow-api".into(), - kind: HostErrorKind::Denied, - code: 0, - message: err.to_string(), - data: None, - }), - Err(err) => Err(internal_error("cow-api", err.to_string())), - }; - tracing::trace!(elapsed_ms = ?start.elapsed(), "cow-api::submit-order done"); - result - } -} - -// -- nexum:host/chain: raw JSON-RPC dispatch over alloy. -- - -impl nexum::host::chain::Host for HostState { - async fn request( - &mut self, - chain_id: u64, - method: String, - params: String, - ) -> Result { - let start = Instant::now(); - tracing::debug!(chain_id, %method, "chain::request"); - let result = match self.chain.request(chain_id, method.clone(), params).await { - Ok(body) => Ok(body), - Err(host::provider_pool::ProviderError::UnknownChain(id)) => Err(HostError { - domain: "chain".into(), - kind: HostErrorKind::Unsupported, - code: 0, - message: format!("chain {id} has no engine.toml RPC entry"), - data: None, - }), - Err(host::provider_pool::ProviderError::InvalidParams { detail, .. }) => { - Err(HostError { - domain: "chain".into(), - kind: HostErrorKind::InvalidInput, - code: -32602, - message: detail, - data: None, - }) - } - Err(host::provider_pool::ProviderError::Rpc { detail, .. }) => Err(HostError { - domain: "chain".into(), - kind: HostErrorKind::Internal, - code: -32603, - message: detail, - data: None, - }), - Err(err) => Err(internal_error("chain", err.to_string())), - }; - tracing::trace!(elapsed_ms = ?start.elapsed(), "chain::request done"); - result - } - - async fn request_batch( - &mut self, - chain_id: u64, - requests: Vec, - ) -> Result, HostError> { - let start = Instant::now(); - tracing::debug!(chain_id, count = requests.len(), "chain::request-batch"); - let mut out = Vec::with_capacity(requests.len()); - for req in requests { - match nexum::host::chain::Host::request(self, chain_id, req.method, req.params).await { - Ok(s) => out.push(nexum::host::chain::RpcResult::Ok(s)), - Err(e) => out.push(nexum::host::chain::RpcResult::Err(e)), - } - } - tracing::trace!(elapsed_ms = ?start.elapsed(), "chain::request-batch done"); - Ok(out) - } -} +use wasmtime::component::Linker; -// -- nexum:host/identity: deferred to 0.3 (keystore/KMS backend). -- - -impl nexum::host::identity::Host for HostState { - async fn accounts(&mut self) -> Result>, HostError> { - // No keystore wired yet - return an empty roster so guests can - // probe-then-skip without erroring. Real keystore lands in 0.3. - Ok(vec![]) - } - - async fn sign(&mut self, _account: Vec, _message: Vec) -> Result, HostError> { - Err(unimplemented("identity", "sign requires a keystore (0.3)")) - } - - async fn sign_typed_data( - &mut self, - _account: Vec, - _typed_data: String, - ) -> Result, HostError> { - Err(unimplemented( - "identity", - "sign-typed-data requires a keystore (0.3)", - )) - } -} - -// -- nexum:host/local-store: redb backend with host-side namespacing. -- - -impl nexum::host::local_store::Host for HostState { - async fn get(&mut self, key: String) -> Result>, HostError> { - self.store - .get(&self.module_namespace, &key) - .map_err(|err| internal_error("local-store", err.to_string())) - } - - async fn set(&mut self, key: String, value: Vec) -> Result<(), HostError> { - self.store - .set(&self.module_namespace, &key, &value) - .map_err(|err| internal_error("local-store", err.to_string())) - } - - async fn delete(&mut self, key: String) -> Result<(), HostError> { - self.store - .delete(&self.module_namespace, &key) - .map_err(|err| internal_error("local-store", err.to_string())) - } - - async fn list_keys(&mut self, prefix: String) -> Result, HostError> { - self.store - .list_keys(&self.module_namespace, &prefix) - .map_err(|err| internal_error("local-store", err.to_string())) - } -} - -impl nexum::host::remote_store::Host for HostState { - async fn upload(&mut self, _data: Vec) -> Result, HostError> { - Err(unimplemented( - "remote-store", - "Swarm backend deferred to 0.3", - )) - } - - async fn download(&mut self, _reference: Vec) -> Result, HostError> { - Err(unimplemented( - "remote-store", - "Swarm backend deferred to 0.3", - )) - } - - async fn read_feed( - &mut self, - _owner: Vec, - _topic: Vec, - ) -> Result>, HostError> { - Err(unimplemented( - "remote-store", - "Swarm backend deferred to 0.3", - )) - } - - async fn write_feed(&mut self, _topic: Vec, _data: Vec) -> Result, HostError> { - Err(unimplemented( - "remote-store", - "Swarm backend deferred to 0.3", - )) - } -} - -impl nexum::host::messaging::Host for HostState { - async fn publish( - &mut self, - _content_topic: String, - _payload: Vec, - ) -> Result<(), HostError> { - Err(unimplemented("messaging", "Waku backend deferred to 0.3")) - } - - async fn query( - &mut self, - _content_topic: String, - _start_time: Option, - _end_time: Option, - _limit: Option, - ) -> Result, HostError> { - // Empty result - same posture as `identity::accounts`. - Ok(vec![]) - } -} - -impl nexum::host::logging::Host for HostState { - async fn log(&mut self, level: nexum::host::logging::Level, message: String) { - let module = self.module_namespace.as_str(); - match level { - nexum::host::logging::Level::Trace => tracing::trace!(module, "{}", message), - nexum::host::logging::Level::Debug => tracing::debug!(module, "{}", message), - nexum::host::logging::Level::Info => tracing::info!(module, "{}", message), - nexum::host::logging::Level::Warn => tracing::warn!(module, "{}", message), - nexum::host::logging::Level::Error => tracing::error!(module, "{}", message), - } - } -} - -// -- Additive 0.2 capabilities -- - -impl nexum::host::clock::Host for HostState { - async fn now_ms(&mut self) -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) - } - - async fn monotonic_ns(&mut self) -> u64 { - self.monotonic_baseline.elapsed().as_nanos() as u64 - } -} - -impl nexum::host::random::Host for HostState { - async fn fill(&mut self, len: u32) -> Vec { - let mut buf = vec![0u8; len as usize]; - // getrandom 0.4: fill() returns Result<(), Error>. CSPRNG failures - // are exceptionally rare on supported platforms; on failure we - // return zero-filled bytes - guests that need a strong-failure - // signal should use identity or chain primitives instead. - let _ = getrandom::fill(&mut buf); - buf - } -} - -impl nexum::host::http::Host for HostState { - async fn fetch( - &mut self, - req: nexum::host::http::Request, - ) -> Result { - // Manifest allowlist enforcement runs before any I/O. Hosts that - // never link a manifest leave `http_allowlist` empty, which denies - // every request - matching the "no implicit network" stance. - let host = match manifest::extract_host(&req.url) { - Some(h) => h, - None => { - return Err(HostError { - domain: "http".into(), - kind: HostErrorKind::InvalidInput, - code: 0, - message: format!("not an http(s) URL: {}", req.url), - data: None, - }); - } - }; - if !manifest::host_allowed(host, &self.http_allowlist) { - warn!(host, "[http] denied by allowlist"); - return Err(HostError { - domain: "http".into(), - kind: HostErrorKind::Denied, - code: 0, - message: format!( - "host {host} not in [capabilities.http].allow; \ - add it to module.toml to permit" - ), - data: None, - }); - } - // 0.2: allowlist passed, but the reference runtime does not perform - // real HTTP yet. Real fetch lands in 0.3. - Err(unimplemented( - "http", - "fetch not implemented in 0.2 reference runtime (allowlist passed)", - )) - } -} - -/// Lowercase hex encoder. Kept in the engine binary rather than -/// pulling a `hex` crate just for one call site. Writes into the -/// pre-allocated buffer to avoid the per-byte `String` allocation -/// `format!("{b:02x}")` would do. -fn hex_encode(bytes: &[u8]) -> String { - use std::fmt::Write as _; - let mut s = String::with_capacity(bytes.len() * 2); - for b in bytes { - write!(s, "{b:02x}").expect("writing to String never fails"); - } - s -} +use crate::bindings::Shepherd; +use crate::cli::Cli; +use crate::host::state::HostState; #[tokio::main] async fn main() -> anyhow::Result<()> { - // CLI surface (clap-derived, see `Cli`): - // nexum-engine [ []] [--engine-config ] - // - // Positional `` is a backwards-compat shortcut that - // synthesises a one-module engine config. Production deployments - // pass `--engine-config` and declare modules in TOML. let cli = Cli::parse(); let engine_cfg = engine_config::load_or_default(cli.engine_config.as_deref())?; @@ -500,19 +43,17 @@ async fn main() -> anyhow::Result<()> { info!("nexum-engine starting"); // Bring up shared host backends. - std::fs::create_dir_all(&engine_cfg.engine.state_dir).with_context(|| { - format!( - "create state directory {}", + std::fs::create_dir_all(&engine_cfg.engine.state_dir).map_err(|e| { + anyhow::anyhow!( + "create state directory {}: {e}", engine_cfg.engine.state_dir.display() ) })?; let store_path = engine_cfg.engine.state_dir.join("local-store.redb"); let local_store = host::local_store_redb::LocalStore::open(&store_path) - .with_context(|| format!("open local-store at {}", store_path.display()))?; + .map_err(|e| anyhow::anyhow!("open local-store at {}: {e}", store_path.display()))?; let cow_pool = host::cow_orderbook::OrderBookPool::default(); - let provider_pool = host::provider_pool::ProviderPool::from_config(&engine_cfg) - .await - .context("open chain providers")?; + let provider_pool = host::provider_pool::ProviderPool::from_config(&engine_cfg).await?; // wasmtime engine + linker - one of each, shared across modules. let mut config = wasmtime::Config::new(); @@ -527,8 +68,7 @@ async fn main() -> anyhow::Result<()> { )?; wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; - // Boot supervisor - `engine.toml.[[modules]]` first, CLI - // positional second. + // Boot supervisor - `engine.toml.[[modules]]` first, CLI positional second. let mut supervisor = if let Some(wasm) = cli.wasm.as_deref() { if !engine_cfg.modules.is_empty() { warn!("ignoring engine.toml [[modules]] because a positional was given"); @@ -576,164 +116,17 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } - let block_streams = open_block_streams(&provider_pool, &block_chains).await; - let log_streams = open_log_streams(&provider_pool, log_subs).await; + let block_streams = runtime::event_loop::open_block_streams(&provider_pool, &block_chains).await; + let log_streams = runtime::event_loop::open_log_streams(&provider_pool, log_subs).await; let shutdown = async { - match wait_for_shutdown_signal().await { + match runtime::event_loop::wait_for_shutdown_signal().await { Ok(name) => info!(signal = %name, "shutdown signal received"), Err(err) => warn!(error = %err, "signal handler failed - using ctrl-c"), } }; - run_event_loop(&mut supervisor, block_streams, log_streams, shutdown).await; + runtime::event_loop::run(&mut supervisor, block_streams, log_streams, shutdown).await; info!("done"); Ok(()) } - -/// Per-chain block subscriptions, one shared stream per chain id. -async fn open_block_streams( - pool: &host::provider_pool::ProviderPool, - chains: &std::collections::BTreeSet, -) -> Vec { - let mut openings: FuturesUnordered<_> = chains - .iter() - .copied() - .map(|chain_id| async move { (chain_id, pool.subscribe_blocks(chain_id).await) }) - .collect(); - - let mut streams = Vec::new(); - while let Some((chain_id, result)) = openings.next().await { - match result { - Ok(stream) => { - info!(chain_id, "block subscription open"); - let tagged: TaggedBlockStream = Box::pin(stream.map(move |item| { - item.map(|header| (chain_id, header)) - .map_err(anyhow::Error::from) - })); - streams.push(tagged); - } - Err(err) => { - warn!(chain_id, error = %err, "block subscription failed"); - } - } - } - streams -} - -/// Per-module log subscriptions. Each entry is a stream tagged with -/// the owning module name + chain id. -async fn open_log_streams( - pool: &host::provider_pool::ProviderPool, - subs: Vec<(String, u64, alloy_rpc_types_eth::Filter)>, -) -> Vec { - let mut openings: FuturesUnordered<_> = subs - .into_iter() - .map(|(module, chain_id, filter)| async move { - let stream = pool.subscribe_logs(chain_id, filter).await; - (module, chain_id, stream) - }) - .collect(); - - let mut streams = Vec::new(); - while let Some((module, chain_id, result)) = openings.next().await { - match result { - Ok(stream) => { - info!(module = %module, chain_id, "log subscription open"); - let module_name = module.clone(); - let tagged: TaggedLogStream = Box::pin(stream.map(move |item| { - item.map(|log| (module_name.clone(), chain_id, log)) - .map_err(anyhow::Error::from) - })); - streams.push(tagged); - } - Err(err) => { - warn!(module = %module, chain_id, error = %err, "log subscription failed"); - } - } - } - streams -} - -type TaggedBlockStream = std::pin::Pin< - Box< - dyn futures::Stream> - + Send, - >, ->; -type TaggedLogStream = std::pin::Pin< - Box< - dyn futures::Stream> - + Send, - >, ->; - -/// Drive the supervisor with events until `shutdown` resolves. -async fn run_event_loop( - supervisor: &mut supervisor::Supervisor, - block_streams: Vec, - log_streams: Vec, - shutdown: impl std::future::Future + Send, -) { - let mut blocks = select_all(block_streams); - let mut logs = select_all(log_streams); - let mut shutdown = Box::pin(shutdown); - loop { - tokio::select! { - biased; - () = &mut shutdown => return, - next = blocks.next() => match next { - Some(Ok((chain_id, header))) => { - let block = nexum::host::types::Block { - chain_id, - number: header.number, - hash: header.hash.as_slice().to_vec(), - timestamp: header.timestamp.saturating_mul(1000), - }; - supervisor.dispatch_block(block).await; - } - Some(Err(err)) => warn!(error = %err, "block stream error - continuing"), - None => { - // alloy ends the stream with None when the - // WebSocket drops. Without this branch the loop - // keeps polling a dead stream and the operator - // sees no events with no indication anything is - // wrong. Bail out so the supervisor (or whatever - // wraps the engine) restarts us; a reconnect- - // with-backoff is the 0.3 fix. - warn!("block stream ended (WebSocket dropped?) - shutting down for restart"); - return; - } - }, - next = logs.next() => match next { - Some(Ok((module, chain_id, log))) => { - supervisor.dispatch_log(&module, chain_id, log).await; - } - Some(Err(err)) => warn!(error = %err, "log stream error - continuing"), - None => { - warn!("log stream ended (WebSocket dropped?) - shutting down for restart"); - return; - } - }, - } - } -} - -/// Wait for SIGINT or (on Unix) SIGTERM, whichever arrives first. -async fn wait_for_shutdown_signal() -> anyhow::Result<&'static str> { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - let mut sigterm = signal(SignalKind::terminate())?; - let mut sigint = signal(SignalKind::interrupt())?; - tokio::select! { - _ = sigterm.recv() => Ok("SIGTERM"), - _ = sigint.recv() => Ok("SIGINT"), - } - } - #[cfg(not(unix))] - { - tokio::signal::ctrl_c().await?; - Ok("ctrl-c") - } -} diff --git a/crates/nexum-engine/src/runtime/event_loop.rs b/crates/nexum-engine/src/runtime/event_loop.rs new file mode 100644 index 0000000..94c7433 --- /dev/null +++ b/crates/nexum-engine/src/runtime/event_loop.rs @@ -0,0 +1,157 @@ +//! Open live `eth_subscribe` streams and dispatch their events to the +//! supervisor until a shutdown signal arrives. + +use futures::StreamExt; +use futures::stream::{FuturesUnordered, select_all}; +use tracing::{info, warn}; + +use crate::bindings::nexum; +use crate::host::provider_pool::ProviderPool; +use crate::supervisor::Supervisor; + +/// Per-chain block subscriptions, one shared stream per chain id. +pub async fn open_block_streams( + pool: &ProviderPool, + chains: &std::collections::BTreeSet, +) -> Vec { + let mut openings: FuturesUnordered<_> = chains + .iter() + .copied() + .map(|chain_id| async move { (chain_id, pool.subscribe_blocks(chain_id).await) }) + .collect(); + + let mut streams = Vec::new(); + while let Some((chain_id, result)) = openings.next().await { + match result { + Ok(stream) => { + info!(chain_id, "block subscription open"); + let tagged: TaggedBlockStream = Box::pin(stream.map(move |item| { + item.map(|header| (chain_id, header)) + .map_err(anyhow::Error::from) + })); + streams.push(tagged); + } + Err(err) => { + warn!(chain_id, error = %err, "block subscription failed"); + } + } + } + streams +} + +/// Per-module log subscriptions. Each entry is a stream tagged with +/// the owning module name + chain id. +pub async fn open_log_streams( + pool: &ProviderPool, + subs: Vec<(String, u64, alloy_rpc_types_eth::Filter)>, +) -> Vec { + let mut openings: FuturesUnordered<_> = subs + .into_iter() + .map(|(module, chain_id, filter)| async move { + let stream = pool.subscribe_logs(chain_id, filter).await; + (module, chain_id, stream) + }) + .collect(); + + let mut streams = Vec::new(); + while let Some((module, chain_id, result)) = openings.next().await { + match result { + Ok(stream) => { + info!(module = %module, chain_id, "log subscription open"); + let module_name = module.clone(); + let tagged: TaggedLogStream = Box::pin(stream.map(move |item| { + item.map(|log| (module_name.clone(), chain_id, log)) + .map_err(anyhow::Error::from) + })); + streams.push(tagged); + } + Err(err) => { + warn!(module = %module, chain_id, error = %err, "log subscription failed"); + } + } + } + streams +} + +pub type TaggedBlockStream = std::pin::Pin< + Box< + dyn futures::Stream> + + Send, + >, +>; +pub type TaggedLogStream = std::pin::Pin< + Box< + dyn futures::Stream> + + Send, + >, +>; + +/// Drive the supervisor with events until `shutdown` resolves. +pub async fn run( + supervisor: &mut Supervisor, + block_streams: Vec, + log_streams: Vec, + shutdown: impl std::future::Future + Send, +) { + let mut blocks = select_all(block_streams); + let mut logs = select_all(log_streams); + let mut shutdown = Box::pin(shutdown); + loop { + tokio::select! { + biased; + () = &mut shutdown => return, + next = blocks.next() => match next { + Some(Ok((chain_id, header))) => { + let block = nexum::host::types::Block { + chain_id, + number: header.number, + hash: header.hash.as_slice().to_vec(), + timestamp: header.timestamp.saturating_mul(1000), + }; + supervisor.dispatch_block(block).await; + } + Some(Err(err)) => warn!(error = %err, "block stream error - continuing"), + None => { + // alloy ends the stream with None when the + // WebSocket drops. Without this branch the loop + // keeps polling a dead stream and the operator + // sees no events with no indication anything is + // wrong. Bail out so the supervisor (or whatever + // wraps the engine) restarts us; a reconnect- + // with-backoff is the 0.3 fix. + warn!("block stream ended (WebSocket dropped?) - shutting down for restart"); + return; + } + }, + next = logs.next() => match next { + Some(Ok((module, chain_id, log))) => { + supervisor.dispatch_log(&module, chain_id, log).await; + } + Some(Err(err)) => warn!(error = %err, "log stream error - continuing"), + None => { + warn!("log stream ended (WebSocket dropped?) - shutting down for restart"); + return; + } + }, + } + } +} + +/// Wait for SIGINT or (on Unix) SIGTERM, whichever arrives first. +pub async fn wait_for_shutdown_signal() -> anyhow::Result<&'static str> { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut sigterm = signal(SignalKind::terminate())?; + let mut sigint = signal(SignalKind::interrupt())?; + tokio::select! { + _ = sigterm.recv() => Ok("SIGTERM"), + _ = sigint.recv() => Ok("SIGINT"), + } + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await?; + Ok("ctrl-c") + } +} diff --git a/crates/nexum-engine/src/runtime/limits.rs b/crates/nexum-engine/src/runtime/limits.rs new file mode 100644 index 0000000..1f4b1d9 --- /dev/null +++ b/crates/nexum-engine/src/runtime/limits.rs @@ -0,0 +1,14 @@ +//! Per-module wasmtime fuel + memory limits. The supervisor refuels +//! the store before every `on_event` so each invocation gets a fresh +//! budget; a module that exhausts fuel traps with `OutOfFuel` and is +//! marked dead. + +/// Default fuel budget granted per `on_event` invocation +/// (~ 1 billion WASM instructions). Configurable per-module via +/// `engine.toml` in 0.3. +pub const DEFAULT_FUEL_PER_EVENT: u64 = 1_000_000_000; + +/// Default linear-memory cap per module store (64 MiB). Prevents a +/// single runaway module from exhausting process memory. Configurable +/// in 0.3. +pub const DEFAULT_MEMORY_LIMIT: usize = 64 * 1024 * 1024; diff --git a/crates/nexum-engine/src/runtime/mod.rs b/crates/nexum-engine/src/runtime/mod.rs new file mode 100644 index 0000000..72ea95f --- /dev/null +++ b/crates/nexum-engine/src/runtime/mod.rs @@ -0,0 +1,5 @@ +//! Engine-side runtime: per-module resource limits and the event loop +//! that drives the supervisor from live chain subscriptions. + +pub mod event_loop; +pub mod limits; diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 631696e..88f3615 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -20,12 +20,14 @@ use wasmtime::component::{Component, Linker, ResourceTable}; use wasmtime::{Engine, Store}; use wasmtime_wasi::WasiCtxBuilder; +use crate::bindings::{Config, Shepherd, nexum}; use crate::engine_config::{EngineConfig, ModuleEntry}; use crate::host::cow_orderbook::OrderBookPool; use crate::host::local_store_redb::LocalStore; use crate::host::provider_pool::ProviderPool; +use crate::host::state::HostState; use crate::manifest::{self, LoadedManifest, Subscription}; -use crate::{HostState, Shepherd}; +use crate::runtime::limits::{DEFAULT_FUEL_PER_EVENT, DEFAULT_MEMORY_LIMIT}; /// Owns every loaded module and exposes the dispatch surface the /// event loop needs. @@ -155,7 +157,7 @@ impl Supervisor { loaded_manifest.manifest.module.name.clone() }; let limits = wasmtime::StoreLimitsBuilder::new() - .memory_size(crate::DEFAULT_MEMORY_LIMIT) + .memory_size(DEFAULT_MEMORY_LIMIT) .build(); let mut store = Store::new( engine, @@ -172,14 +174,14 @@ impl Supervisor { }, ); store.limiter(|state| &mut state.limits); - store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT)?; + store.set_fuel(DEFAULT_FUEL_PER_EVENT)?; let bindings = Shepherd::instantiate_async(&mut store, &component, linker) .await .map_err(Error::from) .with_context(|| format!("instantiate {}", entry.path.display()))?; // Call `init` with the manifest's `[config]`. - let config: crate::Config = if loaded_manifest.config.is_empty() { + let config: Config = if loaded_manifest.config.is_empty() { vec![("name".into(), module_namespace.clone())] } else { loaded_manifest.config.clone() @@ -200,7 +202,7 @@ impl Supervisor { ), } // Refuel after init so the first on_event starts with a full budget. - store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT)?; + store.set_fuel(DEFAULT_FUEL_PER_EVENT)?; // Surface any `[[subscription]]` entries the host cannot // service yet, so an operator running 0.2 against a 0.3 @@ -275,9 +277,9 @@ impl Supervisor { /// Dispatch a block event to every module subscribed to /// `block.chain_id`. Returns the number of modules invoked. /// Modules that trap are marked dead and excluded from future dispatch. - pub async fn dispatch_block(&mut self, block: crate::nexum::host::types::Block) -> usize { + pub async fn dispatch_block(&mut self, block: nexum::host::types::Block) -> usize { let chain_id = block.chain_id; - let event = crate::nexum::host::types::Event::Block(block); + let event = nexum::host::types::Event::Block(block); let mut dispatched = 0; for module in &mut self.modules { if !module.alive { @@ -291,7 +293,7 @@ impl Supervisor { continue; } // Refuel before each invocation so each event gets a fresh budget. - if let Err(e) = module.store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT) { + if let Err(e) = module.store.set_fuel(DEFAULT_FUEL_PER_EVENT) { error!(module = %module.name, error = %e, "set_fuel failed - skipping"); continue; } @@ -343,11 +345,11 @@ impl Supervisor { if !target.alive { return false; } - if let Err(e) = target.store.set_fuel(crate::DEFAULT_FUEL_PER_EVENT) { + if let Err(e) = target.store.set_fuel(DEFAULT_FUEL_PER_EVENT) { error!(module = %module_name, error = %e, "set_fuel failed - skipping"); return false; } - let event = crate::nexum::host::types::Event::Logs(vec![project_log(chain_id, &log)]); + let event = nexum::host::types::Event::Logs(vec![project_log(chain_id, &log)]); match target .bindings .call_on_event(&mut target.store, &event) @@ -388,8 +390,8 @@ impl Supervisor { /// Project an alloy `Log` onto the WIT `log` record. The chain id /// is not on the alloy log (the subscription context carries it), /// so we receive it alongside. -fn project_log(chain_id: u64, log: &alloy_rpc_types_eth::Log) -> crate::nexum::host::types::Log { - crate::nexum::host::types::Log { +fn project_log(chain_id: u64, log: &alloy_rpc_types_eth::Log) -> nexum::host::types::Log { + nexum::host::types::Log { chain_id, address: log.address().as_slice().to_vec(), topics: log.topics().iter().map(|t| t.as_slice().to_vec()).collect(), @@ -580,7 +582,7 @@ chain_id = 1 .await .expect("boot_single"); - let block = crate::nexum::host::types::Block { + let block = nexum::host::types::Block { chain_id: 1, number: 19_000_000, hash: vec![0xab; 32], From 615b90a5c4fa66d2c47483fa9c19edf43aee2be4 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Sat, 13 Jun 2026 09:39:59 -0300 Subject: [PATCH 16/29] refactor: move large #[cfg(test)] modules to sibling files local_store_redb.rs was 89% tests, cow_orderbook.rs was 60%, and supervisor.rs was 32% (205 lines absolute). Promote each to a directory module with the test suite living in a sibling tests.rs so impl-side diffs stop competing with test churn for attention. --- crates/nexum-engine/src/host/cow_orderbook.rs | 214 +----------------- .../src/host/cow_orderbook/tests.rs | 208 +++++++++++++++++ .../nexum-engine/src/host/local_store_redb.rs | 83 +------ .../src/host/local_store_redb/tests.rs | 80 +++++++ crates/nexum-engine/src/supervisor.rs | 206 +---------------- crates/nexum-engine/src/supervisor/tests.rs | 202 +++++++++++++++++ 6 files changed, 495 insertions(+), 498 deletions(-) create mode 100644 crates/nexum-engine/src/host/cow_orderbook/tests.rs create mode 100644 crates/nexum-engine/src/host/local_store_redb/tests.rs create mode 100644 crates/nexum-engine/src/supervisor/tests.rs diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index bac376d..49c1945 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -136,216 +136,6 @@ pub enum CowApiError { Orderbook(#[from] cowprotocol::Error), } -#[cfg(test)] -mod tests { - use super::*; - use wiremock::matchers::{method, path}; - use wiremock::{Mock, MockServer, ResponseTemplate}; - - #[test] - fn pool_indexes_default_chains() { - let pool = OrderBookPool::default(); - assert!(pool.get(1).is_ok(), "mainnet present"); - assert!(pool.get(100).is_ok(), "gnosis present"); - assert!(pool.get(11_155_111).is_ok(), "sepolia present"); - assert!(pool.get(42_161).is_ok(), "arbitrum present"); - assert!(pool.get(8_453).is_ok(), "base present"); - } - - #[test] - fn unknown_chain_surfaces_typed_error() { - let pool = OrderBookPool::default(); - assert!(matches!( - pool.get(99_999), - Err(CowApiError::UnknownChain(99_999)) - )); - } - - /// Build a pool whose Mainnet entry points at `mock.uri()`. - /// `OrderBookApi::new_with_base_url` ships in cowprotocol; we - /// rely on it so wiremock-driven tests can exercise the full - /// request path without re-implementing the HTTP client. - fn pool_with_mainnet_at(mock: &MockServer) -> OrderBookPool { - let mut clients = std::collections::BTreeMap::new(); - clients.insert( - Chain::Mainnet.id(), - OrderBookApi::new_with_base_url(mock.uri().parse().expect("mock uri parses")), - ); - OrderBookPool { - clients, - http: reqwest::Client::new(), - } - } - - #[tokio::test] - async fn request_passes_get_path_through() { - let mock = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/v1/version")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"version":"x.y.z"}"#)) - .expect(1) - .mount(&mock) - .await; - - let pool = pool_with_mainnet_at(&mock); - let body = pool - .request(Chain::Mainnet.id(), "GET", "/api/v1/version", None) - .await - .expect("request succeeds"); - assert_eq!(body, r#"{"version":"x.y.z"}"#); - } - - #[tokio::test] - async fn request_relative_path_works() { - // Module passes a path without a leading slash. The - // passthrough should still resolve against the orderbook - // base URL. - let mock = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/v1/native_price/0xabc")) - .respond_with(ResponseTemplate::new(200).set_body_string("1.23")) - .expect(1) - .mount(&mock) - .await; - - let pool = pool_with_mainnet_at(&mock); - let body = pool - .request( - Chain::Mainnet.id(), - "GET", - "api/v1/native_price/0xabc", - None, - ) - .await - .expect("relative path resolves"); - assert_eq!(body, "1.23"); - } - - #[tokio::test] - async fn request_rejects_unknown_method() { - let pool = OrderBookPool::default(); - let err = pool - .request(Chain::Mainnet.id(), "PATCH", "/x", None) - .await - .unwrap_err(); - assert!(matches!(err, CowApiError::BadMethod(_))); - } - - #[tokio::test] - async fn request_post_with_body_is_forwarded() { - let mock = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/api/v1/quote")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"quote":"ok"}"#)) - .expect(1) - .mount(&mock) - .await; - let pool = pool_with_mainnet_at(&mock); - let body = pool - .request( - Chain::Mainnet.id(), - "POST", - "/api/v1/quote", - Some(r#"{"sellToken":"0x01"}"#), - ) - .await - .expect("post with body succeeds"); - assert_eq!(body, r#"{"quote":"ok"}"#); - } - - #[tokio::test] - async fn request_4xx_response_is_returned_verbatim() { - // The host must NOT surface a 4xx as an error - the module - // needs the structured JSON body to decode `OrderPostError`. - let mock = MockServer::start().await; - let error_body = r#"{"errorType":"InsufficientFee","description":"fee too low"}"#; - Mock::given(method("POST")) - .and(path("/api/v1/orders")) - .respond_with( - ResponseTemplate::new(400).set_body_string(error_body), - ) - .expect(1) - .mount(&mock) - .await; - - let pool = pool_with_mainnet_at(&mock); - let body = pool - .request( - Chain::Mainnet.id(), - "POST", - "/api/v1/orders", - Some(r#"{"test":true}"#), - ) - .await - .expect("4xx body is returned, not an Err"); - assert_eq!(body, error_body); - } - - #[tokio::test] - async fn request_rejects_unknown_chain() { - let pool = OrderBookPool::default(); - let err = pool.request(99_999, "GET", "/x", None).await.unwrap_err(); - assert!(matches!(err, CowApiError::UnknownChain(99_999))); - } - - #[tokio::test] - async fn submit_order_propagates_orderbook_response() { - let mock = MockServer::start().await; - let body_json = sample_order_json(); - // cowprotocol POST /api/v1/orders returns the order UID - // (56-byte hex) as a JSON string body. - let returned_uid = format!("\"0x{}\"", "ab".repeat(56)); - Mock::given(method("POST")) - .and(path("/api/v1/orders")) - .respond_with(ResponseTemplate::new(201).set_body_string(returned_uid.clone())) - .expect(1) - .mount(&mock) - .await; - - let pool = pool_with_mainnet_at(&mock); - let uid = pool - .submit_order_json(Chain::Mainnet.id(), body_json.as_bytes()) - .await - .expect("submit succeeds"); - assert_eq!(uid.as_slice().len(), 56); - assert_eq!(uid.as_slice(), &[0xab; 56]); - } - - /// A minimal but accepted-by-cowprotocol OrderCreation JSON. We - /// generate it inside the test so the JSON shape stays in lockstep - /// with the published `cowprotocol` version. - fn sample_order_json() -> String { - use alloy_primitives::{Address, U256}; - use cowprotocol::OrderCreation; - use cowprotocol::app_data::{EMPTY_APP_DATA_HASH, EMPTY_APP_DATA_JSON}; - use cowprotocol::order::{BuyTokenDestination, OrderData, OrderKind, SellTokenSource}; - use cowprotocol::signature::Signature; - use cowprotocol::signing_scheme::SigningScheme; - - let order_data = OrderData { - sell_token: Address::from([0x01; 20]), - buy_token: Address::from([0x02; 20]), - receiver: None, - sell_amount: U256::from(100u64), - buy_amount: U256::from(99u64), - valid_to: u32::MAX, - app_data: EMPTY_APP_DATA_HASH, - fee_amount: U256::ZERO, - kind: OrderKind::Sell, - partially_fillable: false, - sell_token_balance: SellTokenSource::Erc20, - buy_token_balance: BuyTokenDestination::Erc20, - }; - let signature = Signature::from_bytes(SigningScheme::PreSign, &[]).expect("presign empty"); - let creation = OrderCreation::from_signed_order_data( - &order_data, - signature, - Address::from([0x03; 20]), - EMPTY_APP_DATA_JSON.to_owned(), - None, - ) - .expect("valid OrderCreation"); - serde_json::to_string(&creation).expect("serialise OrderCreation") - } -} +#[cfg(test)] +mod tests; diff --git a/crates/nexum-engine/src/host/cow_orderbook/tests.rs b/crates/nexum-engine/src/host/cow_orderbook/tests.rs new file mode 100644 index 0000000..ef318c9 --- /dev/null +++ b/crates/nexum-engine/src/host/cow_orderbook/tests.rs @@ -0,0 +1,208 @@ +use super::*; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[test] +fn pool_indexes_default_chains() { + let pool = OrderBookPool::default(); + assert!(pool.get(1).is_ok(), "mainnet present"); + assert!(pool.get(100).is_ok(), "gnosis present"); + assert!(pool.get(11_155_111).is_ok(), "sepolia present"); + assert!(pool.get(42_161).is_ok(), "arbitrum present"); + assert!(pool.get(8_453).is_ok(), "base present"); +} + +#[test] +fn unknown_chain_surfaces_typed_error() { + let pool = OrderBookPool::default(); + assert!(matches!( + pool.get(99_999), + Err(CowApiError::UnknownChain(99_999)) + )); +} + +/// Build a pool whose Mainnet entry points at `mock.uri()`. +/// `OrderBookApi::new_with_base_url` ships in cowprotocol; we +/// rely on it so wiremock-driven tests can exercise the full +/// request path without re-implementing the HTTP client. +fn pool_with_mainnet_at(mock: &MockServer) -> OrderBookPool { + let mut clients = std::collections::BTreeMap::new(); + clients.insert( + Chain::Mainnet.id(), + OrderBookApi::new_with_base_url(mock.uri().parse().expect("mock uri parses")), + ); + OrderBookPool { + clients, + http: reqwest::Client::new(), + } +} + +#[tokio::test] +async fn request_passes_get_path_through() { + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v1/version")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"version":"x.y.z"}"#)) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request(Chain::Mainnet.id(), "GET", "/api/v1/version", None) + .await + .expect("request succeeds"); + assert_eq!(body, r#"{"version":"x.y.z"}"#); +} + +#[tokio::test] +async fn request_relative_path_works() { + // Module passes a path without a leading slash. The + // passthrough should still resolve against the orderbook + // base URL. + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v1/native_price/0xabc")) + .respond_with(ResponseTemplate::new(200).set_body_string("1.23")) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request( + Chain::Mainnet.id(), + "GET", + "api/v1/native_price/0xabc", + None, + ) + .await + .expect("relative path resolves"); + assert_eq!(body, "1.23"); +} + +#[tokio::test] +async fn request_rejects_unknown_method() { + let pool = OrderBookPool::default(); + let err = pool + .request(Chain::Mainnet.id(), "PATCH", "/x", None) + .await + .unwrap_err(); + assert!(matches!(err, CowApiError::BadMethod(_))); +} + +#[tokio::test] +async fn request_post_with_body_is_forwarded() { + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v1/quote")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"quote":"ok"}"#)) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request( + Chain::Mainnet.id(), + "POST", + "/api/v1/quote", + Some(r#"{"sellToken":"0x01"}"#), + ) + .await + .expect("post with body succeeds"); + assert_eq!(body, r#"{"quote":"ok"}"#); +} + +#[tokio::test] +async fn request_4xx_response_is_returned_verbatim() { + // The host must NOT surface a 4xx as an error - the module + // needs the structured JSON body to decode `OrderPostError`. + let mock = MockServer::start().await; + let error_body = r#"{"errorType":"InsufficientFee","description":"fee too low"}"#; + Mock::given(method("POST")) + .and(path("/api/v1/orders")) + .respond_with(ResponseTemplate::new(400).set_body_string(error_body)) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request( + Chain::Mainnet.id(), + "POST", + "/api/v1/orders", + Some(r#"{"test":true}"#), + ) + .await + .expect("4xx body is returned, not an Err"); + assert_eq!(body, error_body); +} + +#[tokio::test] +async fn request_rejects_unknown_chain() { + let pool = OrderBookPool::default(); + let err = pool.request(99_999, "GET", "/x", None).await.unwrap_err(); + assert!(matches!(err, CowApiError::UnknownChain(99_999))); +} + +#[tokio::test] +async fn submit_order_propagates_orderbook_response() { + let mock = MockServer::start().await; + let body_json = sample_order_json(); + // cowprotocol POST /api/v1/orders returns the order UID + // (56-byte hex) as a JSON string body. + let returned_uid = format!("\"0x{}\"", "ab".repeat(56)); + Mock::given(method("POST")) + .and(path("/api/v1/orders")) + .respond_with(ResponseTemplate::new(201).set_body_string(returned_uid.clone())) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let uid = pool + .submit_order_json(Chain::Mainnet.id(), body_json.as_bytes()) + .await + .expect("submit succeeds"); + assert_eq!(uid.as_slice().len(), 56); + assert_eq!(uid.as_slice(), &[0xab; 56]); +} + +/// A minimal but accepted-by-cowprotocol OrderCreation JSON. We +/// generate it inside the test so the JSON shape stays in lockstep +/// with the published `cowprotocol` version. +fn sample_order_json() -> String { + use alloy_primitives::{Address, U256}; + use cowprotocol::OrderCreation; + use cowprotocol::app_data::{EMPTY_APP_DATA_HASH, EMPTY_APP_DATA_JSON}; + use cowprotocol::order::{BuyTokenDestination, OrderData, OrderKind, SellTokenSource}; + use cowprotocol::signature::Signature; + use cowprotocol::signing_scheme::SigningScheme; + + let order_data = OrderData { + sell_token: Address::from([0x01; 20]), + buy_token: Address::from([0x02; 20]), + receiver: None, + sell_amount: U256::from(100u64), + buy_amount: U256::from(99u64), + valid_to: u32::MAX, + app_data: EMPTY_APP_DATA_HASH, + fee_amount: U256::ZERO, + kind: OrderKind::Sell, + partially_fillable: false, + sell_token_balance: SellTokenSource::Erc20, + buy_token_balance: BuyTokenDestination::Erc20, + }; + let signature = Signature::from_bytes(SigningScheme::PreSign, &[]).expect("presign empty"); + let creation = OrderCreation::from_signed_order_data( + &order_data, + signature, + Address::from([0x03; 20]), + EMPTY_APP_DATA_JSON.to_owned(), + None, + ) + .expect("valid OrderCreation"); + serde_json::to_string(&creation).expect("serialise OrderCreation") +} diff --git a/crates/nexum-engine/src/host/local_store_redb.rs b/crates/nexum-engine/src/host/local_store_redb.rs index 96f02eb..c832137 100644 --- a/crates/nexum-engine/src/host/local_store_redb.rs +++ b/crates/nexum-engine/src/host/local_store_redb.rs @@ -155,85 +155,4 @@ pub enum StorageError { } #[cfg(test)] -mod tests { - use super::*; - - fn fresh() -> (tempfile::TempDir, LocalStore) { - let dir = tempfile::tempdir().expect("tempdir"); - let store = LocalStore::open(dir.path().join("ls.redb")).expect("open"); - (dir, store) - } - - #[test] - fn set_get_roundtrip() { - let (_dir, store) = fresh(); - store.set("twap", "k", b"v").unwrap(); - assert_eq!(store.get("twap", "k").unwrap().as_deref(), Some(&b"v"[..])); - } - - #[test] - fn namespaces_isolate_modules() { - let (_dir, store) = fresh(); - store.set("a", "k", b"from-a").unwrap(); - store.set("b", "k", b"from-b").unwrap(); - assert_eq!( - store.get("a", "k").unwrap().as_deref(), - Some(&b"from-a"[..]) - ); - assert_eq!( - store.get("b", "k").unwrap().as_deref(), - Some(&b"from-b"[..]) - ); - } - - #[test] - fn delete_then_get_is_none() { - let (_dir, store) = fresh(); - store.set("twap", "k", b"v").unwrap(); - store.delete("twap", "k").unwrap(); - assert!(store.get("twap", "k").unwrap().is_none()); - } - - #[test] - fn list_keys_strips_namespace_prefix() { - let (_dir, store) = fresh(); - store.set("twap", "posted:1", b"x").unwrap(); - store.set("twap", "posted:2", b"y").unwrap(); - store.set("twap", "other", b"z").unwrap(); - let keys = store.list_keys("twap", "posted:").unwrap(); - assert_eq!(keys.len(), 2); - assert!(keys.iter().all(|k| k.starts_with("posted:"))); - } - - #[test] - fn rejects_empty_namespace() { - let (_dir, store) = fresh(); - let err = store.set("", "k", b"v").unwrap_err(); - assert!(matches!(err, StorageError::InvalidNamespace(_))); - } - - #[test] - fn prefix_is_fixed_32_bytes() { - let short = namespace_prefix("a").unwrap(); - let long = namespace_prefix(&"a".repeat(300)).unwrap(); - assert_eq!(short.len(), PREFIX_LEN); - assert_eq!(long.len(), PREFIX_LEN); - // Different inputs produce different prefixes. - assert_ne!(short, long); - } - - #[test] - fn prefix_is_deterministic() { - let p1 = namespace_prefix("twap-monitor").unwrap(); - let p2 = namespace_prefix("twap-monitor").unwrap(); - assert_eq!(p1, p2); - } - - #[test] - fn similar_names_differ() { - // Verify that names that share a common prefix don't collide. - let pa = namespace_prefix("module-a").unwrap(); - let pb = namespace_prefix("module-b").unwrap(); - assert_ne!(pa, pb); - } -} +mod tests; diff --git a/crates/nexum-engine/src/host/local_store_redb/tests.rs b/crates/nexum-engine/src/host/local_store_redb/tests.rs new file mode 100644 index 0000000..5c4feba --- /dev/null +++ b/crates/nexum-engine/src/host/local_store_redb/tests.rs @@ -0,0 +1,80 @@ +use super::*; + +fn fresh() -> (tempfile::TempDir, LocalStore) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = LocalStore::open(dir.path().join("ls.redb")).expect("open"); + (dir, store) +} + +#[test] +fn set_get_roundtrip() { + let (_dir, store) = fresh(); + store.set("twap", "k", b"v").unwrap(); + assert_eq!(store.get("twap", "k").unwrap().as_deref(), Some(&b"v"[..])); +} + +#[test] +fn namespaces_isolate_modules() { + let (_dir, store) = fresh(); + store.set("a", "k", b"from-a").unwrap(); + store.set("b", "k", b"from-b").unwrap(); + assert_eq!( + store.get("a", "k").unwrap().as_deref(), + Some(&b"from-a"[..]) + ); + assert_eq!( + store.get("b", "k").unwrap().as_deref(), + Some(&b"from-b"[..]) + ); +} + +#[test] +fn delete_then_get_is_none() { + let (_dir, store) = fresh(); + store.set("twap", "k", b"v").unwrap(); + store.delete("twap", "k").unwrap(); + assert!(store.get("twap", "k").unwrap().is_none()); +} + +#[test] +fn list_keys_strips_namespace_prefix() { + let (_dir, store) = fresh(); + store.set("twap", "posted:1", b"x").unwrap(); + store.set("twap", "posted:2", b"y").unwrap(); + store.set("twap", "other", b"z").unwrap(); + let keys = store.list_keys("twap", "posted:").unwrap(); + assert_eq!(keys.len(), 2); + assert!(keys.iter().all(|k| k.starts_with("posted:"))); +} + +#[test] +fn rejects_empty_namespace() { + let (_dir, store) = fresh(); + let err = store.set("", "k", b"v").unwrap_err(); + assert!(matches!(err, StorageError::InvalidNamespace(_))); +} + +#[test] +fn prefix_is_fixed_32_bytes() { + let short = namespace_prefix("a").unwrap(); + let long = namespace_prefix(&"a".repeat(300)).unwrap(); + assert_eq!(short.len(), PREFIX_LEN); + assert_eq!(long.len(), PREFIX_LEN); + // Different inputs produce different prefixes. + assert_ne!(short, long); +} + +#[test] +fn prefix_is_deterministic() { + let p1 = namespace_prefix("twap-monitor").unwrap(); + let p2 = namespace_prefix("twap-monitor").unwrap(); + assert_eq!(p1, p2); +} + +#[test] +fn similar_names_differ() { + // Verify that names that share a common prefix don't collide. + let pa = namespace_prefix("module-a").unwrap(); + let pb = namespace_prefix("module-b").unwrap(); + assert_ne!(pa, pb); +} diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 88f3615..7aaec9e 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -427,208 +427,6 @@ fn build_alloy_filter( Ok(filter) } -#[cfg(test)] -mod tests { - use std::path::{Path, PathBuf}; - - use super::*; - - #[test] - fn empty_supervisor_returns_no_subscriptions() { - let sup = Supervisor { - modules: Vec::new(), - }; - assert!(sup.block_chains().is_empty()); - assert!(sup.log_subscriptions().is_empty()); - assert_eq!(sup.module_count(), 0); - } - - // ── E2E helpers ─────────────────────────────────────────────────────── - - /// Path to the pre-built example WASM component. Tests that need it - /// call `example_wasm_or_skip()` which skips gracefully if absent. - fn example_wasm() -> PathBuf { - // CARGO_MANIFEST_DIR → crates/nexum-engine - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .join("target/wasm32-wasip2/release/example.wasm") - } - - fn example_module_toml() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .join("modules/example/module.toml") - } - - /// Returns `None` and prints a skip message if the fixture isn't built. - fn example_wasm_or_skip() -> Option { - let p = example_wasm(); - if p.exists() { - Some(p) - } else { - eprintln!( - "SKIP: {} not found - run `just build-module` to enable E2E tests", - p.display() - ); - None - } - } - - fn make_wasmtime_engine() -> wasmtime::Engine { - let mut config = wasmtime::Config::new(); - config.wasm_component_model(true); - config.consume_fuel(true); - wasmtime::Engine::new(&config).expect("wasmtime engine") - } - - fn make_linker(engine: &wasmtime::Engine) -> Linker { - let mut linker = Linker::::new(engine); - crate::Shepherd::add_to_linker::>( - &mut linker, - |s| s, - ) - .expect("add_to_linker"); - wasmtime_wasi::p2::add_to_linker_async(&mut linker).expect("add_wasi"); - linker - } - /// Return `(dir, store)` so the test holds the `TempDir` for the - /// duration of the test scope and cleans it up on drop. Forgetting - /// the dir (the old `ManuallyDrop` approach) leaks it for the - /// entire process lifetime. - fn temp_local_store() -> (tempfile::TempDir, crate::host::local_store_redb::LocalStore) { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("ls.redb"); - let store = crate::host::local_store_redb::LocalStore::open(path).expect("local store"); - (dir, store) - } - - // ── E2E tests ───────────────────────────────────────────────────────── - - /// Boot supervisor with the example module; verify it starts alive. - #[tokio::test] - async fn e2e_supervisor_boots_example_module() { - let Some(wasm) = example_wasm_or_skip() else { - return; - }; - let engine = make_wasmtime_engine(); - let linker = make_linker(&engine); - let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); - let provider_pool = crate::host::provider_pool::ProviderPool::empty(); - let (_dir, local_store) = temp_local_store(); - - let supervisor = Supervisor::boot_single( - &engine, - &linker, - &wasm, - Some(example_module_toml()).as_deref(), - &cow_pool, - &provider_pool, - &local_store, - ) - .await - .expect("boot_single"); - - assert_eq!(supervisor.module_count(), 1); - assert_eq!(supervisor.alive_count(), 1); - } - - /// Boot with a manifest that subscribes to block events; dispatch one - /// block event and verify the module was invoked and stayed alive. - #[tokio::test] - async fn e2e_block_subscription_dispatched() { - let Some(wasm) = example_wasm_or_skip() else { - return; - }; - let dir = tempfile::tempdir().unwrap(); - let manifest = dir.path().join("module.toml"); - std::fs::write( - &manifest, - r#" -[module] -name = "example" - -[capabilities] -required = ["logging"] - -[[subscription]] -kind = "block" -chain_id = 1 -"#, - ) - .unwrap(); - - let engine = make_wasmtime_engine(); - let linker = make_linker(&engine); - let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); - let provider_pool = crate::host::provider_pool::ProviderPool::empty(); - let (_dir, local_store) = temp_local_store(); - - let mut supervisor = Supervisor::boot_single( - &engine, - &linker, - &wasm, - Some(&manifest), - &cow_pool, - &provider_pool, - &local_store, - ) - .await - .expect("boot_single"); - - let block = nexum::host::types::Block { - chain_id: 1, - number: 19_000_000, - hash: vec![0xab; 32], - timestamp: 1_700_000_000_000, - }; - let dispatched = supervisor.dispatch_block(block).await; - assert_eq!(dispatched, 1, "one module subscribed to chain 1 blocks"); - assert_eq!(supervisor.alive_count(), 1, "module must remain alive"); - } - - // ── build_alloy_filter ──────────────────────────────────────────────── - - #[test] - fn alloy_filter_with_address_and_topic() { - let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; - let topic = "0x237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c00"; - let filter = build_alloy_filter(Some(addr), Some(topic)).unwrap(); - // Check address is set (alloy Filter doesn't expose a simple getter, - // but we can verify the filter serialises the address field). - let serialised = serde_json::to_value(&filter).unwrap(); - let addr_field = serialised.get("address").unwrap().to_string().to_lowercase(); - assert!(addr_field.contains(&addr.to_lowercase()[2..])); // strip 0x - } - - #[test] - fn alloy_filter_no_address_no_topic() { - let filter = build_alloy_filter(None, None).unwrap(); - let serialised = serde_json::to_value(&filter).unwrap(); - // Address and topics should be absent or null. - assert!( - serialised.get("address").is_none() - || serialised["address"].is_null() - || serialised["address"] == serde_json::json!([]) - ); - } - - #[test] - fn alloy_filter_rejects_bad_address() { - let err = build_alloy_filter(Some("not-an-address"), None); - assert!(err.is_err()); - } - - #[test] - fn alloy_filter_rejects_bad_topic() { - let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; - let err = build_alloy_filter(Some(addr), Some("not-a-topic")); - assert!(err.is_err()); - } -} +#[cfg(test)] +mod tests; diff --git a/crates/nexum-engine/src/supervisor/tests.rs b/crates/nexum-engine/src/supervisor/tests.rs new file mode 100644 index 0000000..b290ed4 --- /dev/null +++ b/crates/nexum-engine/src/supervisor/tests.rs @@ -0,0 +1,202 @@ + use std::path::{Path, PathBuf}; + + use super::*; + + #[test] + fn empty_supervisor_returns_no_subscriptions() { + let sup = Supervisor { + modules: Vec::new(), + }; + assert!(sup.block_chains().is_empty()); + assert!(sup.log_subscriptions().is_empty()); + assert_eq!(sup.module_count(), 0); + } + + // ── E2E helpers ─────────────────────────────────────────────────────── + + /// Path to the pre-built example WASM component. Tests that need it + /// call `example_wasm_or_skip()` which skips gracefully if absent. + fn example_wasm() -> PathBuf { + // CARGO_MANIFEST_DIR → crates/nexum-engine + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("target/wasm32-wasip2/release/example.wasm") + } + + fn example_module_toml() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("modules/example/module.toml") + } + + /// Returns `None` and prints a skip message if the fixture isn't built. + fn example_wasm_or_skip() -> Option { + let p = example_wasm(); + if p.exists() { + Some(p) + } else { + eprintln!( + "SKIP: {} not found - run `just build-module` to enable E2E tests", + p.display() + ); + None + } + } + + fn make_wasmtime_engine() -> wasmtime::Engine { + let mut config = wasmtime::Config::new(); + config.wasm_component_model(true); + config.consume_fuel(true); + wasmtime::Engine::new(&config).expect("wasmtime engine") + } + + fn make_linker(engine: &wasmtime::Engine) -> Linker { + let mut linker = Linker::::new(engine); + crate::Shepherd::add_to_linker::>( + &mut linker, + |s| s, + ) + .expect("add_to_linker"); + wasmtime_wasi::p2::add_to_linker_async(&mut linker).expect("add_wasi"); + linker + } + + /// Return `(dir, store)` so the test holds the `TempDir` for the + /// duration of the test scope and cleans it up on drop. Forgetting + /// the dir (the old `ManuallyDrop` approach) leaks it for the + /// entire process lifetime. + fn temp_local_store() -> (tempfile::TempDir, crate::host::local_store_redb::LocalStore) { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("ls.redb"); + let store = crate::host::local_store_redb::LocalStore::open(path).expect("local store"); + (dir, store) + } + + // ── E2E tests ───────────────────────────────────────────────────────── + + /// Boot supervisor with the example module; verify it starts alive. + #[tokio::test] + async fn e2e_supervisor_boots_example_module() { + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let (_dir, local_store) = temp_local_store(); + + let supervisor = Supervisor::boot_single( + &engine, + &linker, + &wasm, + Some(example_module_toml()).as_deref(), + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot_single"); + + assert_eq!(supervisor.module_count(), 1); + assert_eq!(supervisor.alive_count(), 1); + } + + /// Boot with a manifest that subscribes to block events; dispatch one + /// block event and verify the module was invoked and stayed alive. + #[tokio::test] + async fn e2e_block_subscription_dispatched() { + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let manifest = dir.path().join("module.toml"); + std::fs::write( + &manifest, + r#" +[module] +name = "example" + +[capabilities] +required = ["logging"] + +[[subscription]] +kind = "block" +chain_id = 1 +"#, + ) + .unwrap(); + + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let (_dir, local_store) = temp_local_store(); + + let mut supervisor = Supervisor::boot_single( + &engine, + &linker, + &wasm, + Some(&manifest), + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot_single"); + + let block = nexum::host::types::Block { + chain_id: 1, + number: 19_000_000, + hash: vec![0xab; 32], + timestamp: 1_700_000_000_000, + }; + let dispatched = supervisor.dispatch_block(block).await; + assert_eq!(dispatched, 1, "one module subscribed to chain 1 blocks"); + assert_eq!(supervisor.alive_count(), 1, "module must remain alive"); + } + + // ── build_alloy_filter ──────────────────────────────────────────────── + + #[test] + fn alloy_filter_with_address_and_topic() { + let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; + let topic = "0x237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c00"; + let filter = build_alloy_filter(Some(addr), Some(topic)).unwrap(); + // Check address is set (alloy Filter doesn't expose a simple getter, + // but we can verify the filter serialises the address field). + let serialised = serde_json::to_value(&filter).unwrap(); + let addr_field = serialised.get("address").unwrap().to_string().to_lowercase(); + assert!(addr_field.contains(&addr.to_lowercase()[2..])); // strip 0x + } + + #[test] + fn alloy_filter_no_address_no_topic() { + let filter = build_alloy_filter(None, None).unwrap(); + let serialised = serde_json::to_value(&filter).unwrap(); + // Address and topics should be absent or null. + assert!( + serialised.get("address").is_none() + || serialised["address"].is_null() + || serialised["address"] == serde_json::json!([]) + ); + } + + #[test] + fn alloy_filter_rejects_bad_address() { + let err = build_alloy_filter(Some("not-an-address"), None); + assert!(err.is_err()); + } + + #[test] + fn alloy_filter_rejects_bad_topic() { + let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; + let err = build_alloy_filter(Some(addr), Some("not-a-topic")); + assert!(err.is_err()); + } From 7294b5d046d173486c1847b21c6cefe5837b5efe Mon Sep 17 00:00:00 2001 From: brunota20 Date: Mon, 1 Jun 2026 16:12:50 -0300 Subject: [PATCH 17/29] chore(deps): patch cowprotocol to bleu/cow-rs main (post-alpha.3) --- Cargo.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 4652c25..df12a9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,6 +98,18 @@ unsafe_op_in_unsafe_fn = "warn" dbg_macro = "deny" todo = "deny" +# `cowprotocol` v1.0.0-alpha.3 (the crates.io release the engine +# depends on) was cut from `cowdao-grants/cow-rs` PR #5 at commit +# `1742ffa`. `bleu/cow-rs` main has 18 commits since, including the +# `composable::Proof` width fix (relevant to the TWAP poll path), +# `OrderCreation` zero-from-address fast-fail (closes a MEDIUM +# review finding from PR #5), and the `order_book` / `composable` +# submodule splits. Patching to that commit picks them up without +# waiting for an alpha.4 publish. Drop once `cowprotocol >= 1.0.0-alpha.4` +# ships. +[patch.crates-io] +cowprotocol = { git = "https://github.com/bleu/cow-rs", rev = "c012404ffefc411bff543d2290e19ba7fbef2516" } + [profile.dev] panic = "abort" From 24398b938b1e6319b06289b92646d3c15b6f83ee Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 2 Jun 2026 17:04:19 -0300 Subject: [PATCH 18/29] docs(adr): add 0001-0007 capturing engine and CoW architecture decisions --- .gitignore | 7 + .../adr/0001-cow-twap-ethflow-host-helpers.md | 134 ++++++++++++++++++ .../0002-patch-cowprotocol-to-bleu-cow-rs.md | 60 ++++++++ docs/adr/0003-local-store-namespacing.md | 57 ++++++++ .../0004-cow-api-via-cached-orderbookapi.md | 65 +++++++++ .../0005-provider-pool-transport-by-scheme.md | 59 ++++++++ ...06-engine-toml-separate-from-nexum-toml.md | 65 +++++++++ .../0007-upstream-protocol-logic-to-cow-rs.md | 87 ++++++++++++ 8 files changed, 534 insertions(+) create mode 100644 docs/adr/0001-cow-twap-ethflow-host-helpers.md create mode 100644 docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md create mode 100644 docs/adr/0003-local-store-namespacing.md create mode 100644 docs/adr/0004-cow-api-via-cached-orderbookapi.md create mode 100644 docs/adr/0005-provider-pool-transport-by-scheme.md create mode 100644 docs/adr/0006-engine-toml-separate-from-nexum-toml.md create mode 100644 docs/adr/0007-upstream-protocol-logic-to-cow-rs.md diff --git a/.gitignore b/.gitignore index e43a15c..25b6a11 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,11 @@ Thumbs.db # Environment .env .env.* + +# Agent skills / AI tooling — installed locally, never committed. +.agents/ +.claude/ +skills-lock.json + +# Engine runtime state (default state_dir from engine.toml). data/ diff --git a/docs/adr/0001-cow-twap-ethflow-host-helpers.md b/docs/adr/0001-cow-twap-ethflow-host-helpers.md new file mode 100644 index 0000000..8daefce --- /dev/null +++ b/docs/adr/0001-cow-twap-ethflow-host-helpers.md @@ -0,0 +1,134 @@ +--- +status: proposed +--- + +# TWAP and EthFlow as intent helpers in `shepherd:cow@0.2.0` + +## Context + +The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook +access (REST passthrough + `submit-order`). Two further CoW workflows show up +in every non-trivial module: ComposableCoW conditional orders (TWAP being the +canonical example) and EthFlow native-ETH orders. Both share a tight pattern +— observe an on-chain event, derive a signed `OrderCreation`, submit it to +the orderbook — but the derivation has enough protocol detail (digest, +signature scheme, app-data resolution, `getTradeableOrderWithSignature` +eth_call against ComposableCoW) that a guest module would either ship that +logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or +make ten round-trips to the host through generic `chain`/`cow-api` calls. + +The protocol logic itself — TWAP polling, EthFlow log decoding, app-data +resolution — is not engine-specific. Every Rust consumer of CoW Protocol +(indexers, bots, this engine) needs the same primitives. Per ADR-0007, those +primitives belong in the `cowprotocol` crate, not in `nexum-engine`. This +ADR consequently scopes the engine-side helpers to the WIT surface and the +glue that wires the upstream primitives into the host call boundary. + +## Decision + +Add two new interfaces to package `shepherd:cow@0.2.0`: + +```wit +interface twap { + use nexum:host/types@0.2.0.{chain-id, log, host-error}; + use cow-api.{order-uid}; + poll-and-submit: func( + chain-id: chain-id, + registration: log, + ) -> result, host-error>; +} + +interface ethflow { + use nexum:host/types@0.2.0.{chain-id, log, host-error}; + use cow-api.{order-uid}; + submit-from-log: func( + chain-id: chain-id, + placement: log, + ) -> result; +} +``` + +Both interfaces ship in the existing `shepherd` world alongside `cow-api`. +`order-uid` is added to `cow-api` as `type order-uid = list` (56 bytes, +validated host-side) and reused by all three interfaces; `cow-api/submit-order` +keeps returning it instead of `string`. Capability names `"twap"` and +`"ethflow"` are appended to `KNOWN_CAPABILITIES` so manifests can declare +them under `[capabilities].required`. + +Host implementations are thin wrappers (~20–30 LOC each) over three +upstream primitives that land in `cowprotocol` first (see ADR-0007): + +- `cowprotocol::composable::poll_and_build_order(provider, owner, params, + proof)` — returns `Ready(OrderCreation, signature)` or `NotReady` on + contract revert. Backs `twap.poll-and-submit`. +- `cowprotocol::eth_flow::decode_placement(log)` — returns + `(owner, OrderCreation, OrderUid)` from an `OrderPlacement` event log. + Backs `ethflow.submit-from-log`. +- `cowprotocol::app_data::OrderBookAppDataResolver` — given a chain id and + a `bytes32` hash, returns the JSON document (with `EMPTY_APP_DATA_HASH` + fast-path and LRU cache built in). Used by both helpers and any future + module-facing path. + +The engine wires these primitives into HostState and maps their errors to +`host-error` kinds; no protocol logic lives in `nexum-engine`. Modules +continue to declare their own log subscriptions via `[[subscription]]` in +`nexum.toml`; the helpers only decode and submit, they do not +auto-subscribe. + +## Considered options + +- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, + `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, + but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW + glue. mfw78's "reuse over reimplement" applied: that derivation already + lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and + should not be re-shipped in every WASM artifact. +- **Implement the protocol glue inside `nexum-engine` host code, port + upstream later.** Rejected per ADR-0007: every line of TWAP polling or + EthFlow decoding that lives in the engine is a line that future Rust + consumers cannot reuse, and a line that diverges as cow-rs evolves. +- **Single combined interface** `shepherd:cow/orders` with both helpers. + Cheaper world surface but harder to gate per-capability — a module that + only watches EthFlow shouldn't have to import TWAP and vice versa. + Splitting keeps `[capabilities].required` honest. +- **Symmetric `result, host-error>` for both.** TWAP and + EthFlow are genuinely asymmetric: TWAP is poll-driven and a `None` ("not + tradeable yet") is the normal steady-state; EthFlow is event-driven and + every accepted log produces exactly one UID. Forcing symmetry obscures + semantics for callers. +- **`log-json: list` payload** instead of the typed `nexum:host/types.log` + record. The record already exists and the engine's event dispatch already + projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both + ergonomics and "no duplicate decoders". +- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 helper + only handles `ComposableCoW.create()` (empty proof, single conditional + order). `setRoot` polling requires off-chain proof derivation that itself + warrants a separate helper (`twap.poll-and-submit-with-proof`) once a + module actually needs it. +- **Bumping the package to `shepherd:cow@0.3.0`.** Not needed: adding + imports to an existing world is additive under WIT subsumption rules. + Modules compiled against the current 0.2.0 surface continue to build. + +## Consequences + +- `cow-api/submit-order` return type changes from `string` to `order-uid`. + No external consumers today (0.2 is unreleased), so this is internal. +- Host helpers require a chain to be configured in `[chains.]` — + uncovered chains return `host-error.unsupported`. Same posture as + `cow-api`. +- Orderbook idempotency (same UID on duplicate submit) is preserved but + invisible to the module. Modules that need dedup must record UIDs in + `local-store` themselves. +- App-data resolution adds a GET to `api.cow.fi/{chain}/api/v1/app_data/{hash}` + on the first sighting of a non-empty hash. The LRU cache and the GET + itself live in `cowprotocol::app_data::OrderBookAppDataResolver` + (ADR-0007 item 3); cache miss + orderbook miss surfaces as + `host-error.unavailable`. +- Implementation order: the three `cowprotocol` primitives (`composable:: + poll_and_build_order`, `eth_flow::decode_placement`, + `app_data::OrderBookAppDataResolver`) land in `bleu/cow-rs` first; + `nullis-shepherd` adopts via the existing `[patch.crates-io]` rev bump + (ADR-0002). Host-side issues stay blocked on upstream merges. +- Failure modes map onto existing `host-error-kind` variants + (`invalid-input`, `denied`, `rate-limited`, `timeout`, `unavailable`, + `unsupported`, `internal`). No new error taxonomy. diff --git a/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md b/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md new file mode 100644 index 0000000..ed73ccb --- /dev/null +++ b/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md @@ -0,0 +1,60 @@ +--- +status: proposed +implemented-in: nullislabs/shepherd#10 +--- + +# Patch `cowprotocol` crate to `bleu/cow-rs` main + +## Context + +`cowprotocol` v1.0.0-alpha.3 (the version on crates.io) was cut from +`cowdao-grants/cow-rs` PR #5 at commit `1742ffa`. The published artifact +predates 18 follow-up commits on `bleu/cow-rs` main that the engine +materially depends on, in particular: + +- `composable::Proof` byte-width fix (consumed by the TWAP poll path); +- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding + from mfw78's review of PR #5); +- `order_book` / `composable` submodule splits (cleaner imports on the + engine side, no more `cowprotocol::order_book::*` re-export gymnastics). + +ADR-0007 additionally commits us to pushing TWAP / EthFlow / app-data +protocol logic upstream into `cowprotocol` first and consuming it via the +same patched dependency, so the patch surface will continue growing +through M2. + +There is no published `alpha.4` and no scheduled date for one. + +## Decision + +Add a workspace-level `[patch.crates-io]` redirecting `cowprotocol` to +`https://github.com/bleu/cow-rs` at commit `c012404`. Every crate that +declares `cowprotocol = "1.0.0-alpha.3"` (engine, modules, future SDK) +silently picks up the patched build with no `Cargo.toml` change at the +dependent site. + +## Considered options + +- **Vendor the missing types locally.** Rejected: re-implementing + `composable::Proof`, `OrderCreation`, etc. in the engine repo is exactly + the AI-duplication anti-pattern mfw78 flagged in cow-rs PR #5. Reuse + over reimplement applies. +- **Pin every dependent to `cow-rs` git directly.** Works but every new + workspace member has to remember the git source. `[patch.crates-io]` + centralises the override. +- **Wait for `alpha.4` to publish.** No ETA; the TWAP/EthFlow milestone + cannot land without `composable::Proof` correct. + +## Consequences + +- `cargo update` will re-resolve to the same `rev` — the lock pins it. +- Bumping the rev is a single-line workspace edit; reviewers see one diff. +- Drop the patch entirely once a published `cowprotocol` release contains + both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions + (`composable::poll_and_build_order`, `eth_flow::decode_placement`, + `app_data::OrderBookAppDataResolver`). Until then, expect the patch + rev to advance with every cow-rs merge that the engine consumes. +- Modules built against this workspace inherit the patch transitively; + modules built standalone against crates.io will see `alpha.3` and may + hit the very bugs the patch closes — flag in the SDK README when M3 + lands. diff --git a/docs/adr/0003-local-store-namespacing.md b/docs/adr/0003-local-store-namespacing.md new file mode 100644 index 0000000..ee5be1e --- /dev/null +++ b/docs/adr/0003-local-store-namespacing.md @@ -0,0 +1,57 @@ +--- +status: proposed +implemented-in: nullislabs/shepherd#8 +--- + +# Per-module namespacing in `local-store` via `[len:u8][module][key]` prefix + +## Context + +`nexum:host/local-store` is a key-value store shared across all modules +the engine runs. Two modules using the same key string (e.g. +`"last-block"`) must see disjoint values; one module must never read or +overwrite another's data. The engine knows each module's name at +instantiation time, so namespacing is a host-side concern. + +## Decision + +Single redb database file at `EngineConfig.engine.state_dir`, single +shared table `nexum:local-store`. Every key handed to redb is composed +host-side as: + +``` +[len: u8] [module_name: len bytes] [raw key: rest of the bytes] +``` + +Module names longer than 255 bytes are rejected at `LocalStore` +construction (matches the one-byte length prefix). Modules see plain +key strings on both the read and write paths; the prefix is invisible +to the WIT-facing API. + +## Considered options + +- **Separator string** (`{module}:{key}`). Rejected: any module name + containing `:` collides with another module's `:`-bearing key. Length + prefix is unambiguous regardless of payload bytes. +- **One redb database file per module.** Rejected: multiplies open + file handles linearly in modules, blocks any future cross-module + atomic operations (not currently planned but cheap to keep on the + table), and complicates backup tooling (N files vs 1). +- **One redb *table* per module within a single file.** Rejected: redb + `TableDefinition` lifetimes are `'static`, so table names must be + known at compile time. Dynamic table opening per module would force + string-leak workarounds and exposes the same name-collision question + as separator-based keys. + +## Consequences + +- Module data is physically interleaved in the redb tree (range scans + for one module's keys are O(log n + module-key-count) — fine for our + workload). +- Migrations changing the namespacing layout break every existing + module's persisted state. The format must stay stable through 0.x. +- A module's `list-keys` (when added) iterates over the namespace + range; the host strips the prefix before returning to the guest. +- 255-byte module-name limit is enforced loudly at construction, so + configuration errors surface at boot rather than silently corrupting + data at first write. diff --git a/docs/adr/0004-cow-api-via-cached-orderbookapi.md b/docs/adr/0004-cow-api-via-cached-orderbookapi.md new file mode 100644 index 0000000..bf4fd1a --- /dev/null +++ b/docs/adr/0004-cow-api-via-cached-orderbookapi.md @@ -0,0 +1,65 @@ +--- +status: proposed +implemented-in: nullislabs/shepherd#8 +--- + +# `cow-api` host backend routes both `request` and `submit-order` through `cowprotocol::OrderBookApi` + +## Context + +`shepherd:cow/cow-api` exposes two operations: a generic REST passthrough +(`request`) and a typed order submission (`submit-order`). Either could +be implemented with raw `reqwest` against `api.cow.fi/{slug}/api/v1`, +but the published `cowprotocol` crate already ships an `OrderBookApi` +client that knows the chain-specific base URL, the canonical paths, and +the `post_order` codec. + +## Decision + +At engine boot, construct one `cowprotocol::OrderBookApi` per +`cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, +ArbitrumOne, Base) into a `BTreeMap` keyed by EVM +chain id. Both `cow-api` operations consult this pool: + +- `request` resolves the chain's `OrderBookApi`, reads + `api.base_url()` for the prefix, joins the module-supplied path, + and dispatches via a shared `reqwest::Client`. +- `submit-order` deserialises the JSON `OrderCreation` and calls + `OrderBookApi::post_order` directly. The crate handles signing-scheme + encoding, error mapping, and `OrderUid` extraction. + +Chains not in `cowprotocol::Chain` return `HostError { kind: unsupported }` +at the host call boundary. + +## Considered options + +- **Raw `reqwest` for both.** Rejected: forces us to maintain the + chain → base-URL table (drifts whenever cowprotocol adds a chain) and + reimplement `post_order`'s body codec and error mapping — the exact + duplication mfw78 called out in cow-rs PR #5. +- **`OrderBookApi` for `submit-order`, raw `reqwest` for `request`.** + Tempting (request is opaque to the crate) but means two separate + chain-resolution paths, two HTTP clients, and a second place to keep + the chain set in sync. +- **Build `OrderBookApi` lazily on first call per chain.** Rejected: + hides config errors at runtime. Up-front boot construction surfaces + unknown chains immediately and amortises away the per-call cost. + +## Consequences + +- Operator-supplied custom orderbook URLs (barn, staging, forked + deployments) are out of scope for the default constructor and require + a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` + constructor in the cow-rs crate (ADR-0007, upstream item 4 — not + vendored locally). +- App-data resolution moves out of the engine entirely once ADR-0007 + item 3 lands: `OrderBookPool` exposes an `AppDataResolver` constructed + from the same per-chain `OrderBookApi` clients, and both `cow-api` + call sites and the new twap/ethflow helpers (ADR-0001) consume that + shared resolver. No engine-side LRU. +- Adding a chain means a `cowprotocol::Chain` variant lands in cow-rs + first; the engine inherits it on the next patched rev bump. +- The shared `reqwest::Client` enables connection pooling across both + `request` and `submit-order` paths. +- TWAP and EthFlow helpers (ADR-0001) reuse the same pool — no + duplicated client construction in those host wrappers. diff --git a/docs/adr/0005-provider-pool-transport-by-scheme.md b/docs/adr/0005-provider-pool-transport-by-scheme.md new file mode 100644 index 0000000..aba0a72 --- /dev/null +++ b/docs/adr/0005-provider-pool-transport-by-scheme.md @@ -0,0 +1,59 @@ +--- +status: proposed +implemented-in: nullislabs/shepherd#8, nullislabs/shepherd#9 +--- + +# Per-chain alloy provider transport selected by URL scheme + +## Context + +`nexum:host/chain` covers both generic JSON-RPC dispatch (`request`) +and event subscriptions (`subscribe-blocks`, `subscribe-logs`). +Subscriptions require a duplex transport (`eth_subscribe` is push-only +over a long-lived connection); request/response works on either HTTP +or WebSocket. The operator configures one RPC endpoint per chain in +`engine.toml`; the engine has to decide which alloy transport to use. + +## Decision + +The `ProviderPool::from_config` constructor reads each chain's +`rpc_url` and switches by URL scheme prefix: + +- `ws://` or `wss://` → `ProviderBuilder::new().connect_ws(WsConnect::new(url))`. + Pubsub transport. Subscriptions and request/response both work. +- `http://` or `https://` → `ProviderBuilder::new().connect_http(parsed)`. + HTTP transport. Request/response only; `subscribe-blocks` and + `subscribe-logs` surface as a host error to the guest. + +Both transports erase to `DynProvider` so the rest of the engine is +transport-agnostic. + +## Considered options + +- **Force WSS everywhere.** Rejected: many providers (Alchemy, Infura, + self-hosted RPC) expose HTTP-only on free tiers, and modules that + only need `request` (no subscriptions) shouldn't be blocked by a + WSS requirement. +- **Explicit `transport = "ws" | "http"` field per chain in + `engine.toml`.** Rejected for 0.2: redundant with the URL scheme, + and operators already distinguish `wss://` from `https://` + endpoints when copying them from their RPC provider's dashboard. + Revisit if we add IPC (`/path/to/geth.ipc`) — scheme alone won't + carry that. +- **Open both an HTTP and a WSS connection per chain.** Rejected: doubles + connection count for the common case where one endpoint serves + both, and forces operators to provide two URLs even when their + provider returns identical data on both. + +## Consequences + +- Operators that need subscriptions must supply WSS URLs; HTTP-only + chains downgrade to request-only mode at the host call boundary. +- Connection failures at boot are fatal (the engine refuses to start + with a broken chain). This is intentional — silent fall-back to a + half-functioning state masks misconfiguration that a module then + rediscovers at first event. +- Adding IPC support is additive: extend the scheme match with + `/` / `file://` and call `connect_ipc`. +- The `DynProvider` erasure costs a virtual dispatch per call — a + measurable concern at scale, deferred to M4 if profiling shows it. diff --git a/docs/adr/0006-engine-toml-separate-from-nexum-toml.md b/docs/adr/0006-engine-toml-separate-from-nexum-toml.md new file mode 100644 index 0000000..57f0dd7 --- /dev/null +++ b/docs/adr/0006-engine-toml-separate-from-nexum-toml.md @@ -0,0 +1,65 @@ +--- +status: proposed +implemented-in: nullislabs/shepherd#8, nullislabs/shepherd#9 +--- + +# Operator config (`engine.toml`) is separate from module manifest (`nexum.toml`) + +## Context + +The engine needs two distinct kinds of configuration: what the +**operator** decides at deployment time (which chains to connect to, +where the local-store database lives, which modules to boot) and +what the **module developer** declares at build time (required and +optional capabilities, HTTP allowlist, module-specific config keys). +These have different reviewers, different threat models, and change +on different cadences. + +## Decision + +Two distinct files, distinct schemas, distinct loaders: + +- **`engine.toml`** — operator-owned, lives next to the engine binary + or pointed to by `--engine-config`. Defines `[engine]` (state_dir, + log_level), `[chains.]` (rpc_url), and `[[modules]]` (path, + manifest). Loaded by `engine_config::EngineConfig::load`. +- **`nexum.toml`** — module-developer-owned, ships in the module's + bundle alongside its `.wasm` component. Defines `[module]`, + `[capabilities]` (required, optional, http allowlist), `[config]`. + Loaded by `manifest::load`. + +The engine config carries the path to each module's manifest; the +two never collapse into one file. + +## Considered options + +- **Single `shepherd.toml` with `[engine]`, `[chains]`, `[[modules]]` + *and* nested `[modules..capabilities]` per module.** Rejected: + conflates operator and developer concerns. A module's capability + declaration is a property of the build, not the deployment — it + belongs in the artifact, not in the operator's local file. Auditing + a module's capabilities also becomes a per-deployment exercise + instead of a property visible in the published bundle. +- **`nexum.toml` inside the engine config (module entries embed it + inline).** Rejected for the same reason; also bloats `engine.toml`. +- **Drop `engine.toml` entirely; pass everything as CLI flags or + env vars.** Rejected: per-chain RPC URLs and module lists are + awkward as flags, and `RUST_LOG` already covers the only thing + that env vars naturally express. + +## Consequences + +- A deployment needs both files. A missing `engine.toml` falls back + to "no chains, default state_dir" — the example logging module + still runs; cow-api / chain backends report `unsupported`. +- A missing `nexum.toml` triggers the 0.1-compat deprecation warning + in `manifest::fallback_manifest()` (defined in + `crates/nexum-engine/src/manifest.rs`) and treats every linked + capability as required. This fallback is scheduled for removal in + 0.3 per `docs/migration/0.1-to-0.2.md`. +- Module-bundle redistribution carries `nexum.toml` with the + artifact; engines do not need to ship templates. +- Future content-addressed module distribution (0.3) embeds + `nexum.toml` in the bundle hash; `engine.toml` references modules + by content address rather than filesystem path. The split survives + that migration unchanged. diff --git a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md new file mode 100644 index 0000000..e60c895 --- /dev/null +++ b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md @@ -0,0 +1,87 @@ +--- +status: proposed +--- + +# Push CoW Protocol logic to `cow-rs` first, adopt in `nexum-engine` second + +## Context + +Implementing ADR-0001 (twap + ethflow host helpers) and ADR-0004 (cow-api +backend) surfaces a recurring question: when the engine needs a piece of +CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose +(TWAP polling glue, EthFlow log decoding, app-data hash-to-document +resolution, custom orderbook URLs), do we write that logic locally in +`nexum-engine` and tidy it up upstream later, or do we open the cow-rs +PR first and only land the engine wiring afterwards? + +mfw78's review of cow-rs PR #5 named the failure mode explicitly: +duplicating work that an existing crate could do is the AI-coding +anti-pattern most likely to land in a Bleu PR. The same risk applies to +any engine-side reimplementation of protocol logic. + +## Decision + +Protocol-level CoW logic — anything that an indexer, a bot, or a +non-`nexum` Rust consumer of CoW Protocol would also need — lands in +`bleu/cow-rs` first as an upstream PR, and is consumed by `nexum-engine` +via the existing `[patch.crates-io]` rev bump (ADR-0002). The engine +never writes throwaway local copies of the same logic with the intent +to "port later". + +The concrete set of primitives we know we need is, in priority order: + +1. `cowprotocol::composable::poll_and_build_order(provider, owner, + params, proof) -> Result` — eth_call against + `ComposableCoW.getTradeableOrderWithSignature`, decode return, + rebuild `OrderCreation`. Backs `twap.poll-and-submit`. +2. `cowprotocol::eth_flow::decode_placement(log) -> + Result` — decode `OrderPlacement` event log, + reconstruct `OrderCreation` and `OrderUid`. Backs + `ethflow.submit-from-log`. +3. `cowprotocol::app_data::OrderBookAppDataResolver` — `AppDataResolver` + trait + cached implementation around `OrderBookApi::app_data(hash)`, + with `EMPTY_APP_DATA_HASH` fast-path. Used by twap, ethflow, and any + future caller that needs to materialise an app-data document. +4. `cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)` — + custom-URL constructor for barn / staging / forked deployments. +5. `cowprotocol` `wasm32` compatibility — feature-gate the `reqwest` + dependency so guest modules can use the pure types + (`Order`, `OrderCreation`, `OrderUid`, `composable::*`, + `eth_flow::decode_*`) without dragging in an HTTP client. + +Lower-priority follow-ons (richer `OrderBookApiError` variants, +`OrderUid::from_slice`, retry middleware, `OrderCreation::from_gpv2`) +are good-to-have but are not blocking for the M2 host scope. + +## Considered options + +- **Implement locally, refactor upstream later.** Faster short term but + predictably leaves an indeterminate amount of duplicated logic in the + engine, contradicts mfw78's stated conventions, and grows technical + debt every time cow-rs evolves the underlying types. Rejected. +- **Wait for cow-rs upstream maintainers to add these on their own.** + No evidence anyone else is doing this work; the grant timeline does + not permit waiting. +- **Vendor a fork of cow-rs inside `nullislabs/shepherd`.** Worst of all + worlds: blocks neither the engine nor cow-rs from drifting, and + forces every other CoW consumer to re-derive the same primitives. + +## Consequences + +- Every M2 engine issue that consumes one of the five primitives above + is blocked on its cow-rs PR merging. We sequence issues so that + upstream PRs and engine adoption can land in parallel where possible + (e.g., open all three protocol-helper PRs against `bleu/cow-rs` + simultaneously rather than serially). +- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0002) is + bumped after each cow-rs merge; the bump is the engine's signal that a + new primitive is consumable. +- PRs in `bleu/cow-rs` follow the existing mfw78 conventions established + by cow-rs PR #5: severity-tagged reviews, alloy reuse over local + reimplementation, GPL-3.0, edition 2024, terse rustdoc. +- After acceptance in `bleu/cow-rs`, each primitive is also surfaced as + a PR (or backport) against `cowdao-grants/cow-rs` so the wider + ecosystem benefits and the bleu fork narrows over time. +- The engine repo stays small: `nexum-engine` contains WIT, host + wiring, supervisor, redb store, alloy provider pool, and `engine.toml` + schema — nothing about CoW Protocol semantics. From bab4311ca0fa95b17b1aef70fd127aa76b832990 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 2 Jun 2026 17:18:11 -0300 Subject: [PATCH 19/29] docs(adr): unwrap hard-wrapped paragraphs to single line each --- .../adr/0001-cow-twap-ethflow-host-helpers.md | 120 ++++-------------- .../0002-patch-cowprotocol-to-bleu-cow-rs.md | 45 ++----- docs/adr/0003-local-store-namespacing.md | 44 ++----- .../0004-cow-api-via-cached-orderbookapi.md | 58 ++------- .../0005-provider-pool-transport-by-scheme.md | 51 ++------ ...06-engine-toml-separate-from-nexum-toml.md | 55 ++------ .../0007-upstream-protocol-logic-to-cow-rs.md | 82 +++--------- 7 files changed, 98 insertions(+), 357 deletions(-) diff --git a/docs/adr/0001-cow-twap-ethflow-host-helpers.md b/docs/adr/0001-cow-twap-ethflow-host-helpers.md index 8daefce..384e368 100644 --- a/docs/adr/0001-cow-twap-ethflow-host-helpers.md +++ b/docs/adr/0001-cow-twap-ethflow-host-helpers.md @@ -6,23 +6,9 @@ status: proposed ## Context -The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook -access (REST passthrough + `submit-order`). Two further CoW workflows show up -in every non-trivial module: ComposableCoW conditional orders (TWAP being the -canonical example) and EthFlow native-ETH orders. Both share a tight pattern -— observe an on-chain event, derive a signed `OrderCreation`, submit it to -the orderbook — but the derivation has enough protocol detail (digest, -signature scheme, app-data resolution, `getTradeableOrderWithSignature` -eth_call against ComposableCoW) that a guest module would either ship that -logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or -make ten round-trips to the host through generic `chain`/`cow-api` calls. - -The protocol logic itself — TWAP polling, EthFlow log decoding, app-data -resolution — is not engine-specific. Every Rust consumer of CoW Protocol -(indexers, bots, this engine) needs the same primitives. Per ADR-0007, those -primitives belong in the `cowprotocol` crate, not in `nexum-engine`. This -ADR consequently scopes the engine-side helpers to the WIT surface and the -glue that wires the upstream primitives into the host call boundary. +The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook access (REST passthrough + `submit-order`). Two further CoW workflows show up in every non-trivial module: ComposableCoW conditional orders (TWAP being the canonical example) and EthFlow native-ETH orders. Both share a tight pattern — observe an on-chain event, derive a signed `OrderCreation`, submit it to the orderbook — but the derivation has enough protocol detail (digest, signature scheme, app-data resolution, `getTradeableOrderWithSignature` eth_call against ComposableCoW) that a guest module would either ship that logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or make ten round-trips to the host through generic `chain`/`cow-api` calls. + +The protocol logic itself — TWAP polling, EthFlow log decoding, app-data resolution — is not engine-specific. Every Rust consumer of CoW Protocol (indexers, bots, this engine) needs the same primitives. Per ADR-0007, those primitives belong in the `cowprotocol` crate, not in `nexum-engine`. This ADR consequently scopes the engine-side helpers to the WIT surface and the glue that wires the upstream primitives into the host call boundary. ## Decision @@ -48,87 +34,31 @@ interface ethflow { } ``` -Both interfaces ship in the existing `shepherd` world alongside `cow-api`. -`order-uid` is added to `cow-api` as `type order-uid = list` (56 bytes, -validated host-side) and reused by all three interfaces; `cow-api/submit-order` -keeps returning it instead of `string`. Capability names `"twap"` and -`"ethflow"` are appended to `KNOWN_CAPABILITIES` so manifests can declare -them under `[capabilities].required`. - -Host implementations are thin wrappers (~20–30 LOC each) over three -upstream primitives that land in `cowprotocol` first (see ADR-0007): - -- `cowprotocol::composable::poll_and_build_order(provider, owner, params, - proof)` — returns `Ready(OrderCreation, signature)` or `NotReady` on - contract revert. Backs `twap.poll-and-submit`. -- `cowprotocol::eth_flow::decode_placement(log)` — returns - `(owner, OrderCreation, OrderUid)` from an `OrderPlacement` event log. - Backs `ethflow.submit-from-log`. -- `cowprotocol::app_data::OrderBookAppDataResolver` — given a chain id and - a `bytes32` hash, returns the JSON document (with `EMPTY_APP_DATA_HASH` - fast-path and LRU cache built in). Used by both helpers and any future - module-facing path. - -The engine wires these primitives into HostState and maps their errors to -`host-error` kinds; no protocol logic lives in `nexum-engine`. Modules -continue to declare their own log subscriptions via `[[subscription]]` in -`nexum.toml`; the helpers only decode and submit, they do not -auto-subscribe. +Both interfaces ship in the existing `shepherd` world alongside `cow-api`. `order-uid` is added to `cow-api` as `type order-uid = list` (56 bytes, validated host-side) and reused by all three interfaces; `cow-api/submit-order` keeps returning it instead of `string`. Capability names `"twap"` and `"ethflow"` are appended to `KNOWN_CAPABILITIES` so manifests can declare them under `[capabilities].required`. + +Host implementations are thin wrappers (~20–30 LOC each) over three upstream primitives that land in `cowprotocol` first (see ADR-0007): + +- `cowprotocol::composable::poll_and_build_order(provider, owner, params, proof)` — returns `Ready(OrderCreation, signature)` or `NotReady` on contract revert. Backs `twap.poll-and-submit`. +- `cowprotocol::eth_flow::decode_placement(log)` — returns `(owner, OrderCreation, OrderUid)` from an `OrderPlacement` event log. Backs `ethflow.submit-from-log`. +- `cowprotocol::app_data::OrderBookAppDataResolver` — given a chain id and a `bytes32` hash, returns the JSON document (with `EMPTY_APP_DATA_HASH` fast-path and LRU cache built in). Used by both helpers and any future module-facing path. + +The engine wires these primitives into HostState and maps their errors to `host-error` kinds; no protocol logic lives in `nexum-engine`. Modules continue to declare their own log subscriptions via `[[subscription]]` in `nexum.toml`; the helpers only decode and submit, they do not auto-subscribe. ## Considered options -- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, - `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, - but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW - glue. mfw78's "reuse over reimplement" applied: that derivation already - lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and - should not be re-shipped in every WASM artifact. -- **Implement the protocol glue inside `nexum-engine` host code, port - upstream later.** Rejected per ADR-0007: every line of TWAP polling or - EthFlow decoding that lives in the engine is a line that future Rust - consumers cannot reuse, and a line that diverges as cow-rs evolves. -- **Single combined interface** `shepherd:cow/orders` with both helpers. - Cheaper world surface but harder to gate per-capability — a module that - only watches EthFlow shouldn't have to import TWAP and vice versa. - Splitting keeps `[capabilities].required` honest. -- **Symmetric `result, host-error>` for both.** TWAP and - EthFlow are genuinely asymmetric: TWAP is poll-driven and a `None` ("not - tradeable yet") is the normal steady-state; EthFlow is event-driven and - every accepted log produces exactly one UID. Forcing symmetry obscures - semantics for callers. -- **`log-json: list` payload** instead of the typed `nexum:host/types.log` - record. The record already exists and the engine's event dispatch already - projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both - ergonomics and "no duplicate decoders". -- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 helper - only handles `ComposableCoW.create()` (empty proof, single conditional - order). `setRoot` polling requires off-chain proof derivation that itself - warrants a separate helper (`twap.poll-and-submit-with-proof`) once a - module actually needs it. -- **Bumping the package to `shepherd:cow@0.3.0`.** Not needed: adding - imports to an existing world is additive under WIT subsumption rules. - Modules compiled against the current 0.2.0 surface continue to build. +- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW glue. mfw78's "reuse over reimplement" applied: that derivation already lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and should not be re-shipped in every WASM artifact. +- **Implement the protocol glue inside `nexum-engine` host code, port upstream later.** Rejected per ADR-0007: every line of TWAP polling or EthFlow decoding that lives in the engine is a line that future Rust consumers cannot reuse, and a line that diverges as cow-rs evolves. +- **Single combined interface** `shepherd:cow/orders` with both helpers. Cheaper world surface but harder to gate per-capability — a module that only watches EthFlow shouldn't have to import TWAP and vice versa. Splitting keeps `[capabilities].required` honest. +- **Symmetric `result, host-error>` for both.** TWAP and EthFlow are genuinely asymmetric: TWAP is poll-driven and a `None` ("not tradeable yet") is the normal steady-state; EthFlow is event-driven and every accepted log produces exactly one UID. Forcing symmetry obscures semantics for callers. +- **`log-json: list` payload** instead of the typed `nexum:host/types.log` record. The record already exists and the engine's event dispatch already projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both ergonomics and "no duplicate decoders". +- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 helper only handles `ComposableCoW.create()` (empty proof, single conditional order). `setRoot` polling requires off-chain proof derivation that itself warrants a separate helper (`twap.poll-and-submit-with-proof`) once a module actually needs it. +- **Bumping the package to `shepherd:cow@0.3.0`.** Not needed: adding imports to an existing world is additive under WIT subsumption rules. Modules compiled against the current 0.2.0 surface continue to build. ## Consequences -- `cow-api/submit-order` return type changes from `string` to `order-uid`. - No external consumers today (0.2 is unreleased), so this is internal. -- Host helpers require a chain to be configured in `[chains.]` — - uncovered chains return `host-error.unsupported`. Same posture as - `cow-api`. -- Orderbook idempotency (same UID on duplicate submit) is preserved but - invisible to the module. Modules that need dedup must record UIDs in - `local-store` themselves. -- App-data resolution adds a GET to `api.cow.fi/{chain}/api/v1/app_data/{hash}` - on the first sighting of a non-empty hash. The LRU cache and the GET - itself live in `cowprotocol::app_data::OrderBookAppDataResolver` - (ADR-0007 item 3); cache miss + orderbook miss surfaces as - `host-error.unavailable`. -- Implementation order: the three `cowprotocol` primitives (`composable:: - poll_and_build_order`, `eth_flow::decode_placement`, - `app_data::OrderBookAppDataResolver`) land in `bleu/cow-rs` first; - `nullis-shepherd` adopts via the existing `[patch.crates-io]` rev bump - (ADR-0002). Host-side issues stay blocked on upstream merges. -- Failure modes map onto existing `host-error-kind` variants - (`invalid-input`, `denied`, `rate-limited`, `timeout`, `unavailable`, - `unsupported`, `internal`). No new error taxonomy. +- `cow-api/submit-order` return type changes from `string` to `order-uid`. No external consumers today (0.2 is unreleased), so this is internal. +- Host helpers require a chain to be configured in `[chains.]` — uncovered chains return `host-error.unsupported`. Same posture as `cow-api`. +- Orderbook idempotency (same UID on duplicate submit) is preserved but invisible to the module. Modules that need dedup must record UIDs in `local-store` themselves. +- App-data resolution adds a GET to `api.cow.fi/{chain}/api/v1/app_data/{hash}` on the first sighting of a non-empty hash. The LRU cache and the GET itself live in `cowprotocol::app_data::OrderBookAppDataResolver` (ADR-0007 item 3); cache miss + orderbook miss surfaces as `host-error.unavailable`. +- Implementation order: the three `cowprotocol` primitives (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `app_data::OrderBookAppDataResolver`) land in `bleu/cow-rs` first; `nullis-shepherd` adopts via the existing `[patch.crates-io]` rev bump (ADR-0002). Host-side issues stay blocked on upstream merges. +- Failure modes map onto existing `host-error-kind` variants (`invalid-input`, `denied`, `rate-limited`, `timeout`, `unavailable`, `unsupported`, `internal`). No new error taxonomy. diff --git a/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md b/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md index ed73ccb..380b632 100644 --- a/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md +++ b/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md @@ -7,54 +7,29 @@ implemented-in: nullislabs/shepherd#10 ## Context -`cowprotocol` v1.0.0-alpha.3 (the version on crates.io) was cut from -`cowdao-grants/cow-rs` PR #5 at commit `1742ffa`. The published artifact -predates 18 follow-up commits on `bleu/cow-rs` main that the engine -materially depends on, in particular: +`cowprotocol` v1.0.0-alpha.3 (the version on crates.io) was cut from `cowdao-grants/cow-rs` PR #5 at commit `1742ffa`. The published artifact predates 18 follow-up commits on `bleu/cow-rs` main that the engine materially depends on, in particular: - `composable::Proof` byte-width fix (consumed by the TWAP poll path); -- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding - from mfw78's review of PR #5); -- `order_book` / `composable` submodule splits (cleaner imports on the - engine side, no more `cowprotocol::order_book::*` re-export gymnastics). +- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding from mfw78's review of PR #5); +- `order_book` / `composable` submodule splits (cleaner imports on the engine side, no more `cowprotocol::order_book::*` re-export gymnastics). -ADR-0007 additionally commits us to pushing TWAP / EthFlow / app-data -protocol logic upstream into `cowprotocol` first and consuming it via the -same patched dependency, so the patch surface will continue growing -through M2. +ADR-0007 additionally commits us to pushing TWAP / EthFlow / app-data protocol logic upstream into `cowprotocol` first and consuming it via the same patched dependency, so the patch surface will continue growing through M2. There is no published `alpha.4` and no scheduled date for one. ## Decision -Add a workspace-level `[patch.crates-io]` redirecting `cowprotocol` to -`https://github.com/bleu/cow-rs` at commit `c012404`. Every crate that -declares `cowprotocol = "1.0.0-alpha.3"` (engine, modules, future SDK) -silently picks up the patched build with no `Cargo.toml` change at the -dependent site. +Add a workspace-level `[patch.crates-io]` redirecting `cowprotocol` to `https://github.com/bleu/cow-rs` at commit `c012404`. Every crate that declares `cowprotocol = "1.0.0-alpha.3"` (engine, modules, future SDK) silently picks up the patched build with no `Cargo.toml` change at the dependent site. ## Considered options -- **Vendor the missing types locally.** Rejected: re-implementing - `composable::Proof`, `OrderCreation`, etc. in the engine repo is exactly - the AI-duplication anti-pattern mfw78 flagged in cow-rs PR #5. Reuse - over reimplement applies. -- **Pin every dependent to `cow-rs` git directly.** Works but every new - workspace member has to remember the git source. `[patch.crates-io]` - centralises the override. -- **Wait for `alpha.4` to publish.** No ETA; the TWAP/EthFlow milestone - cannot land without `composable::Proof` correct. +- **Vendor the missing types locally.** Rejected: re-implementing `composable::Proof`, `OrderCreation`, etc. in the engine repo is exactly the AI-duplication anti-pattern mfw78 flagged in cow-rs PR #5. Reuse over reimplement applies. +- **Pin every dependent to `cow-rs` git directly.** Works but every new workspace member has to remember the git source. `[patch.crates-io]` centralises the override. +- **Wait for `alpha.4` to publish.** No ETA; the TWAP/EthFlow milestone cannot land without `composable::Proof` correct. ## Consequences - `cargo update` will re-resolve to the same `rev` — the lock pins it. - Bumping the rev is a single-line workspace edit; reviewers see one diff. -- Drop the patch entirely once a published `cowprotocol` release contains - both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions - (`composable::poll_and_build_order`, `eth_flow::decode_placement`, - `app_data::OrderBookAppDataResolver`). Until then, expect the patch - rev to advance with every cow-rs merge that the engine consumes. -- Modules built against this workspace inherit the patch transitively; - modules built standalone against crates.io will see `alpha.3` and may - hit the very bugs the patch closes — flag in the SDK README when M3 - lands. +- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `app_data::OrderBookAppDataResolver`). Until then, expect the patch rev to advance with every cow-rs merge that the engine consumes. +- Modules built against this workspace inherit the patch transitively; modules built standalone against crates.io will see `alpha.3` and may hit the very bugs the patch closes — flag in the SDK README when M3 lands. diff --git a/docs/adr/0003-local-store-namespacing.md b/docs/adr/0003-local-store-namespacing.md index ee5be1e..f265137 100644 --- a/docs/adr/0003-local-store-namespacing.md +++ b/docs/adr/0003-local-store-namespacing.md @@ -7,51 +7,27 @@ implemented-in: nullislabs/shepherd#8 ## Context -`nexum:host/local-store` is a key-value store shared across all modules -the engine runs. Two modules using the same key string (e.g. -`"last-block"`) must see disjoint values; one module must never read or -overwrite another's data. The engine knows each module's name at -instantiation time, so namespacing is a host-side concern. +`nexum:host/local-store` is a key-value store shared across all modules the engine runs. Two modules using the same key string (e.g. `"last-block"`) must see disjoint values; one module must never read or overwrite another's data. The engine knows each module's name at instantiation time, so namespacing is a host-side concern. ## Decision -Single redb database file at `EngineConfig.engine.state_dir`, single -shared table `nexum:local-store`. Every key handed to redb is composed -host-side as: +Single redb database file at `EngineConfig.engine.state_dir`, single shared table `nexum:local-store`. Every key handed to redb is composed host-side as: ``` [len: u8] [module_name: len bytes] [raw key: rest of the bytes] ``` -Module names longer than 255 bytes are rejected at `LocalStore` -construction (matches the one-byte length prefix). Modules see plain -key strings on both the read and write paths; the prefix is invisible -to the WIT-facing API. +Module names longer than 255 bytes are rejected at `LocalStore` construction (matches the one-byte length prefix). Modules see plain key strings on both the read and write paths; the prefix is invisible to the WIT-facing API. ## Considered options -- **Separator string** (`{module}:{key}`). Rejected: any module name - containing `:` collides with another module's `:`-bearing key. Length - prefix is unambiguous regardless of payload bytes. -- **One redb database file per module.** Rejected: multiplies open - file handles linearly in modules, blocks any future cross-module - atomic operations (not currently planned but cheap to keep on the - table), and complicates backup tooling (N files vs 1). -- **One redb *table* per module within a single file.** Rejected: redb - `TableDefinition` lifetimes are `'static`, so table names must be - known at compile time. Dynamic table opening per module would force - string-leak workarounds and exposes the same name-collision question - as separator-based keys. +- **Separator string** (`{module}:{key}`). Rejected: any module name containing `:` collides with another module's `:`-bearing key. Length prefix is unambiguous regardless of payload bytes. +- **One redb database file per module.** Rejected: multiplies open file handles linearly in modules, blocks any future cross-module atomic operations (not currently planned but cheap to keep on the table), and complicates backup tooling (N files vs 1). +- **One redb *table* per module within a single file.** Rejected: redb `TableDefinition` lifetimes are `'static`, so table names must be known at compile time. Dynamic table opening per module would force string-leak workarounds and exposes the same name-collision question as separator-based keys. ## Consequences -- Module data is physically interleaved in the redb tree (range scans - for one module's keys are O(log n + module-key-count) — fine for our - workload). -- Migrations changing the namespacing layout break every existing - module's persisted state. The format must stay stable through 0.x. -- A module's `list-keys` (when added) iterates over the namespace - range; the host strips the prefix before returning to the guest. -- 255-byte module-name limit is enforced loudly at construction, so - configuration errors surface at boot rather than silently corrupting - data at first write. +- Module data is physically interleaved in the redb tree (range scans for one module's keys are O(log n + module-key-count) — fine for our workload). +- Migrations changing the namespacing layout break every existing module's persisted state. The format must stay stable through 0.x. +- A module's `list-keys` (when added) iterates over the namespace range; the host strips the prefix before returning to the guest. +- 255-byte module-name limit is enforced loudly at construction, so configuration errors surface at boot rather than silently corrupting data at first write. diff --git a/docs/adr/0004-cow-api-via-cached-orderbookapi.md b/docs/adr/0004-cow-api-via-cached-orderbookapi.md index bf4fd1a..52d5b8f 100644 --- a/docs/adr/0004-cow-api-via-cached-orderbookapi.md +++ b/docs/adr/0004-cow-api-via-cached-orderbookapi.md @@ -7,59 +7,27 @@ implemented-in: nullislabs/shepherd#8 ## Context -`shepherd:cow/cow-api` exposes two operations: a generic REST passthrough -(`request`) and a typed order submission (`submit-order`). Either could -be implemented with raw `reqwest` against `api.cow.fi/{slug}/api/v1`, -but the published `cowprotocol` crate already ships an `OrderBookApi` -client that knows the chain-specific base URL, the canonical paths, and -the `post_order` codec. +`shepherd:cow/cow-api` exposes two operations: a generic REST passthrough (`request`) and a typed order submission (`submit-order`). Either could be implemented with raw `reqwest` against `api.cow.fi/{slug}/api/v1`, but the published `cowprotocol` crate already ships an `OrderBookApi` client that knows the chain-specific base URL, the canonical paths, and the `post_order` codec. ## Decision -At engine boot, construct one `cowprotocol::OrderBookApi` per -`cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, -ArbitrumOne, Base) into a `BTreeMap` keyed by EVM -chain id. Both `cow-api` operations consult this pool: +At engine boot, construct one `cowprotocol::OrderBookApi` per `cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, ArbitrumOne, Base) into a `BTreeMap` keyed by EVM chain id. Both `cow-api` operations consult this pool: -- `request` resolves the chain's `OrderBookApi`, reads - `api.base_url()` for the prefix, joins the module-supplied path, - and dispatches via a shared `reqwest::Client`. -- `submit-order` deserialises the JSON `OrderCreation` and calls - `OrderBookApi::post_order` directly. The crate handles signing-scheme - encoding, error mapping, and `OrderUid` extraction. +- `request` resolves the chain's `OrderBookApi`, reads `api.base_url()` for the prefix, joins the module-supplied path, and dispatches via a shared `reqwest::Client`. +- `submit-order` deserialises the JSON `OrderCreation` and calls `OrderBookApi::post_order` directly. The crate handles signing-scheme encoding, error mapping, and `OrderUid` extraction. -Chains not in `cowprotocol::Chain` return `HostError { kind: unsupported }` -at the host call boundary. +Chains not in `cowprotocol::Chain` return `HostError { kind: unsupported }` at the host call boundary. ## Considered options -- **Raw `reqwest` for both.** Rejected: forces us to maintain the - chain → base-URL table (drifts whenever cowprotocol adds a chain) and - reimplement `post_order`'s body codec and error mapping — the exact - duplication mfw78 called out in cow-rs PR #5. -- **`OrderBookApi` for `submit-order`, raw `reqwest` for `request`.** - Tempting (request is opaque to the crate) but means two separate - chain-resolution paths, two HTTP clients, and a second place to keep - the chain set in sync. -- **Build `OrderBookApi` lazily on first call per chain.** Rejected: - hides config errors at runtime. Up-front boot construction surfaces - unknown chains immediately and amortises away the per-call cost. +- **Raw `reqwest` for both.** Rejected: forces us to maintain the chain → base-URL table (drifts whenever cowprotocol adds a chain) and reimplement `post_order`'s body codec and error mapping — the exact duplication mfw78 called out in cow-rs PR #5. +- **`OrderBookApi` for `submit-order`, raw `reqwest` for `request`.** Tempting (request is opaque to the crate) but means two separate chain-resolution paths, two HTTP clients, and a second place to keep the chain set in sync. +- **Build `OrderBookApi` lazily on first call per chain.** Rejected: hides config errors at runtime. Up-front boot construction surfaces unknown chains immediately and amortises away the per-call cost. ## Consequences -- Operator-supplied custom orderbook URLs (barn, staging, forked - deployments) are out of scope for the default constructor and require - a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` - constructor in the cow-rs crate (ADR-0007, upstream item 4 — not - vendored locally). -- App-data resolution moves out of the engine entirely once ADR-0007 - item 3 lands: `OrderBookPool` exposes an `AppDataResolver` constructed - from the same per-chain `OrderBookApi` clients, and both `cow-api` - call sites and the new twap/ethflow helpers (ADR-0001) consume that - shared resolver. No engine-side LRU. -- Adding a chain means a `cowprotocol::Chain` variant lands in cow-rs - first; the engine inherits it on the next patched rev bump. -- The shared `reqwest::Client` enables connection pooling across both - `request` and `submit-order` paths. -- TWAP and EthFlow helpers (ADR-0001) reuse the same pool — no - duplicated client construction in those host wrappers. +- Operator-supplied custom orderbook URLs (barn, staging, forked deployments) are out of scope for the default constructor and require a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` constructor in the cow-rs crate (ADR-0007, upstream item 4 — not vendored locally). +- App-data resolution moves out of the engine entirely once ADR-0007 item 3 lands: `OrderBookPool` exposes an `AppDataResolver` constructed from the same per-chain `OrderBookApi` clients, and both `cow-api` call sites and the new twap/ethflow helpers (ADR-0001) consume that shared resolver. No engine-side LRU. +- Adding a chain means a `cowprotocol::Chain` variant lands in cow-rs first; the engine inherits it on the next patched rev bump. +- The shared `reqwest::Client` enables connection pooling across both `request` and `submit-order` paths. +- TWAP and EthFlow helpers (ADR-0001) reuse the same pool — no duplicated client construction in those host wrappers. diff --git a/docs/adr/0005-provider-pool-transport-by-scheme.md b/docs/adr/0005-provider-pool-transport-by-scheme.md index aba0a72..f027480 100644 --- a/docs/adr/0005-provider-pool-transport-by-scheme.md +++ b/docs/adr/0005-provider-pool-transport-by-scheme.md @@ -7,53 +7,26 @@ implemented-in: nullislabs/shepherd#8, nullislabs/shepherd#9 ## Context -`nexum:host/chain` covers both generic JSON-RPC dispatch (`request`) -and event subscriptions (`subscribe-blocks`, `subscribe-logs`). -Subscriptions require a duplex transport (`eth_subscribe` is push-only -over a long-lived connection); request/response works on either HTTP -or WebSocket. The operator configures one RPC endpoint per chain in -`engine.toml`; the engine has to decide which alloy transport to use. +`nexum:host/chain` covers both generic JSON-RPC dispatch (`request`) and event subscriptions (`subscribe-blocks`, `subscribe-logs`). Subscriptions require a duplex transport (`eth_subscribe` is push-only over a long-lived connection); request/response works on either HTTP or WebSocket. The operator configures one RPC endpoint per chain in `engine.toml`; the engine has to decide which alloy transport to use. ## Decision -The `ProviderPool::from_config` constructor reads each chain's -`rpc_url` and switches by URL scheme prefix: +The `ProviderPool::from_config` constructor reads each chain's `rpc_url` and switches by URL scheme prefix: -- `ws://` or `wss://` → `ProviderBuilder::new().connect_ws(WsConnect::new(url))`. - Pubsub transport. Subscriptions and request/response both work. -- `http://` or `https://` → `ProviderBuilder::new().connect_http(parsed)`. - HTTP transport. Request/response only; `subscribe-blocks` and - `subscribe-logs` surface as a host error to the guest. +- `ws://` or `wss://` → `ProviderBuilder::new().connect_ws(WsConnect::new(url))`. Pubsub transport. Subscriptions and request/response both work. +- `http://` or `https://` → `ProviderBuilder::new().connect_http(parsed)`. HTTP transport. Request/response only; `subscribe-blocks` and `subscribe-logs` surface as a host error to the guest. -Both transports erase to `DynProvider` so the rest of the engine is -transport-agnostic. +Both transports erase to `DynProvider` so the rest of the engine is transport-agnostic. ## Considered options -- **Force WSS everywhere.** Rejected: many providers (Alchemy, Infura, - self-hosted RPC) expose HTTP-only on free tiers, and modules that - only need `request` (no subscriptions) shouldn't be blocked by a - WSS requirement. -- **Explicit `transport = "ws" | "http"` field per chain in - `engine.toml`.** Rejected for 0.2: redundant with the URL scheme, - and operators already distinguish `wss://` from `https://` - endpoints when copying them from their RPC provider's dashboard. - Revisit if we add IPC (`/path/to/geth.ipc`) — scheme alone won't - carry that. -- **Open both an HTTP and a WSS connection per chain.** Rejected: doubles - connection count for the common case where one endpoint serves - both, and forces operators to provide two URLs even when their - provider returns identical data on both. +- **Force WSS everywhere.** Rejected: many providers (Alchemy, Infura, self-hosted RPC) expose HTTP-only on free tiers, and modules that only need `request` (no subscriptions) shouldn't be blocked by a WSS requirement. +- **Explicit `transport = "ws" | "http"` field per chain in `engine.toml`.** Rejected for 0.2: redundant with the URL scheme, and operators already distinguish `wss://` from `https://` endpoints when copying them from their RPC provider's dashboard. Revisit if we add IPC (`/path/to/geth.ipc`) — scheme alone won't carry that. +- **Open both an HTTP and a WSS connection per chain.** Rejected: doubles connection count for the common case where one endpoint serves both, and forces operators to provide two URLs even when their provider returns identical data on both. ## Consequences -- Operators that need subscriptions must supply WSS URLs; HTTP-only - chains downgrade to request-only mode at the host call boundary. -- Connection failures at boot are fatal (the engine refuses to start - with a broken chain). This is intentional — silent fall-back to a - half-functioning state masks misconfiguration that a module then - rediscovers at first event. -- Adding IPC support is additive: extend the scheme match with - `/` / `file://` and call `connect_ipc`. -- The `DynProvider` erasure costs a virtual dispatch per call — a - measurable concern at scale, deferred to M4 if profiling shows it. +- Operators that need subscriptions must supply WSS URLs; HTTP-only chains downgrade to request-only mode at the host call boundary. +- Connection failures at boot are fatal (the engine refuses to start with a broken chain). This is intentional — silent fall-back to a half-functioning state masks misconfiguration that a module then rediscovers at first event. +- Adding IPC support is additive: extend the scheme match with `/` / `file://` and call `connect_ipc`. +- The `DynProvider` erasure costs a virtual dispatch per call — a measurable concern at scale, deferred to M4 if profiling shows it. diff --git a/docs/adr/0006-engine-toml-separate-from-nexum-toml.md b/docs/adr/0006-engine-toml-separate-from-nexum-toml.md index 57f0dd7..fd77ac3 100644 --- a/docs/adr/0006-engine-toml-separate-from-nexum-toml.md +++ b/docs/adr/0006-engine-toml-separate-from-nexum-toml.md @@ -7,59 +7,26 @@ implemented-in: nullislabs/shepherd#8, nullislabs/shepherd#9 ## Context -The engine needs two distinct kinds of configuration: what the -**operator** decides at deployment time (which chains to connect to, -where the local-store database lives, which modules to boot) and -what the **module developer** declares at build time (required and -optional capabilities, HTTP allowlist, module-specific config keys). -These have different reviewers, different threat models, and change -on different cadences. +The engine needs two distinct kinds of configuration: what the **operator** decides at deployment time (which chains to connect to, where the local-store database lives, which modules to boot) and what the **module developer** declares at build time (required and optional capabilities, HTTP allowlist, module-specific config keys). These have different reviewers, different threat models, and change on different cadences. ## Decision Two distinct files, distinct schemas, distinct loaders: -- **`engine.toml`** — operator-owned, lives next to the engine binary - or pointed to by `--engine-config`. Defines `[engine]` (state_dir, - log_level), `[chains.]` (rpc_url), and `[[modules]]` (path, - manifest). Loaded by `engine_config::EngineConfig::load`. -- **`nexum.toml`** — module-developer-owned, ships in the module's - bundle alongside its `.wasm` component. Defines `[module]`, - `[capabilities]` (required, optional, http allowlist), `[config]`. - Loaded by `manifest::load`. +- **`engine.toml`** — operator-owned, lives next to the engine binary or pointed to by `--engine-config`. Defines `[engine]` (state_dir, log_level), `[chains.]` (rpc_url), and `[[modules]]` (path, manifest). Loaded by `engine_config::EngineConfig::load`. +- **`nexum.toml`** — module-developer-owned, ships in the module's bundle alongside its `.wasm` component. Defines `[module]`, `[capabilities]` (required, optional, http allowlist), `[config]`. Loaded by `manifest::load`. -The engine config carries the path to each module's manifest; the -two never collapse into one file. +The engine config carries the path to each module's manifest; the two never collapse into one file. ## Considered options -- **Single `shepherd.toml` with `[engine]`, `[chains]`, `[[modules]]` - *and* nested `[modules..capabilities]` per module.** Rejected: - conflates operator and developer concerns. A module's capability - declaration is a property of the build, not the deployment — it - belongs in the artifact, not in the operator's local file. Auditing - a module's capabilities also becomes a per-deployment exercise - instead of a property visible in the published bundle. -- **`nexum.toml` inside the engine config (module entries embed it - inline).** Rejected for the same reason; also bloats `engine.toml`. -- **Drop `engine.toml` entirely; pass everything as CLI flags or - env vars.** Rejected: per-chain RPC URLs and module lists are - awkward as flags, and `RUST_LOG` already covers the only thing - that env vars naturally express. +- **Single `shepherd.toml` with `[engine]`, `[chains]`, `[[modules]]` *and* nested `[modules..capabilities]` per module.** Rejected: conflates operator and developer concerns. A module's capability declaration is a property of the build, not the deployment — it belongs in the artifact, not in the operator's local file. Auditing a module's capabilities also becomes a per-deployment exercise instead of a property visible in the published bundle. +- **`nexum.toml` inside the engine config (module entries embed it inline).** Rejected for the same reason; also bloats `engine.toml`. +- **Drop `engine.toml` entirely; pass everything as CLI flags or env vars.** Rejected: per-chain RPC URLs and module lists are awkward as flags, and `RUST_LOG` already covers the only thing that env vars naturally express. ## Consequences -- A deployment needs both files. A missing `engine.toml` falls back - to "no chains, default state_dir" — the example logging module - still runs; cow-api / chain backends report `unsupported`. -- A missing `nexum.toml` triggers the 0.1-compat deprecation warning - in `manifest::fallback_manifest()` (defined in - `crates/nexum-engine/src/manifest.rs`) and treats every linked - capability as required. This fallback is scheduled for removal in - 0.3 per `docs/migration/0.1-to-0.2.md`. -- Module-bundle redistribution carries `nexum.toml` with the - artifact; engines do not need to ship templates. -- Future content-addressed module distribution (0.3) embeds - `nexum.toml` in the bundle hash; `engine.toml` references modules - by content address rather than filesystem path. The split survives - that migration unchanged. +- A deployment needs both files. A missing `engine.toml` falls back to "no chains, default state_dir" — the example logging module still runs; cow-api / chain backends report `unsupported`. +- A missing `nexum.toml` triggers the 0.1-compat deprecation warning in `manifest::fallback_manifest()` (defined in `crates/nexum-engine/src/manifest.rs`) and treats every linked capability as required. This fallback is scheduled for removal in 0.3 per `docs/migration/0.1-to-0.2.md`. +- Module-bundle redistribution carries `nexum.toml` with the artifact; engines do not need to ship templates. +- Future content-addressed module distribution (0.3) embeds `nexum.toml` in the bundle hash; `engine.toml` references modules by content address rather than filesystem path. The split survives that migration unchanged. diff --git a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md index e60c895..6ed2a36 100644 --- a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md +++ b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md @@ -6,82 +6,34 @@ status: proposed ## Context -Implementing ADR-0001 (twap + ethflow host helpers) and ADR-0004 (cow-api -backend) surfaces a recurring question: when the engine needs a piece of -CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose -(TWAP polling glue, EthFlow log decoding, app-data hash-to-document -resolution, custom orderbook URLs), do we write that logic locally in -`nexum-engine` and tidy it up upstream later, or do we open the cow-rs -PR first and only land the engine wiring afterwards? +Implementing ADR-0001 (twap + ethflow host helpers) and ADR-0004 (cow-api backend) surfaces a recurring question: when the engine needs a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (TWAP polling glue, EthFlow log decoding, app-data hash-to-document resolution, custom orderbook URLs), do we write that logic locally in `nexum-engine` and tidy it up upstream later, or do we open the cow-rs PR first and only land the engine wiring afterwards? -mfw78's review of cow-rs PR #5 named the failure mode explicitly: -duplicating work that an existing crate could do is the AI-coding -anti-pattern most likely to land in a Bleu PR. The same risk applies to -any engine-side reimplementation of protocol logic. +mfw78's review of cow-rs PR #5 named the failure mode explicitly: duplicating work that an existing crate could do is the AI-coding anti-pattern most likely to land in a Bleu PR. The same risk applies to any engine-side reimplementation of protocol logic. ## Decision -Protocol-level CoW logic — anything that an indexer, a bot, or a -non-`nexum` Rust consumer of CoW Protocol would also need — lands in -`bleu/cow-rs` first as an upstream PR, and is consumed by `nexum-engine` -via the existing `[patch.crates-io]` rev bump (ADR-0002). The engine -never writes throwaway local copies of the same logic with the intent -to "port later". +Protocol-level CoW logic — anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need — lands in `bleu/cow-rs` first as an upstream PR, and is consumed by `nexum-engine` via the existing `[patch.crates-io]` rev bump (ADR-0002). The engine never writes throwaway local copies of the same logic with the intent to "port later". The concrete set of primitives we know we need is, in priority order: -1. `cowprotocol::composable::poll_and_build_order(provider, owner, - params, proof) -> Result` — eth_call against - `ComposableCoW.getTradeableOrderWithSignature`, decode return, - rebuild `OrderCreation`. Backs `twap.poll-and-submit`. -2. `cowprotocol::eth_flow::decode_placement(log) -> - Result` — decode `OrderPlacement` event log, - reconstruct `OrderCreation` and `OrderUid`. Backs - `ethflow.submit-from-log`. -3. `cowprotocol::app_data::OrderBookAppDataResolver` — `AppDataResolver` - trait + cached implementation around `OrderBookApi::app_data(hash)`, - with `EMPTY_APP_DATA_HASH` fast-path. Used by twap, ethflow, and any - future caller that needs to materialise an app-data document. -4. `cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)` — - custom-URL constructor for barn / staging / forked deployments. -5. `cowprotocol` `wasm32` compatibility — feature-gate the `reqwest` - dependency so guest modules can use the pure types - (`Order`, `OrderCreation`, `OrderUid`, `composable::*`, - `eth_flow::decode_*`) without dragging in an HTTP client. +1. `cowprotocol::composable::poll_and_build_order(provider, owner, params, proof) -> Result` — eth_call against `ComposableCoW.getTradeableOrderWithSignature`, decode return, rebuild `OrderCreation`. Backs `twap.poll-and-submit`. +2. `cowprotocol::eth_flow::decode_placement(log) -> Result` — decode `OrderPlacement` event log, reconstruct `OrderCreation` and `OrderUid`. Backs `ethflow.submit-from-log`. +3. `cowprotocol::app_data::OrderBookAppDataResolver` — `AppDataResolver` trait + cached implementation around `OrderBookApi::app_data(hash)`, with `EMPTY_APP_DATA_HASH` fast-path. Used by twap, ethflow, and any future caller that needs to materialise an app-data document. +4. `cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)` — custom-URL constructor for barn / staging / forked deployments. +5. `cowprotocol` `wasm32` compatibility — feature-gate the `reqwest` dependency so guest modules can use the pure types (`Order`, `OrderCreation`, `OrderUid`, `composable::*`, `eth_flow::decode_*`) without dragging in an HTTP client. -Lower-priority follow-ons (richer `OrderBookApiError` variants, -`OrderUid::from_slice`, retry middleware, `OrderCreation::from_gpv2`) -are good-to-have but are not blocking for the M2 host scope. +Lower-priority follow-ons (richer `OrderBookApiError` variants, `OrderUid::from_slice`, retry middleware, `OrderCreation::from_gpv2`) are good-to-have but are not blocking for the M2 host scope. ## Considered options -- **Implement locally, refactor upstream later.** Faster short term but - predictably leaves an indeterminate amount of duplicated logic in the - engine, contradicts mfw78's stated conventions, and grows technical - debt every time cow-rs evolves the underlying types. Rejected. -- **Wait for cow-rs upstream maintainers to add these on their own.** - No evidence anyone else is doing this work; the grant timeline does - not permit waiting. -- **Vendor a fork of cow-rs inside `nullislabs/shepherd`.** Worst of all - worlds: blocks neither the engine nor cow-rs from drifting, and - forces every other CoW consumer to re-derive the same primitives. +- **Implement locally, refactor upstream later.** Faster short term but predictably leaves an indeterminate amount of duplicated logic in the engine, contradicts mfw78's stated conventions, and grows technical debt every time cow-rs evolves the underlying types. Rejected. +- **Wait for cow-rs upstream maintainers to add these on their own.** No evidence anyone else is doing this work; the grant timeline does not permit waiting. +- **Vendor a fork of cow-rs inside `nullislabs/shepherd`.** Worst of all worlds: blocks neither the engine nor cow-rs from drifting, and forces every other CoW consumer to re-derive the same primitives. ## Consequences -- Every M2 engine issue that consumes one of the five primitives above - is blocked on its cow-rs PR merging. We sequence issues so that - upstream PRs and engine adoption can land in parallel where possible - (e.g., open all three protocol-helper PRs against `bleu/cow-rs` - simultaneously rather than serially). -- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0002) is - bumped after each cow-rs merge; the bump is the engine's signal that a - new primitive is consumable. -- PRs in `bleu/cow-rs` follow the existing mfw78 conventions established - by cow-rs PR #5: severity-tagged reviews, alloy reuse over local - reimplementation, GPL-3.0, edition 2024, terse rustdoc. -- After acceptance in `bleu/cow-rs`, each primitive is also surfaced as - a PR (or backport) against `cowdao-grants/cow-rs` so the wider - ecosystem benefits and the bleu fork narrows over time. -- The engine repo stays small: `nexum-engine` contains WIT, host - wiring, supervisor, redb store, alloy provider pool, and `engine.toml` - schema — nothing about CoW Protocol semantics. +- Every M2 engine issue that consumes one of the five primitives above is blocked on its cow-rs PR merging. We sequence issues so that upstream PRs and engine adoption can land in parallel where possible (e.g., open all three protocol-helper PRs against `bleu/cow-rs` simultaneously rather than serially). +- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0002) is bumped after each cow-rs merge; the bump is the engine's signal that a new primitive is consumable. +- PRs in `bleu/cow-rs` follow the existing mfw78 conventions established by cow-rs PR #5: severity-tagged reviews, alloy reuse over local reimplementation, GPL-3.0, edition 2024, terse rustdoc. +- After acceptance in `bleu/cow-rs`, each primitive is also surfaced as a PR (or backport) against `cowdao-grants/cow-rs` so the wider ecosystem benefits and the bleu fork narrows over time. +- The engine repo stays small: `nexum-engine` contains WIT, host wiring, supervisor, redb store, alloy provider pool, and `engine.toml` schema — nothing about CoW Protocol semantics. From 70423429460571292ed0e458b40a406d4a3a2f00 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Wed, 3 Jun 2026 15:36:59 -0300 Subject: [PATCH 20/29] docs(adr): revise CoW design and reorder ADRs (0001-0008) --- .../adr/0001-cow-twap-ethflow-host-helpers.md | 64 -------- ...1-engine-toml-separate-from-nexum-toml.md} | 0 ...0002-provider-pool-transport-by-scheme.md} | 0 ... 0004-patch-cowprotocol-to-bleu-cow-rs.md} | 2 +- ...> 0005-cow-api-via-cached-orderbookapi.md} | 5 +- .../adr/0006-cow-twap-ethflow-host-helpers.md | 84 +++++++++++ .../0007-upstream-protocol-logic-to-cow-rs.md | 29 ++-- .../0008-factory-subscriptions-in-manifest.md | 140 ++++++++++++++++++ 8 files changed, 246 insertions(+), 78 deletions(-) delete mode 100644 docs/adr/0001-cow-twap-ethflow-host-helpers.md rename docs/adr/{0006-engine-toml-separate-from-nexum-toml.md => 0001-engine-toml-separate-from-nexum-toml.md} (100%) rename docs/adr/{0005-provider-pool-transport-by-scheme.md => 0002-provider-pool-transport-by-scheme.md} (100%) rename docs/adr/{0002-patch-cowprotocol-to-bleu-cow-rs.md => 0004-patch-cowprotocol-to-bleu-cow-rs.md} (94%) rename docs/adr/{0004-cow-api-via-cached-orderbookapi.md => 0005-cow-api-via-cached-orderbookapi.md} (84%) create mode 100644 docs/adr/0006-cow-twap-ethflow-host-helpers.md create mode 100644 docs/adr/0008-factory-subscriptions-in-manifest.md diff --git a/docs/adr/0001-cow-twap-ethflow-host-helpers.md b/docs/adr/0001-cow-twap-ethflow-host-helpers.md deleted file mode 100644 index 384e368..0000000 --- a/docs/adr/0001-cow-twap-ethflow-host-helpers.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -status: proposed ---- - -# TWAP and EthFlow as intent helpers in `shepherd:cow@0.2.0` - -## Context - -The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook access (REST passthrough + `submit-order`). Two further CoW workflows show up in every non-trivial module: ComposableCoW conditional orders (TWAP being the canonical example) and EthFlow native-ETH orders. Both share a tight pattern — observe an on-chain event, derive a signed `OrderCreation`, submit it to the orderbook — but the derivation has enough protocol detail (digest, signature scheme, app-data resolution, `getTradeableOrderWithSignature` eth_call against ComposableCoW) that a guest module would either ship that logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or make ten round-trips to the host through generic `chain`/`cow-api` calls. - -The protocol logic itself — TWAP polling, EthFlow log decoding, app-data resolution — is not engine-specific. Every Rust consumer of CoW Protocol (indexers, bots, this engine) needs the same primitives. Per ADR-0007, those primitives belong in the `cowprotocol` crate, not in `nexum-engine`. This ADR consequently scopes the engine-side helpers to the WIT surface and the glue that wires the upstream primitives into the host call boundary. - -## Decision - -Add two new interfaces to package `shepherd:cow@0.2.0`: - -```wit -interface twap { - use nexum:host/types@0.2.0.{chain-id, log, host-error}; - use cow-api.{order-uid}; - poll-and-submit: func( - chain-id: chain-id, - registration: log, - ) -> result, host-error>; -} - -interface ethflow { - use nexum:host/types@0.2.0.{chain-id, log, host-error}; - use cow-api.{order-uid}; - submit-from-log: func( - chain-id: chain-id, - placement: log, - ) -> result; -} -``` - -Both interfaces ship in the existing `shepherd` world alongside `cow-api`. `order-uid` is added to `cow-api` as `type order-uid = list` (56 bytes, validated host-side) and reused by all three interfaces; `cow-api/submit-order` keeps returning it instead of `string`. Capability names `"twap"` and `"ethflow"` are appended to `KNOWN_CAPABILITIES` so manifests can declare them under `[capabilities].required`. - -Host implementations are thin wrappers (~20–30 LOC each) over three upstream primitives that land in `cowprotocol` first (see ADR-0007): - -- `cowprotocol::composable::poll_and_build_order(provider, owner, params, proof)` — returns `Ready(OrderCreation, signature)` or `NotReady` on contract revert. Backs `twap.poll-and-submit`. -- `cowprotocol::eth_flow::decode_placement(log)` — returns `(owner, OrderCreation, OrderUid)` from an `OrderPlacement` event log. Backs `ethflow.submit-from-log`. -- `cowprotocol::app_data::OrderBookAppDataResolver` — given a chain id and a `bytes32` hash, returns the JSON document (with `EMPTY_APP_DATA_HASH` fast-path and LRU cache built in). Used by both helpers and any future module-facing path. - -The engine wires these primitives into HostState and maps their errors to `host-error` kinds; no protocol logic lives in `nexum-engine`. Modules continue to declare their own log subscriptions via `[[subscription]]` in `nexum.toml`; the helpers only decode and submit, they do not auto-subscribe. - -## Considered options - -- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW glue. mfw78's "reuse over reimplement" applied: that derivation already lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and should not be re-shipped in every WASM artifact. -- **Implement the protocol glue inside `nexum-engine` host code, port upstream later.** Rejected per ADR-0007: every line of TWAP polling or EthFlow decoding that lives in the engine is a line that future Rust consumers cannot reuse, and a line that diverges as cow-rs evolves. -- **Single combined interface** `shepherd:cow/orders` with both helpers. Cheaper world surface but harder to gate per-capability — a module that only watches EthFlow shouldn't have to import TWAP and vice versa. Splitting keeps `[capabilities].required` honest. -- **Symmetric `result, host-error>` for both.** TWAP and EthFlow are genuinely asymmetric: TWAP is poll-driven and a `None` ("not tradeable yet") is the normal steady-state; EthFlow is event-driven and every accepted log produces exactly one UID. Forcing symmetry obscures semantics for callers. -- **`log-json: list` payload** instead of the typed `nexum:host/types.log` record. The record already exists and the engine's event dispatch already projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both ergonomics and "no duplicate decoders". -- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 helper only handles `ComposableCoW.create()` (empty proof, single conditional order). `setRoot` polling requires off-chain proof derivation that itself warrants a separate helper (`twap.poll-and-submit-with-proof`) once a module actually needs it. -- **Bumping the package to `shepherd:cow@0.3.0`.** Not needed: adding imports to an existing world is additive under WIT subsumption rules. Modules compiled against the current 0.2.0 surface continue to build. - -## Consequences - -- `cow-api/submit-order` return type changes from `string` to `order-uid`. No external consumers today (0.2 is unreleased), so this is internal. -- Host helpers require a chain to be configured in `[chains.]` — uncovered chains return `host-error.unsupported`. Same posture as `cow-api`. -- Orderbook idempotency (same UID on duplicate submit) is preserved but invisible to the module. Modules that need dedup must record UIDs in `local-store` themselves. -- App-data resolution adds a GET to `api.cow.fi/{chain}/api/v1/app_data/{hash}` on the first sighting of a non-empty hash. The LRU cache and the GET itself live in `cowprotocol::app_data::OrderBookAppDataResolver` (ADR-0007 item 3); cache miss + orderbook miss surfaces as `host-error.unavailable`. -- Implementation order: the three `cowprotocol` primitives (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `app_data::OrderBookAppDataResolver`) land in `bleu/cow-rs` first; `nullis-shepherd` adopts via the existing `[patch.crates-io]` rev bump (ADR-0002). Host-side issues stay blocked on upstream merges. -- Failure modes map onto existing `host-error-kind` variants (`invalid-input`, `denied`, `rate-limited`, `timeout`, `unavailable`, `unsupported`, `internal`). No new error taxonomy. diff --git a/docs/adr/0006-engine-toml-separate-from-nexum-toml.md b/docs/adr/0001-engine-toml-separate-from-nexum-toml.md similarity index 100% rename from docs/adr/0006-engine-toml-separate-from-nexum-toml.md rename to docs/adr/0001-engine-toml-separate-from-nexum-toml.md diff --git a/docs/adr/0005-provider-pool-transport-by-scheme.md b/docs/adr/0002-provider-pool-transport-by-scheme.md similarity index 100% rename from docs/adr/0005-provider-pool-transport-by-scheme.md rename to docs/adr/0002-provider-pool-transport-by-scheme.md diff --git a/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md b/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md similarity index 94% rename from docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md rename to docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md index 380b632..4564401 100644 --- a/docs/adr/0002-patch-cowprotocol-to-bleu-cow-rs.md +++ b/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md @@ -31,5 +31,5 @@ Add a workspace-level `[patch.crates-io]` redirecting `cowprotocol` to `https:// - `cargo update` will re-resolve to the same `rev` — the lock pins it. - Bumping the rev is a single-line workspace edit; reviewers see one diff. -- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `app_data::OrderBookAppDataResolver`). Until then, expect the patch rev to advance with every cow-rs merge that the engine consumes. +- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `OrderPostError` rich variants). Until then, expect the patch rev to advance with every cow-rs merge that the engine consumes. - Modules built against this workspace inherit the patch transitively; modules built standalone against crates.io will see `alpha.3` and may hit the very bugs the patch closes — flag in the SDK README when M3 lands. diff --git a/docs/adr/0004-cow-api-via-cached-orderbookapi.md b/docs/adr/0005-cow-api-via-cached-orderbookapi.md similarity index 84% rename from docs/adr/0004-cow-api-via-cached-orderbookapi.md rename to docs/adr/0005-cow-api-via-cached-orderbookapi.md index 52d5b8f..ae9c6b5 100644 --- a/docs/adr/0004-cow-api-via-cached-orderbookapi.md +++ b/docs/adr/0005-cow-api-via-cached-orderbookapi.md @@ -26,8 +26,7 @@ Chains not in `cowprotocol::Chain` return `HostError { kind: unsupported }` at t ## Consequences -- Operator-supplied custom orderbook URLs (barn, staging, forked deployments) are out of scope for the default constructor and require a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` constructor in the cow-rs crate (ADR-0007, upstream item 4 — not vendored locally). -- App-data resolution moves out of the engine entirely once ADR-0007 item 3 lands: `OrderBookPool` exposes an `AppDataResolver` constructed from the same per-chain `OrderBookApi` clients, and both `cow-api` call sites and the new twap/ethflow helpers (ADR-0001) consume that shared resolver. No engine-side LRU. +- Operator-supplied custom orderbook URLs (barn, staging, forked deployments) are out of scope for the default constructor and require a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` constructor in the cow-rs crate (ADR-0007 item 4 — not vendored locally). - Adding a chain means a `cowprotocol::Chain` variant lands in cow-rs first; the engine inherits it on the next patched rev bump. - The shared `reqwest::Client` enables connection pooling across both `request` and `submit-order` paths. -- TWAP and EthFlow helpers (ADR-0001) reuse the same pool — no duplicated client construction in those host wrappers. +- TWAP and EthFlow helpers (ADR-0006) reuse the same pool — no duplicated client construction in those host wrappers. diff --git a/docs/adr/0006-cow-twap-ethflow-host-helpers.md b/docs/adr/0006-cow-twap-ethflow-host-helpers.md new file mode 100644 index 0000000..8870d3d --- /dev/null +++ b/docs/adr/0006-cow-twap-ethflow-host-helpers.md @@ -0,0 +1,84 @@ +--- +status: proposed +--- + +# TWAP and EthFlow as intent helpers in `shepherd:cow@0.2.0` + +## Context + +The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook access (REST passthrough + `submit-order`). Two further CoW workflows show up in every non-trivial module: ComposableCoW conditional orders (TWAP being the canonical example) and EthFlow native-ETH orders. Both follow the same external-indexer/relayer pattern that CoW's own infrastructure has been progressively extracting from the monolithic `cowprotocol/services` repository: + +- **TWAP / ComposableCoW** — extracted as the standalone `cowprotocol/watch-tower` (TypeScript). Listens to `ConditionalOrderCreated`, polls `getTradeableOrderWithSignature` on each block, posts to the orderbook when an order becomes tradeable. +- **EthFlow indexer** — still inside `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`. Listens to `OrderPlacement` / `OrderInvalidation` / `OrderRefund`, inserts into the `ethflow_orders` table. CoW's architectural direction is to extract this into a standalone service, mirroring how `watch-tower` and the `refunder` crate were already extracted. The Shepherd `ethflow-watcher` module is the natural Rust home for that extraction. + +Both flows share the same pattern: observe an on-chain event, derive a signed `OrderCreation`, submit it to the orderbook. The derivation has enough protocol detail (signing scheme, ComposableCoW eth_call, EthFlow EIP-1271 contract signature, log decoding) that a guest module would either ship that logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or make ten round-trips to the host through generic `chain`/`cow-api` calls. + +Per ADR-0007, the protocol logic itself lives in the `cowprotocol` crate — not in `nexum-engine`. This ADR consequently scopes the engine-side helpers to the WIT surface and the glue that wires the upstream primitives into the host call boundary. + +## Decision + +Add two new interfaces to package `shepherd:cow@0.2.0`: + +```wit +interface twap { + use nexum:host/types@0.2.0.{chain-id, log, host-error}; + use cow-api.{order-uid}; + + /// Discriminated outcome of a single poll attempt against + /// ComposableCoW. Mirrors watchtower's PollResultCode so modules + /// avoid spamming RPC/orderbook when an order is known-not-ready. + variant poll-outcome { + submitted(order-uid), + try-at-epoch(u64), // unix seconds; module skips polls until then + try-on-block(u64), // specific block number + try-next-block, // default retry + dont-try-again, // terminal: TWAP completed or cancelled + } + + poll-and-submit: func( + chain-id: chain-id, + registration: log, + ) -> result; +} + +interface ethflow { + use nexum:host/types@0.2.0.{chain-id, log, host-error}; + use cow-api.{order-uid}; + + submit-from-log: func( + chain-id: chain-id, + placement: log, + ) -> result; +} +``` + +Both interfaces ship in the existing `shepherd` world alongside `cow-api`. `order-uid` is added to `cow-api` as `type order-uid = list` (56 bytes, validated host-side) and reused by all three interfaces; `cow-api/submit-order` keeps returning it instead of `string`. Capability names `"twap"` and `"ethflow"` are appended to `KNOWN_CAPABILITIES` so manifests can declare them under `[capabilities].required`. + +Host implementations are thin wrappers (~20–30 LOC each) over three upstream primitives that land in `cowprotocol` first (see ADR-0007): + +- `cowprotocol::composable::poll_and_build_order(provider, owner, params, proof) -> PollOutcome` — returns the same discriminated outcome (`Submitted`, `TryAtEpoch`, `TryOnBlock`, `TryNextBlock`, `DontTryAgain`). Backs `twap.poll-and-submit`. +- `cowprotocol::eth_flow::decode_placement(log)` — decodes `OrderPlacement` into `(owner, OrderCreation, OrderUid)` with the EIP-1271 signing scheme pointing at the EthFlow contract. Backs `ethflow.submit-from-log`. +- `cowprotocol::OrderPostError` (rich variants + `retry_hint()`) — typed orderbook submission errors with backoff/drop classification. Modules consume the hints to react to transient vs permanent failures without spamming. + +The engine wires these primitives into HostState and maps their errors to `host-error` kinds; no protocol logic lives in `nexum-engine`. Modules continue to declare their own log subscriptions via `[[subscription]]` in `nexum.toml`; the helpers only decode and submit, they do not auto-subscribe. + +## Considered options + +- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW / EthFlow glue. mfw78's "reuse over reimplement" applied: that derivation already lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and should not be re-shipped in every WASM artifact. +- **Implement the protocol glue inside `nexum-engine` host code, port upstream later.** Rejected per ADR-0007: every line of TWAP polling or EthFlow decoding that lives in the engine is a line that future Rust consumers cannot reuse, and a line that diverges as cow-rs evolves. +- **EthFlow as pure passive observer (no `submit-from-log`).** Briefly considered after reading "watcher" / "monitor" in mfw78's docs/00 and docs/04 as "no submission". Rejected after verifying that CoW's own autopilot DOES post equivalent (insert into `ethflow_orders` table); the Shepherd module is intended to externalize that role, not replace it with passive observation. The `pending_orders` state mentioned in docs/04 is a side-effect of the relay (local accounting of what's been observed), not the goal. +- **Simple `option` return on twap instead of `poll-outcome` variant.** A 1-hour-spaced TWAP polled every block would spam ~300 RPC calls per part with `None` returns. The richer outcome (`try-at-epoch`, etc.) matches watchtower's existing `PollResult` and lets modules skip polls until the contract says it's worth retrying. Production-critical. +- **Single combined interface** `shepherd:cow/orders` with both helpers. Cheaper world surface but harder to gate per-capability — a module that only watches EthFlow shouldn't have to import TWAP and vice versa. Splitting keeps `[capabilities].required` honest. +- **`log-json: list` payload** instead of the typed `nexum:host/types.log` record. The record already exists and the engine's event dispatch already projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both ergonomics and "no duplicate decoders". +- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 helper only handles `ComposableCoW.create()` (empty proof, single conditional order). `setRoot` polling requires off-chain proof derivation that itself warrants a separate helper (`twap.poll-and-submit-with-proof`) once a module actually needs it. +- **Bumping the package to `shepherd:cow@0.3.0`.** Not needed: adding imports to an existing world is additive under WIT subsumption rules. Modules compiled against the current 0.2.0 surface continue to build. + +## Consequences + +- `cow-api/submit-order` return type changes from `string` to `order-uid`. No external consumers today (0.2 is unreleased), so this is internal. +- Host helpers require a chain to be configured in `[chains.]` — uncovered chains return `host-error.unsupported`. Same posture as `cow-api`. +- Orderbook idempotency (same UID on duplicate submit) is preserved but invisible to the module. Modules that need dedup must record UIDs in `local-store` themselves. +- TWAP modules must implement the `poll-outcome` state machine: persist `next_attempt` hints (epoch or block number) in local-store, skip polls until trigger, remove watches on `dont-try-again`. Without this, the poll loop becomes O(blocks × twaps) with most calls wasted. The M3 SDK is expected to ship a helper that encapsulates the state machine. +- Orderbook errors return as `host-error` with the original CoW error code in `code`. Modules use `OrderPostError::try_from(host_error)` plus `retry_hint()` (ADR-0007 item 3) to map to next-block / backoff / drop. Without this layered approach, modules spam the orderbook with permanently-broken orders. +- Implementation order: the three `cowprotocol` primitives (`composable::poll_and_build_order` with rich `PollOutcome`, `eth_flow::decode_placement`, `OrderPostError` rich + `retry_hint`) land in `bleu/cow-rs` first; `nullis-shepherd` adopts via the existing `[patch.crates-io]` rev bump (ADR-0004). Host-side issues stay blocked on upstream merges. +- Failure modes map onto existing `host-error-kind` variants (`invalid-input`, `denied`, `rate-limited`, `timeout`, `unavailable`, `unsupported`, `internal`). No new error taxonomy. diff --git a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md index 6ed2a36..dac09ea 100644 --- a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md +++ b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md @@ -6,34 +6,43 @@ status: proposed ## Context -Implementing ADR-0001 (twap + ethflow host helpers) and ADR-0004 (cow-api backend) surfaces a recurring question: when the engine needs a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (TWAP polling glue, EthFlow log decoding, app-data hash-to-document resolution, custom orderbook URLs), do we write that logic locally in `nexum-engine` and tidy it up upstream later, or do we open the cow-rs PR first and only land the engine wiring afterwards? +Implementing ADR-0006 (twap + ethflow host helpers) and ADR-0005 (cow-api backend) surfaces a recurring question: when the engine needs a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (TWAP polling glue, EthFlow log decoding, rich orderbook error variants, custom orderbook URLs), do we write that logic locally in `nexum-engine` and tidy it up upstream later, or do we open the cow-rs PR first and only land the engine wiring afterwards? mfw78's review of cow-rs PR #5 named the failure mode explicitly: duplicating work that an existing crate could do is the AI-coding anti-pattern most likely to land in a Bleu PR. The same risk applies to any engine-side reimplementation of protocol logic. +CoW's broader architecture has been moving the same direction: `watch-tower` extracted from `cowprotocol/services` autopilot, the `refunder` crate likewise, with the `ethflow_events` indexer (`crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`) identified as the next extraction target. The Rust-side equivalent of those extractions is the right home for protocol primitives — `bleu/cow-rs` (then upstream into `cowdao-grants/cow-rs`), not the engine. + ## Decision -Protocol-level CoW logic — anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need — lands in `bleu/cow-rs` first as an upstream PR, and is consumed by `nexum-engine` via the existing `[patch.crates-io]` rev bump (ADR-0002). The engine never writes throwaway local copies of the same logic with the intent to "port later". +Protocol-level CoW logic — anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need — lands in `bleu/cow-rs` first as an upstream PR, and is consumed by `nexum-engine` via the existing `[patch.crates-io]` rev bump (ADR-0004). The engine never writes throwaway local copies of the same logic with the intent to "port later". The concrete set of primitives we know we need is, in priority order: -1. `cowprotocol::composable::poll_and_build_order(provider, owner, params, proof) -> Result` — eth_call against `ComposableCoW.getTradeableOrderWithSignature`, decode return, rebuild `OrderCreation`. Backs `twap.poll-and-submit`. -2. `cowprotocol::eth_flow::decode_placement(log) -> Result` — decode `OrderPlacement` event log, reconstruct `OrderCreation` and `OrderUid`. Backs `ethflow.submit-from-log`. -3. `cowprotocol::app_data::OrderBookAppDataResolver` — `AppDataResolver` trait + cached implementation around `OrderBookApi::app_data(hash)`, with `EMPTY_APP_DATA_HASH` fast-path. Used by twap, ethflow, and any future caller that needs to materialise an app-data document. -4. `cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)` — custom-URL constructor for barn / staging / forked deployments. -5. `cowprotocol` `wasm32` compatibility — feature-gate the `reqwest` dependency so guest modules can use the pure types (`Order`, `OrderCreation`, `OrderUid`, `composable::*`, `eth_flow::decode_*`) without dragging in an HTTP client. +1. **`cowprotocol::composable::poll_and_build_order(provider, owner, params, proof) -> Result`** — eth_call against `ComposableCoW.getTradeableOrderWithSignature`, decode return, rebuild `OrderCreation`. `PollOutcome` mirrors watchtower's `PollResultCode` (TS): `Submitted(OrderCreation, Vec)`, `TryAtEpoch(u64)`, `TryOnBlock(u64)`, `TryNextBlock`, `DontTryAgain`. Backs `twap.poll-and-submit` (ADR-0006). + +2. **`cowprotocol::eth_flow::decode_placement(log) -> Result`** — decode `OrderPlacement` event log, reconstruct `OrderCreation` with the EIP-1271 signing scheme pointing at the `CoWSwapEthFlow` contract, compute `OrderUid`. Replicates the indexing logic currently inside `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`. Backs `ethflow.submit-from-log` (ADR-0006). + +3. **`cowprotocol::OrderPostError` rich variants + `retry_hint(&self) -> RetryHint`** — typed orderbook submission errors (`QuoteNotFound`, `InvalidQuote`, `InsufficientAllowance`, `InsufficientBalance`, `TooManyLimitOrders`, `InvalidAppData`, `AppDataFromMismatch`, `SellAmountOverflow`, `ZeroAmount`, `TransferSimulationFailed`, `ExcessiveValidTo`, …) with a `retry_hint()` helper classifying each into `TryNextBlock`, `BackoffSeconds(u64)`, or `Drop`. Mirrors watchtower's `API_ERRORS_TRY_NEXT_BLOCK` / `API_ERRORS_BACKOFF` / `API_ERRORS_DROP` tables. Without this, every Rust consumer of CoW reinvents the same mapping, and modules spam the orderbook with permanently-broken orders. **Critical-path, not optional.** + +4. **`cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)`** — custom-URL constructor for barn / staging / forked deployments. Unblocks per-chain orderbook URL overrides in `engine.toml` (ADR-0005). + +5. **`cowprotocol` `wasm32` compatibility** — feature-gate the `reqwest` dependency so guest modules can use the pure types (`Order`, `OrderCreation`, `OrderUid`, `composable::*`, `eth_flow::decode_*`) without dragging in an HTTP client. Unblocks M3 SDK guest modules consuming `cowprotocol` directly. -Lower-priority follow-ons (richer `OrderBookApiError` variants, `OrderUid::from_slice`, retry middleware, `OrderCreation::from_gpv2`) are good-to-have but are not blocking for the M2 host scope. +Lower-priority follow-ons (`OrderUid::from_slice`, retry middleware on `OrderBookApi`, `OrderCreation::from_gpv2`) are good-to-have but are not blocking for the M2 host scope. ## Considered options - **Implement locally, refactor upstream later.** Faster short term but predictably leaves an indeterminate amount of duplicated logic in the engine, contradicts mfw78's stated conventions, and grows technical debt every time cow-rs evolves the underlying types. Rejected. - **Wait for cow-rs upstream maintainers to add these on their own.** No evidence anyone else is doing this work; the grant timeline does not permit waiting. - **Vendor a fork of cow-rs inside `nullislabs/shepherd`.** Worst of all worlds: blocks neither the engine nor cow-rs from drifting, and forces every other CoW consumer to re-derive the same primitives. +- **Simple `Ready/NotReady` PollOutcome on item 1.** Rejected: doesn't capture watchtower's `TRY_AT_EPOCH(t)` hint, which is what prevents the polling loop from RPC-spamming during the 1-hour gap between TWAP parts. ## Consequences -- Every M2 engine issue that consumes one of the five primitives above is blocked on its cow-rs PR merging. We sequence issues so that upstream PRs and engine adoption can land in parallel where possible (e.g., open all three protocol-helper PRs against `bleu/cow-rs` simultaneously rather than serially). -- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0002) is bumped after each cow-rs merge; the bump is the engine's signal that a new primitive is consumable. +- Every M2 engine issue that consumes one of the five primitives above is blocked on its cow-rs PR merging. We sequence issues so that upstream PRs and engine adoption can land in parallel where possible (e.g., open items 1, 2, 3 against `bleu/cow-rs` simultaneously rather than serially). +- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0004) is bumped after each cow-rs merge; the bump is the engine's signal that a new primitive is consumable. - PRs in `bleu/cow-rs` follow the existing mfw78 conventions established by cow-rs PR #5: severity-tagged reviews, alloy reuse over local reimplementation, GPL-3.0, edition 2024, terse rustdoc. - After acceptance in `bleu/cow-rs`, each primitive is also surfaced as a PR (or backport) against `cowdao-grants/cow-rs` so the wider ecosystem benefits and the bleu fork narrows over time. - The engine repo stays small: `nexum-engine` contains WIT, host wiring, supervisor, redb store, alloy provider pool, and `engine.toml` schema — nothing about CoW Protocol semantics. +- The rich `PollOutcome` (item 1) + `OrderPostError` + `retry_hint` (item 3) design naturally leads to tighter M3 SDK helpers: `WatchSet`, `PollLoop`, `BackoffLedger` patterns that any module re-using `shepherd-sdk` gets for free. +- A follow-on Bleu module — the Rust-side equivalent of `cowprotocol/refunder` (permissionless `invalidateOrder` triggering for expired EthFlow orders) — becomes natural to ship once `ethflow.submit-from-log` lands. Out of scope for M2 but explicitly enabled by the same primitives. diff --git a/docs/adr/0008-factory-subscriptions-in-manifest.md b/docs/adr/0008-factory-subscriptions-in-manifest.md new file mode 100644 index 0000000..af6c6f1 --- /dev/null +++ b/docs/adr/0008-factory-subscriptions-in-manifest.md @@ -0,0 +1,140 @@ +--- +status: proposed +--- + +# Dynamic address registration for log subscriptions (Envio HyperIndex-style) + +## Context + +Some module archetypes need to track contracts deployed dynamically by a factory — Uniswap V3 pools (deployed by `UniswapV3Factory`), Aave V3 reserves (registered via `PoolAddressesProvider`), lending market deployments, NFT marketplace collections. Static `[[subscription]]` declarations in `nexum.toml` cannot express this: the child addresses are not known when the module's manifest is authored. + +Neither TWAP nor EthFlow (the M2 grant deliverables) needs this — both subscribe to a single well-known contract per chain (`ComposableCoW`, `CoWSwapEthFlow`). This ADR is forward-looking: 0.2 is the breaking-change window per `docs/migration/0.1-to-0.2.md` §522, with contracts stable from 0.2.0 onwards. Adding factory support after 0.2 would require another major version bump. + +Two production EVM indexer frameworks define the design space: + +- **Ponder** (`ponder.config.ts`) is fully declarative: the user describes the factory contract, the factory event, and which event parameter holds the child address. Ponder extracts the address itself and indexes children automatically. The framework does ABI-aware extraction; the user writes no factory handler. +- **Envio HyperIndex** (`envio.yaml` + handler code) is a hybrid: the user declares the **template** (ABI, event topics) without an address, then calls `context..register(address)` inside the factory event handler. Topics are static; the watched address set grows at runtime. + +The Envio model decouples "what topics the engine listens for" (static, in the manifest) from "which addresses are interesting" (dynamic, driven by module code). The engine maintains a single aggregated subscription per template; the address set is mutated as the module learns of new contracts. + +## Decision + +Adopt the Envio model. Two pieces: + +**1. Manifest schema gains `[[subscription.template]]`** — a topic-only log subscription whose address set is populated at runtime: + +```toml +[[subscription.template]] +chain_id = 1 +name = "uniswap_v3_pool" +event_topics = [ + "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67", # Swap + "0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde", # Mint +] +# Optional: pre-register a set of addresses at boot (before init runs). +# Useful for protocols where some addresses are known statically. +initial_addresses = [] +``` + +Existing `[[subscription]]` blocks with concrete `address` are unchanged. A module typically declares one static `[[subscription]]` for the factory event itself, plus one `[[subscription.template]]` per child contract type. + +**2. `nexum:host/chain` gains two host functions** for the module to manage the address set: + +```wit +interface chain { + // ... existing request, request-batch, subscribe-blocks, subscribe-logs ... + + /// Add an address to the watch set for `template-name` on `chain-id`. + /// If `from-block` is set and precedes the current head, the engine + /// runs paginated `eth_getLogs` over the template's topics on this + /// address from `from-block` to head before going live. Registration + /// is idempotent — calling twice with the same arguments is a no-op. + register-address: func( + chain-id: chain-id, + template-name: string, + address: list, // 20 bytes + from-block: option, + ) -> result<_, host-error>; + + /// Remove an address from the watch set. Subsequent events on that + /// address are dropped. Idempotent — unregistering an unknown address + /// returns Ok. + unregister-address: func( + chain-id: chain-id, + template-name: string, + address: list, + ) -> result<_, host-error>; +} +``` + +**3. `log-source` variant in `nexum:host/event-module`** gains a `template` case so modules can route events: + +```wit +variant log-source { + static(u32), // existing — index into [[subscription]] + template(string), // new — name of the [[subscription.template]] +} +``` + +Module code (Rust, Uniswap V3 indexer): + +```rust +fn init(config: Vec<(String, String)>) -> Result<(), HostError> { + // Resume: re-register every pool we already discovered. Each pool's + // creation block is persisted; backfill resumes from where we left off. + for key in local_store::list_keys("pool:")? { + let pool = parse_addr(&key); + let from_block = u64::from_le_bytes(local_store::get(&key)?.unwrap()); + chain::register_address(1, "uniswap_v3_pool", &pool, Some(from_block))?; + } + Ok(()) +} + +fn on_event(event: Event) -> Result<(), HostError> { + match event { + // Factory event — declared as a static [[subscription]] for the factory. + Event::Log(LogEvent { log, source: LogSource::Static(0) }) => { + let pool = decode_pool_created(&log)?; + chain::register_address(1, "uniswap_v3_pool", &pool, Some(log.block_number))?; + local_store::set(&format!("pool:{}", hex(&pool)), &log.block_number.to_le_bytes())?; + } + // Pool event — dispatched through the template. + Event::Log(LogEvent { log, source: LogSource::Template(name) }) + if name == "uniswap_v3_pool" => + { + process_pool_event(log)?; + } + _ => {} + } + Ok(()) +} +``` + +Engine internals: + +- **Boot**: read all `[[subscription.template]]` blocks across loaded modules. Initialise per-template address sets from `initial_addresses` plus the reserved key `__nexum:template:{name}:addresses` in the module's `local-store` namespace. +- **Live**: one `eth_subscribe logs` per chain per template, filter = `(topic in event_topics) ∧ (address in current_set)`. When `register-address` mutates the set, the engine re-subscribes (or shards the filter when the set exceeds the provider's address-limit threshold). +- **Backfill on register**: if `from-block` < current head, run paginated `eth_getLogs(topic, address, from-block, head)` synchronously before joining the live stream. Events from backfill are dispatched through the same `on_event` callback as live events. +- **Persistence**: engine writes `__nexum:template:{name}:addresses` (one entry per registered address) and `__nexum:template:{name}:cursor:{address}` (last block dispatched per address) in the module's reserved namespace. Resume is automatic if the module re-calls `register-address` for each persisted address; engine deduplicates via the persisted cursor. + +## Considered options + +- **Ponder full-declarative** (factory address + event + parameter declared in manifest, engine extracts child). Rejected: schema must express ABI-aware extraction (`child = { source = "topic", index = 1 }` or `{ source = "data", offset = 32, length = 20 }`), and grows with every exotic factory shape (nested factories, multi-child events, address computed from multiple fields). The Envio model pushes that complexity into module Rust code — where it belongs, given the module already decodes events with `alloy_sol_types`. +- **Pure imperative `chain.open-log-stream(filter) -> stream-handle`.** Rejected: each call opens a new subscription, so N pools = N WSS connections. Doesn't scale to indexers with 10k+ tracked addresses. The Envio model keeps one subscription per template and mutates its address set — engine batches naturally. +- **Wildcard manifest** (`address = "*"` with topic-only filter, module client-side filters). Rejected: mainnet has ~10k contracts emitting `Transfer` / `Swap` per day. The engine would deliver every matching event to every wildcard subscriber; modules pay fuel and bandwidth to discard 99% of them. +- **Defer factory pattern entirely to 0.3.** Rejected: 0.2 is the breaking-change window per migration:522. Adding either `[[subscription.template]]` or `register-address` after 0.2 requires another major bump. Better to land the small surface now than break later. +- **Templates declared inside `[[subscription]]` with optional `address` (one block, two modes).** Rejected: conflates two semantically distinct cases — modules looking at `[[subscription]]` would have to inspect for the presence of `address` to know whether they need to call `register-address`. Separate block name is clearer. +- **Engine extracts the child address from the static factory `[[subscription]]` (best of both).** Rejected: would require the manifest to identify a relationship between the static subscription and a template, plus the ABI extraction rules. Reintroduces the Ponder schema complexity we just rejected. + +## Consequences + +- `nexum.toml` schema gains `[[subscription.template]]` with `chain_id`, `name`, `event_topics`, optional `initial_addresses`. mfw78 approval needed for the schema extension (his spec). +- `nexum:host/chain` gains `register-address` and `unregister-address` functions; `nexum:host/event-module`'s `log-source` variant gains `template(string)`. mfw78 approval needed for the WIT change (his namespace). These are the only WIT additions in this ADR — small, focused, with no implicit dependencies on other interfaces. +- Reserved key namespace `__nexum:template:*` in each module's `local-store` namespace. Modules MUST NOT write to keys with this prefix; engine reserves them. +- Module boilerplate per factory ≈ 5–10 lines (decode the factory event, call `register-address`, persist for resume). The M3 SDK is expected to ship a helper that encapsulates this — something like `Factory::::on_event(register_template("uniswap_v3_pool"))` — but the host surface is intentionally simple enough that no SDK is required to use it. +- Engine can register addresses sourced from anywhere — factory events, HTTP API responses, governance votes, operator-supplied lists. Composability is a deliberate feature; the engine treats every registration the same way regardless of provenance. +- Nested factories (a child contract that is itself a factory) work without schema changes: the child's event handler decodes its own creation events and calls `register-address` on the grandchild template. Engine has no concept of nesting; it just multiplexes addresses per template. +- Conditional registration ("only register pools with fee = 3000") works without schema changes: the module's factory-event handler inspects the event payload and decides whether to call `register-address`. +- Backfill cost: a module registering 10k addresses with `from-block` deep in history triggers 10k paginated `eth_getLogs` runs, sequentially per address (the engine cannot batch across addresses with different `from-block` cursors without state-machine work that's not in scope for 0.2). Operators should set sensible `start_block` boundaries; the M3 SDK is expected to ship a `BulkBackfill` helper that groups same-cursor addresses into combined filter queries. +- The address set per template is bounded only by `local-store` quota. The engine enforces a soft cap (default 50k addresses per template) configurable in `engine.toml` to prevent a runaway module from saturating the provider's filter limits; exceeding the cap returns `host-error.denied` from `register-address`. +- A module that never calls `register-address` for a declared template receives no events from it — equivalent to declaring an unused subscription. Engine logs a warning on boot if a template is declared with `initial_addresses = []` and the module has no registrations after `init` returns. From 2ddd1ecb181dbcece556a3552d18b14832ed984d Mon Sep 17 00:00:00 2001 From: brunota20 Date: Wed, 3 Jun 2026 19:26:04 -0300 Subject: [PATCH 21/29] fix(docs): reviewed ADRs by bleu --- .../0004-patch-cowprotocol-to-bleu-cow-rs.md | 27 +++--- .../0005-cow-api-via-cached-orderbookapi.md | 6 +- .../adr/0006-cow-twap-ethflow-host-helpers.md | 14 +-- .../0007-upstream-protocol-logic-to-cow-rs.md | 23 +++-- .../0008-factory-subscriptions-in-manifest.md | 89 ++++++++----------- 5 files changed, 76 insertions(+), 83 deletions(-) diff --git a/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md b/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md index 4564401..8dfb931 100644 --- a/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md +++ b/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md @@ -3,33 +3,36 @@ status: proposed implemented-in: nullislabs/shepherd#10 --- -# Patch `cowprotocol` crate to `bleu/cow-rs` main +# Patch `cowprotocol` crate to the head of upstream PR #5 ## Context -`cowprotocol` v1.0.0-alpha.3 (the version on crates.io) was cut from `cowdao-grants/cow-rs` PR #5 at commit `1742ffa`. The published artifact predates 18 follow-up commits on `bleu/cow-rs` main that the engine materially depends on, in particular: +`cowprotocol` v1.0.0-alpha.3 (the version on crates.io) was cut from an early snapshot of `cowdao-grants/cow-rs` PR #5 at commit `1742ffa`. That PR is still open and is the canonical upstream channel for landing additions to the Rust SDK. Its head branch is `bleu/cow-rs:main`, currently at commit `c012404`, carrying 18 follow-up commits the engine materially depends on: -- `composable::Proof` byte-width fix (consumed by the TWAP poll path); -- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding from mfw78's review of PR #5); -- `order_book` / `composable` submodule splits (cleaner imports on the engine side, no more `cowprotocol::order_book::*` re-export gymnastics). +- `composable::Proof` byte-width fix (consumed by the TWAP poll path). +- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding from the PR #5 review). +- `order_book` / `composable` submodule splits (cleaner imports on the engine side). -ADR-0007 additionally commits us to pushing TWAP / EthFlow / app-data protocol logic upstream into `cowprotocol` first and consuming it via the same patched dependency, so the patch surface will continue growing through M2. +ADR-0007 commits us to landing TWAP, EthFlow, and `OrderPostError` primitives into PR #5 directly, by pushing additional commits to its head branch. Each commit advances both PR #5 and the patch rev consumed here. -There is no published `alpha.4` and no scheduled date for one. +There is no published `alpha.4` and no scheduled date for one; the engine cannot wait. ## Decision Add a workspace-level `[patch.crates-io]` redirecting `cowprotocol` to `https://github.com/bleu/cow-rs` at commit `c012404`. Every crate that declares `cowprotocol = "1.0.0-alpha.3"` (engine, modules, future SDK) silently picks up the patched build with no `Cargo.toml` change at the dependent site. +This is not a parallel fork. `bleu/cow-rs:main` IS the head branch of upstream PR #5. Pushing to it updates PR #5; the patch rev advances by bumping a single workspace line. + ## Considered options -- **Vendor the missing types locally.** Rejected: re-implementing `composable::Proof`, `OrderCreation`, etc. in the engine repo is exactly the AI-duplication anti-pattern mfw78 flagged in cow-rs PR #5. Reuse over reimplement applies. +- **Vendor the missing types locally.** Rejected: re-implementing `composable::Proof`, `OrderCreation`, etc. in the engine repo is the AI-duplication anti-pattern flagged by reviewers on cow-rs PR #5. Reuse over reimplement applies. - **Pin every dependent to `cow-rs` git directly.** Works but every new workspace member has to remember the git source. `[patch.crates-io]` centralises the override. +- **Open a separate PR per primitive against `cowdao-grants/cow-rs`.** Rejected: fragments review across multiple PRs when one already exists at the appropriate granularity. Stacking commits on PR #5 keeps the review thread coherent and lets reviewers track the cumulative change. - **Wait for `alpha.4` to publish.** No ETA; the TWAP/EthFlow milestone cannot land without `composable::Proof` correct. ## Consequences -- `cargo update` will re-resolve to the same `rev` — the lock pins it. -- Bumping the rev is a single-line workspace edit; reviewers see one diff. -- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `OrderPostError` rich variants). Until then, expect the patch rev to advance with every cow-rs merge that the engine consumes. -- Modules built against this workspace inherit the patch transitively; modules built standalone against crates.io will see `alpha.3` and may hit the very bugs the patch closes — flag in the SDK README when M3 lands. +- `cargo update` will re-resolve to the same `rev`; the lock pins it. +- Bumping the rev is a single-line workspace edit; reviewers see one diff per primitive added to PR #5. +- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `OrderPostError` rich variants). Until then, expect the patch rev to advance with every push to PR #5. +- Modules built against this workspace inherit the patch transitively; modules built standalone against crates.io will see `alpha.3` and may hit the very bugs the patch closes. Flag this in the SDK README when M3 lands. diff --git a/docs/adr/0005-cow-api-via-cached-orderbookapi.md b/docs/adr/0005-cow-api-via-cached-orderbookapi.md index ae9c6b5..5b37019 100644 --- a/docs/adr/0005-cow-api-via-cached-orderbookapi.md +++ b/docs/adr/0005-cow-api-via-cached-orderbookapi.md @@ -11,7 +11,9 @@ implemented-in: nullislabs/shepherd#8 ## Decision -At engine boot, construct one `cowprotocol::OrderBookApi` per `cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, ArbitrumOne, Base) into a `BTreeMap` keyed by EVM chain id. Both `cow-api` operations consult this pool: +At engine boot, construct one `cowprotocol::OrderBookApi` per `cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, ArbitrumOne, Base) into a `BTreeMap` keyed by EVM chain id. "Cached" here means built once during boot and reused for the engine's lifetime; clients are not lazy-constructed on each call nor LRU-evicted. The map is created in `OrderBookPool::with_default_chains()` and never mutated after. + +Both `cow-api` operations consult this pool: - `request` resolves the chain's `OrderBookApi`, reads `api.base_url()` for the prefix, joins the module-supplied path, and dispatches via a shared `reqwest::Client`. - `submit-order` deserialises the JSON `OrderCreation` and calls `OrderBookApi::post_order` directly. The crate handles signing-scheme encoding, error mapping, and `OrderUid` extraction. @@ -20,7 +22,7 @@ Chains not in `cowprotocol::Chain` return `HostError { kind: unsupported }` at t ## Considered options -- **Raw `reqwest` for both.** Rejected: forces us to maintain the chain → base-URL table (drifts whenever cowprotocol adds a chain) and reimplement `post_order`'s body codec and error mapping — the exact duplication mfw78 called out in cow-rs PR #5. +- **Raw `reqwest` for both.** Rejected: forces us to maintain the chain → base-URL table (drifts whenever cowprotocol adds a chain) and reimplement `post_order`'s body codec and error mapping, the exact duplication flagged in cow-rs PR #5 review. - **`OrderBookApi` for `submit-order`, raw `reqwest` for `request`.** Tempting (request is opaque to the crate) but means two separate chain-resolution paths, two HTTP clients, and a second place to keep the chain set in sync. - **Build `OrderBookApi` lazily on first call per chain.** Rejected: hides config errors at runtime. Up-front boot construction surfaces unknown chains immediately and amortises away the per-call cost. diff --git a/docs/adr/0006-cow-twap-ethflow-host-helpers.md b/docs/adr/0006-cow-twap-ethflow-host-helpers.md index 8870d3d..300709a 100644 --- a/docs/adr/0006-cow-twap-ethflow-host-helpers.md +++ b/docs/adr/0006-cow-twap-ethflow-host-helpers.md @@ -6,14 +6,16 @@ status: proposed ## Context -The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook access (REST passthrough + `submit-order`). Two further CoW workflows show up in every non-trivial module: ComposableCoW conditional orders (TWAP being the canonical example) and EthFlow native-ETH orders. Both follow the same external-indexer/relayer pattern that CoW's own infrastructure has been progressively extracting from the monolithic `cowprotocol/services` repository: +The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook access (REST passthrough + `submit-order`). Two further CoW workflows show up in every non-trivial module: ComposableCoW conditional orders (TWAP being the canonical example) and EthFlow native-ETH orders. Both follow the same external-indexer/relayer pattern that CoW maintainers have signalled intent to extract from the monolithic `cowprotocol/services` repository: -- **TWAP / ComposableCoW** — extracted as the standalone `cowprotocol/watch-tower` (TypeScript). Listens to `ConditionalOrderCreated`, polls `getTradeableOrderWithSignature` on each block, posts to the orderbook when an order becomes tradeable. -- **EthFlow indexer** — still inside `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`. Listens to `OrderPlacement` / `OrderInvalidation` / `OrderRefund`, inserts into the `ethflow_orders` table. CoW's architectural direction is to extract this into a standalone service, mirroring how `watch-tower` and the `refunder` crate were already extracted. The Shepherd `ethflow-watcher` module is the natural Rust home for that extraction. +- **TWAP / ComposableCoW** is already extracted as the standalone `cowprotocol/watch-tower` (TypeScript). Listens to `ConditionalOrderCreated`, polls `getTradeableOrderWithSignature` on each block, posts to the orderbook when an order becomes tradeable. +- **EthFlow indexer** still lives inside `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`. Listens to `OrderPlacement` / `OrderInvalidation` / `OrderRefund`, inserts into the `ethflow_orders` table. The intent is to extract it into a standalone service following the same path `watch-tower` and the `refunder` crate already took. The Shepherd `ethflow-watcher` module is positioned as that extraction. Both flows share the same pattern: observe an on-chain event, derive a signed `OrderCreation`, submit it to the orderbook. The derivation has enough protocol detail (signing scheme, ComposableCoW eth_call, EthFlow EIP-1271 contract signature, log decoding) that a guest module would either ship that logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or make ten round-trips to the host through generic `chain`/`cow-api` calls. -Per ADR-0007, the protocol logic itself lives in the `cowprotocol` crate — not in `nexum-engine`. This ADR consequently scopes the engine-side helpers to the WIT surface and the glue that wires the upstream primitives into the host call boundary. +Per ADR-0007, the protocol logic itself lives in the `cowprotocol` crate, not in `nexum-engine`. This ADR consequently scopes the engine-side helpers to the WIT surface and the glue that wires the upstream primitives into the host call boundary. + +The newer ComposableCoW iteration in development simplifies polling versus the watch-tower TypeScript implementation: less of the SDK's discriminated `PollResultCode` mapping may need to be replicated in `cowprotocol::composable` for `twap.poll-and-submit` to work. The rich `PollOutcome` variants described below remain the target API surface; the upstream implementation may end up simpler than the watch-tower mirror suggests. ## Decision @@ -64,9 +66,9 @@ The engine wires these primitives into HostState and maps their errors to `host- ## Considered options -- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW / EthFlow glue. mfw78's "reuse over reimplement" applied: that derivation already lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and should not be re-shipped in every WASM artifact. +- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW / EthFlow glue. "Reuse over reimplement" applies: that derivation already lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and should not be re-shipped in every WASM artifact. - **Implement the protocol glue inside `nexum-engine` host code, port upstream later.** Rejected per ADR-0007: every line of TWAP polling or EthFlow decoding that lives in the engine is a line that future Rust consumers cannot reuse, and a line that diverges as cow-rs evolves. -- **EthFlow as pure passive observer (no `submit-from-log`).** Briefly considered after reading "watcher" / "monitor" in mfw78's docs/00 and docs/04 as "no submission". Rejected after verifying that CoW's own autopilot DOES post equivalent (insert into `ethflow_orders` table); the Shepherd module is intended to externalize that role, not replace it with passive observation. The `pending_orders` state mentioned in docs/04 is a side-effect of the relay (local accounting of what's been observed), not the goal. +- **EthFlow as pure passive observer (no `submit-from-log`).** Briefly considered after reading "watcher" / "monitor" in docs/00 and docs/04 as "no submission". Rejected after verifying that CoW's own autopilot DOES post equivalent (insert into `ethflow_orders` table); the Shepherd module is intended to externalize that role, not replace it with passive observation. The `pending_orders` state mentioned in docs/04 is a side-effect of the relay (local accounting of what's been observed), not the goal. - **Simple `option` return on twap instead of `poll-outcome` variant.** A 1-hour-spaced TWAP polled every block would spam ~300 RPC calls per part with `None` returns. The richer outcome (`try-at-epoch`, etc.) matches watchtower's existing `PollResult` and lets modules skip polls until the contract says it's worth retrying. Production-critical. - **Single combined interface** `shepherd:cow/orders` with both helpers. Cheaper world surface but harder to gate per-capability — a module that only watches EthFlow shouldn't have to import TWAP and vice versa. Splitting keeps `[capabilities].required` honest. - **`log-json: list` payload** instead of the typed `nexum:host/types.log` record. The record already exists and the engine's event dispatch already projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both ergonomics and "no duplicate decoders". diff --git a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md index dac09ea..b6aeca6 100644 --- a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md +++ b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md @@ -6,15 +6,15 @@ status: proposed ## Context -Implementing ADR-0006 (twap + ethflow host helpers) and ADR-0005 (cow-api backend) surfaces a recurring question: when the engine needs a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (TWAP polling glue, EthFlow log decoding, rich orderbook error variants, custom orderbook URLs), do we write that logic locally in `nexum-engine` and tidy it up upstream later, or do we open the cow-rs PR first and only land the engine wiring afterwards? +Implementing ADR-0006 (twap + ethflow host helpers) and ADR-0005 (cow-api backend) surfaces a recurring question: when the engine needs a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (TWAP polling glue, EthFlow log decoding, rich orderbook error variants, custom orderbook URLs), do we write that logic locally in `nexum-engine` and tidy it up upstream later, or do we add it to the open upstream PR first and only land the engine wiring afterwards? -mfw78's review of cow-rs PR #5 named the failure mode explicitly: duplicating work that an existing crate could do is the AI-coding anti-pattern most likely to land in a Bleu PR. The same risk applies to any engine-side reimplementation of protocol logic. +Review feedback on cow-rs PR #5 named the failure mode explicitly: duplicating work that an existing crate could do is the AI-coding anti-pattern most likely to land in a contribution. The same risk applies to any engine-side reimplementation of protocol logic. -CoW's broader architecture has been moving the same direction: `watch-tower` extracted from `cowprotocol/services` autopilot, the `refunder` crate likewise, with the `ethflow_events` indexer (`crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`) identified as the next extraction target. The Rust-side equivalent of those extractions is the right home for protocol primitives — `bleu/cow-rs` (then upstream into `cowdao-grants/cow-rs`), not the engine. +CoW maintainers have signalled intent to keep extracting services from the `cowprotocol/services` monolith: `watch-tower` is already extracted, the `refunder` crate likewise, and the `ethflow_events` indexer (`crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`) is the next extraction target. The Rust SDK that Bleu is delivering through PR #5 is the natural home for the protocol primitives those extractions need. ## Decision -Protocol-level CoW logic — anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need — lands in `bleu/cow-rs` first as an upstream PR, and is consumed by `nexum-engine` via the existing `[patch.crates-io]` rev bump (ADR-0004). The engine never writes throwaway local copies of the same logic with the intent to "port later". +Protocol-level CoW logic, meaning anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need, lands as additional commits on `cowdao-grants/cow-rs` PR #5 first (head branch `bleu/cow-rs:main`), and is consumed by `nexum-engine` via the `[patch.crates-io]` rev bump (ADR-0004). The engine never writes throwaway local copies of the same logic with the intent to "port later". The concrete set of primitives we know we need is, in priority order: @@ -32,17 +32,16 @@ Lower-priority follow-ons (`OrderUid::from_slice`, retry middleware on `OrderBoo ## Considered options -- **Implement locally, refactor upstream later.** Faster short term but predictably leaves an indeterminate amount of duplicated logic in the engine, contradicts mfw78's stated conventions, and grows technical debt every time cow-rs evolves the underlying types. Rejected. +- **Implement locally, refactor upstream later.** Faster short term but predictably leaves an indeterminate amount of duplicated logic in the engine, contradicts the conventions established on cow-rs PR #5, and grows technical debt every time cow-rs evolves the underlying types. Rejected. - **Wait for cow-rs upstream maintainers to add these on their own.** No evidence anyone else is doing this work; the grant timeline does not permit waiting. - **Vendor a fork of cow-rs inside `nullislabs/shepherd`.** Worst of all worlds: blocks neither the engine nor cow-rs from drifting, and forces every other CoW consumer to re-derive the same primitives. - **Simple `Ready/NotReady` PollOutcome on item 1.** Rejected: doesn't capture watchtower's `TRY_AT_EPOCH(t)` hint, which is what prevents the polling loop from RPC-spamming during the 1-hour gap between TWAP parts. ## Consequences -- Every M2 engine issue that consumes one of the five primitives above is blocked on its cow-rs PR merging. We sequence issues so that upstream PRs and engine adoption can land in parallel where possible (e.g., open items 1, 2, 3 against `bleu/cow-rs` simultaneously rather than serially). -- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0004) is bumped after each cow-rs merge; the bump is the engine's signal that a new primitive is consumable. -- PRs in `bleu/cow-rs` follow the existing mfw78 conventions established by cow-rs PR #5: severity-tagged reviews, alloy reuse over local reimplementation, GPL-3.0, edition 2024, terse rustdoc. -- After acceptance in `bleu/cow-rs`, each primitive is also surfaced as a PR (or backport) against `cowdao-grants/cow-rs` so the wider ecosystem benefits and the bleu fork narrows over time. -- The engine repo stays small: `nexum-engine` contains WIT, host wiring, supervisor, redb store, alloy provider pool, and `engine.toml` schema — nothing about CoW Protocol semantics. -- The rich `PollOutcome` (item 1) + `OrderPostError` + `retry_hint` (item 3) design naturally leads to tighter M3 SDK helpers: `WatchSet`, `PollLoop`, `BackoffLedger` patterns that any module re-using `shepherd-sdk` gets for free. -- A follow-on Bleu module — the Rust-side equivalent of `cowprotocol/refunder` (permissionless `invalidateOrder` triggering for expired EthFlow orders) — becomes natural to ship once `ethflow.submit-from-log` lands. Out of scope for M2 but explicitly enabled by the same primitives. +- Every M2 engine issue that consumes one of the five primitives above is blocked on the corresponding commit landing in PR #5's head branch. Items 1, 2, 3 can be authored as independent commits and pushed in parallel rather than serially. +- `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0004) is bumped after each push to PR #5; the bump is the engine's signal that a new primitive is consumable. +- Commits added to PR #5 follow the conventions established by its review thread: severity-tagged review notes, alloy reuse over local reimplementation, GPL-3.0, edition 2024, terse rustdoc. +- The engine repo stays small: `nexum-engine` contains WIT, host wiring, supervisor, redb store, alloy provider pool, and `engine.toml` schema, with nothing about CoW Protocol semantics. +- The rich `PollOutcome` (item 1) plus `OrderPostError` and `retry_hint` (item 3) design naturally leads to tighter M3 SDK helpers: `WatchSet`, `PollLoop`, `BackoffLedger` patterns that any module re-using `shepherd-sdk` gets for free. +- A follow-on Bleu module, the Rust-side equivalent of `cowprotocol/refunder` (permissionless `invalidateOrder` triggering for expired EthFlow orders), becomes natural to ship once `ethflow.submit-from-log` lands. Out of scope for M2 but explicitly enabled by the same primitives. diff --git a/docs/adr/0008-factory-subscriptions-in-manifest.md b/docs/adr/0008-factory-subscriptions-in-manifest.md index af6c6f1..fead3a9 100644 --- a/docs/adr/0008-factory-subscriptions-in-manifest.md +++ b/docs/adr/0008-factory-subscriptions-in-manifest.md @@ -2,26 +2,23 @@ status: proposed --- -# Dynamic address registration for log subscriptions (Envio HyperIndex-style) +# Dynamic address registration for log subscriptions ## Context -Some module archetypes need to track contracts deployed dynamically by a factory — Uniswap V3 pools (deployed by `UniswapV3Factory`), Aave V3 reserves (registered via `PoolAddressesProvider`), lending market deployments, NFT marketplace collections. Static `[[subscription]]` declarations in `nexum.toml` cannot express this: the child addresses are not known when the module's manifest is authored. +Some module archetypes need to track contracts deployed dynamically by a factory, for example Uniswap V3 pools (deployed by `UniswapV3Factory`). Static `[[subscription]]` declarations in `nexum.toml` cannot express this: the child addresses are not known when the module's manifest is authored. -Neither TWAP nor EthFlow (the M2 grant deliverables) needs this — both subscribe to a single well-known contract per chain (`ComposableCoW`, `CoWSwapEthFlow`). This ADR is forward-looking: 0.2 is the breaking-change window per `docs/migration/0.1-to-0.2.md` §522, with contracts stable from 0.2.0 onwards. Adding factory support after 0.2 would require another major version bump. +Neither TWAP nor EthFlow (the M2 grant deliverables) needs this. Both subscribe to a single well-known contract per chain. This ADR is forward-looking, motivated by `docs/migration/0.1-to-0.2.md` §522 declaring 0.2 the breaking-change window; adding factory support after 0.2 would require another major version bump. -Two production EVM indexer frameworks define the design space: +Envio HyperIndex uses a hybrid pattern that fits Shepherd's design: topics are declared statically in the manifest, and the watched address set is mutated at runtime via a `register()` host call. The engine maintains a single aggregated log subscription per template; the address set grows as the module learns of new contracts. -- **Ponder** (`ponder.config.ts`) is fully declarative: the user describes the factory contract, the factory event, and which event parameter holds the child address. Ponder extracts the address itself and indexes children automatically. The framework does ABI-aware extraction; the user writes no factory handler. -- **Envio HyperIndex** (`envio.yaml` + handler code) is a hybrid: the user declares the **template** (ABI, event topics) without an address, then calls `context..register(address)` inside the factory event handler. Topics are static; the watched address set grows at runtime. - -The Envio model decouples "what topics the engine listens for" (static, in the manifest) from "which addresses are interesting" (dynamic, driven by module code). The engine maintains a single aggregated subscription per template; the address set is mutated as the module learns of new contracts. +Whether the engine should also handle historical backfill on register (the module passes `from-block`, engine paginates `eth_getLogs` from there to head before going live) is a separate decision flagged for upstream review. This ADR keeps the engine surface minimal and defers historical replay to the existing module-driven catch-up pattern documented in `docs/02:260`. ## Decision -Adopt the Envio model. Two pieces: +Two pieces: -**1. Manifest schema gains `[[subscription.template]]`** — a topic-only log subscription whose address set is populated at runtime: +**1. Manifest schema gains `[[subscription.template]]`**, a topic-only log subscription whose address set is populated at runtime: ```toml [[subscription.template]] @@ -31,9 +28,6 @@ event_topics = [ "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67", # Swap "0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde", # Mint ] -# Optional: pre-register a set of addresses at boot (before init runs). -# Useful for protocols where some addresses are known statically. -initial_addresses = [] ``` Existing `[[subscription]]` blocks with concrete `address` are unchanged. A module typically declares one static `[[subscription]]` for the factory event itself, plus one `[[subscription.template]]` per child contract type. @@ -42,23 +36,18 @@ Existing `[[subscription]]` blocks with concrete `address` are unchanged. A modu ```wit interface chain { - // ... existing request, request-batch, subscribe-blocks, subscribe-logs ... + // existing request, request-batch, subscribe-blocks, subscribe-logs ... /// Add an address to the watch set for `template-name` on `chain-id`. - /// If `from-block` is set and precedes the current head, the engine - /// runs paginated `eth_getLogs` over the template's topics on this - /// address from `from-block` to head before going live. Registration - /// is idempotent — calling twice with the same arguments is a no-op. + /// Idempotent: calling twice with the same arguments is a no-op. register-address: func( chain-id: chain-id, template-name: string, address: list, // 20 bytes - from-block: option, ) -> result<_, host-error>; /// Remove an address from the watch set. Subsequent events on that - /// address are dropped. Idempotent — unregistering an unknown address - /// returns Ok. + /// address are dropped. Idempotent. unregister-address: func( chain-id: chain-id, template-name: string, @@ -71,8 +60,8 @@ interface chain { ```wit variant log-source { - static(u32), // existing — index into [[subscription]] - template(string), // new — name of the [[subscription.template]] + static(u32), // existing: index into [[subscription]] + template(string), // new: name of the [[subscription.template]] } ``` @@ -80,25 +69,23 @@ Module code (Rust, Uniswap V3 indexer): ```rust fn init(config: Vec<(String, String)>) -> Result<(), HostError> { - // Resume: re-register every pool we already discovered. Each pool's - // creation block is persisted; backfill resumes from where we left off. + // Resume: re-register every pool we already discovered. for key in local_store::list_keys("pool:")? { let pool = parse_addr(&key); - let from_block = u64::from_le_bytes(local_store::get(&key)?.unwrap()); - chain::register_address(1, "uniswap_v3_pool", &pool, Some(from_block))?; + chain::register_address(1, "uniswap_v3_pool", &pool)?; } Ok(()) } fn on_event(event: Event) -> Result<(), HostError> { match event { - // Factory event — declared as a static [[subscription]] for the factory. + // Factory event, declared as a static [[subscription]] for the factory. Event::Log(LogEvent { log, source: LogSource::Static(0) }) => { let pool = decode_pool_created(&log)?; - chain::register_address(1, "uniswap_v3_pool", &pool, Some(log.block_number))?; - local_store::set(&format!("pool:{}", hex(&pool)), &log.block_number.to_le_bytes())?; + chain::register_address(1, "uniswap_v3_pool", &pool)?; + local_store::set(&format!("pool:{}", hex(&pool)), b"")?; } - // Pool event — dispatched through the template. + // Pool event, dispatched through the template. Event::Log(LogEvent { log, source: LogSource::Template(name) }) if name == "uniswap_v3_pool" => { @@ -112,29 +99,29 @@ fn on_event(event: Event) -> Result<(), HostError> { Engine internals: -- **Boot**: read all `[[subscription.template]]` blocks across loaded modules. Initialise per-template address sets from `initial_addresses` plus the reserved key `__nexum:template:{name}:addresses` in the module's `local-store` namespace. -- **Live**: one `eth_subscribe logs` per chain per template, filter = `(topic in event_topics) ∧ (address in current_set)`. When `register-address` mutates the set, the engine re-subscribes (or shards the filter when the set exceeds the provider's address-limit threshold). -- **Backfill on register**: if `from-block` < current head, run paginated `eth_getLogs(topic, address, from-block, head)` synchronously before joining the live stream. Events from backfill are dispatched through the same `on_event` callback as live events. -- **Persistence**: engine writes `__nexum:template:{name}:addresses` (one entry per registered address) and `__nexum:template:{name}:cursor:{address}` (last block dispatched per address) in the module's reserved namespace. Resume is automatic if the module re-calls `register-address` for each persisted address; engine deduplicates via the persisted cursor. +- **Boot**: read all `[[subscription.template]]` blocks. Initialise per-template address sets from the reserved key `__nexum:template:{name}:addresses` in the module's `local-store` namespace if present from a prior run. +- **Live**: one `eth_subscribe logs` per chain per template with filter `(topic in event_topics) AND (address in current_set)`. When `register-address` mutates the set, the engine re-subscribes. +- **Persistence**: engine writes `__nexum:template:{name}:addresses` whenever the set changes. Resume after restart is automatic if the module re-calls `register-address` in `init`. + +Historical backfill is the module's responsibility, consistent with the catch-up pattern documented in `docs/02:260`. The module calls `chain.request("eth_getLogs", ...)` during `init` to replay history before going live. The engine does not backfill on `register-address`. ## Considered options -- **Ponder full-declarative** (factory address + event + parameter declared in manifest, engine extracts child). Rejected: schema must express ABI-aware extraction (`child = { source = "topic", index = 1 }` or `{ source = "data", offset = 32, length = 20 }`), and grows with every exotic factory shape (nested factories, multi-child events, address computed from multiple fields). The Envio model pushes that complexity into module Rust code — where it belongs, given the module already decodes events with `alloy_sol_types`. -- **Pure imperative `chain.open-log-stream(filter) -> stream-handle`.** Rejected: each call opens a new subscription, so N pools = N WSS connections. Doesn't scale to indexers with 10k+ tracked addresses. The Envio model keeps one subscription per template and mutates its address set — engine batches naturally. -- **Wildcard manifest** (`address = "*"` with topic-only filter, module client-side filters). Rejected: mainnet has ~10k contracts emitting `Transfer` / `Swap` per day. The engine would deliver every matching event to every wildcard subscriber; modules pay fuel and bandwidth to discard 99% of them. -- **Defer factory pattern entirely to 0.3.** Rejected: 0.2 is the breaking-change window per migration:522. Adding either `[[subscription.template]]` or `register-address` after 0.2 requires another major bump. Better to land the small surface now than break later. -- **Templates declared inside `[[subscription]]` with optional `address` (one block, two modes).** Rejected: conflates two semantically distinct cases — modules looking at `[[subscription]]` would have to inspect for the presence of `address` to know whether they need to call `register-address`. Separate block name is clearer. -- **Engine extracts the child address from the static factory `[[subscription]]` (best of both).** Rejected: would require the manifest to identify a relationship between the static subscription and a template, plus the ABI extraction rules. Reintroduces the Ponder schema complexity we just rejected. +- **Ponder full-declarative** (factory address, event, parameter declared in manifest; engine extracts child). Rejected: schema must express ABI-aware extraction (`child = { source = "topic", index = 1 }` or similar), and grows with every exotic factory shape (nested factories, multi-child events, address computed from multiple fields). The Envio model pushes that complexity into module Rust code, where it belongs given the module already decodes events with `alloy_sol_types`. +- **Pure imperative `chain.open-log-stream(filter) -> stream-handle`.** Rejected: each call opens a new subscription, so N pools means N WSS connections. Doesn't scale to indexers with 10k+ tracked addresses. The Envio model keeps one subscription per template and mutates its address set; engine batches naturally. +- **Engine-driven historical backfill on register** (with `from-block` parameter). Rejected after PR review flagged the added complexity. Module-driven catch-up via `init` + `eth_getLogs` already exists in mfw's design (`docs/02:260`) and covers the same use case without adding engine state (per-address cursor, paginated `eth_getLogs` orchestration). M3 SDK can ship a helper that wraps the pattern. +- **Wildcard manifest** (`address = "*"` with topic-only filter, module client-side filters). Rejected: mainnet has ~10k contracts emitting `Transfer` or `Swap` per day. The engine would deliver every matching event to every wildcard subscriber; modules pay fuel and bandwidth to discard 99% of them. +- **Defer factory pattern entirely to 0.3.** Rejected: 0.2 is the breaking-change window per migration:522. Adding either `[[subscription.template]]` or `register-address` after 0.2 requires another major bump. +- **Templates declared inside `[[subscription]]` with optional `address` (one block, two modes).** Rejected: conflates two semantically distinct cases. Modules looking at `[[subscription]]` would have to inspect for the presence of `address` to know whether they need to call `register-address`. Separate block name is clearer. ## Consequences -- `nexum.toml` schema gains `[[subscription.template]]` with `chain_id`, `name`, `event_topics`, optional `initial_addresses`. mfw78 approval needed for the schema extension (his spec). -- `nexum:host/chain` gains `register-address` and `unregister-address` functions; `nexum:host/event-module`'s `log-source` variant gains `template(string)`. mfw78 approval needed for the WIT change (his namespace). These are the only WIT additions in this ADR — small, focused, with no implicit dependencies on other interfaces. -- Reserved key namespace `__nexum:template:*` in each module's `local-store` namespace. Modules MUST NOT write to keys with this prefix; engine reserves them. -- Module boilerplate per factory ≈ 5–10 lines (decode the factory event, call `register-address`, persist for resume). The M3 SDK is expected to ship a helper that encapsulates this — something like `Factory::::on_event(register_template("uniswap_v3_pool"))` — but the host surface is intentionally simple enough that no SDK is required to use it. -- Engine can register addresses sourced from anywhere — factory events, HTTP API responses, governance votes, operator-supplied lists. Composability is a deliberate feature; the engine treats every registration the same way regardless of provenance. -- Nested factories (a child contract that is itself a factory) work without schema changes: the child's event handler decodes its own creation events and calls `register-address` on the grandchild template. Engine has no concept of nesting; it just multiplexes addresses per template. -- Conditional registration ("only register pools with fee = 3000") works without schema changes: the module's factory-event handler inspects the event payload and decides whether to call `register-address`. -- Backfill cost: a module registering 10k addresses with `from-block` deep in history triggers 10k paginated `eth_getLogs` runs, sequentially per address (the engine cannot batch across addresses with different `from-block` cursors without state-machine work that's not in scope for 0.2). Operators should set sensible `start_block` boundaries; the M3 SDK is expected to ship a `BulkBackfill` helper that groups same-cursor addresses into combined filter queries. -- The address set per template is bounded only by `local-store` quota. The engine enforces a soft cap (default 50k addresses per template) configurable in `engine.toml` to prevent a runaway module from saturating the provider's filter limits; exceeding the cap returns `host-error.denied` from `register-address`. -- A module that never calls `register-address` for a declared template receives no events from it — equivalent to declaring an unused subscription. Engine logs a warning on boot if a template is declared with `initial_addresses = []` and the module has no registrations after `init` returns. +- `nexum.toml` schema gains `[[subscription.template]]` with `chain_id`, `name`, `event_topics`. Schema extension needs upstream approval. +- `nexum:host/chain` gains `register-address` and `unregister-address`; `nexum:host/event-module`'s `log-source` variant gains `template(string)`. WIT change needs upstream approval. +- Reserved key namespace `__nexum:template:*` in each module's `local-store` namespace. Modules MUST NOT write to keys with this prefix. +- Module boilerplate per factory is roughly 5 lines (decode the factory event, call `register-address`, persist for resume). The M3 SDK can ship a helper that wraps it. +- Register sources are not limited to factory events. A module can register addresses from any signal: HTTP API responses, governance votes, operator-supplied lists. Composability is a deliberate feature. +- Nested factories (a child contract that is itself a factory) work without schema changes. The child's event handler calls `register-address` on the grandchild template. +- Conditional registration ("only register pools with fee = 3000") works without schema changes. The module's factory-event handler decides. +- The address set per template is bounded only by `local-store` quota. The engine enforces a soft cap (default 50k addresses per template) configurable in `engine.toml`; exceeding the cap returns `host-error.denied` from `register-address`. +- Open follow-up: whether to support `from-block` historical backfill on register is left for upstream discussion. The minimal surface here can be extended additively if needed. From 57283be3a559609bcbab2238425cf3194a25290b Mon Sep 17 00:00:00 2001 From: brunota20 Date: Mon, 8 Jun 2026 14:39:17 -0300 Subject: [PATCH 22/29] fix(docs): revised ADRs and diagrams --- ...01-engine-toml-separate-from-nexum-toml.md | 18 +- .../0002-provider-pool-transport-by-scheme.md | 11 +- docs/adr/0003-local-store-namespacing.md | 34 +- .../0004-patch-cowprotocol-to-bleu-cow-rs.md | 10 +- .../0005-cow-api-via-cached-orderbookapi.md | 8 +- .../adr/0006-cow-twap-ethflow-host-helpers.md | 91 +--- .../0007-upstream-protocol-logic-to-cow-rs.md | 37 +- .../0008-factory-subscriptions-in-manifest.md | 171 ++----- docs/diagrams/README.md | 31 ++ docs/diagrams/architecture.mmd | 69 +++ docs/diagrams/diagrams.md | 472 ++++++++++++++++++ docs/diagrams/engine-boot.mmd | 59 +++ docs/diagrams/module-lifecycle.mmd | 39 ++ docs/diagrams/sequence-ethflow.mmd | 39 ++ docs/diagrams/sequence-twap.mmd | 56 +++ docs/diagrams/subscription-dispatch.mmd | 61 +++ docs/diagrams/wit-call-path.mmd | 34 ++ 17 files changed, 1009 insertions(+), 231 deletions(-) create mode 100644 docs/diagrams/README.md create mode 100644 docs/diagrams/architecture.mmd create mode 100644 docs/diagrams/diagrams.md create mode 100644 docs/diagrams/engine-boot.mmd create mode 100644 docs/diagrams/module-lifecycle.mmd create mode 100644 docs/diagrams/sequence-ethflow.mmd create mode 100644 docs/diagrams/sequence-twap.mmd create mode 100644 docs/diagrams/subscription-dispatch.mmd create mode 100644 docs/diagrams/wit-call-path.mmd diff --git a/docs/adr/0001-engine-toml-separate-from-nexum-toml.md b/docs/adr/0001-engine-toml-separate-from-nexum-toml.md index fd77ac3..ab6c1f4 100644 --- a/docs/adr/0001-engine-toml-separate-from-nexum-toml.md +++ b/docs/adr/0001-engine-toml-separate-from-nexum-toml.md @@ -3,30 +3,34 @@ status: proposed implemented-in: nullislabs/shepherd#8, nullislabs/shepherd#9 --- -# Operator config (`engine.toml`) is separate from module manifest (`nexum.toml`) +# Operator config (`engine.toml`) is separate from module manifest (`module.toml`) ## Context The engine needs two distinct kinds of configuration: what the **operator** decides at deployment time (which chains to connect to, where the local-store database lives, which modules to boot) and what the **module developer** declares at build time (required and optional capabilities, HTTP allowlist, module-specific config keys). These have different reviewers, different threat models, and change on different cadences. +The filenames need to signal who owns each file directly. An operator opening a config file should know without prior context whether the file is their concern or the module developer's. A name like `nexum.toml` requires the reader to know that "nexum" refers to the runtime that hosts the module, which is one indirection too many; `module.toml` reads as "the module's manifest" with no prior context. + ## Decision Two distinct files, distinct schemas, distinct loaders: - **`engine.toml`** — operator-owned, lives next to the engine binary or pointed to by `--engine-config`. Defines `[engine]` (state_dir, log_level), `[chains.]` (rpc_url), and `[[modules]]` (path, manifest). Loaded by `engine_config::EngineConfig::load`. -- **`nexum.toml`** — module-developer-owned, ships in the module's bundle alongside its `.wasm` component. Defines `[module]`, `[capabilities]` (required, optional, http allowlist), `[config]`. Loaded by `manifest::load`. +- **`module.toml`** — module-developer-owned, ships in the module's bundle alongside its `.wasm` component. Defines `[module]`, `[capabilities]` (required, optional, http allowlist), `[config]`. Loaded by `manifest::load`. -The engine config carries the path to each module's manifest; the two never collapse into one file. +The engine config carries the path to each module's manifest; the two never collapse into one file. The names `engine.toml` and `module.toml` map directly onto the two distinct roles, so a reader reaching either file knows whose concerns it covers. ## Considered options - **Single `shepherd.toml` with `[engine]`, `[chains]`, `[[modules]]` *and* nested `[modules..capabilities]` per module.** Rejected: conflates operator and developer concerns. A module's capability declaration is a property of the build, not the deployment — it belongs in the artifact, not in the operator's local file. Auditing a module's capabilities also becomes a per-deployment exercise instead of a property visible in the published bundle. -- **`nexum.toml` inside the engine config (module entries embed it inline).** Rejected for the same reason; also bloats `engine.toml`. +- **Keep the `nexum.toml` filename for the module manifest.** Rejected: the name does not signal who owns the file (engine vs module). `module.toml` reads as "the module's manifest" without prior context. +- **`module.toml` inside the engine config (module entries embed it inline).** Rejected for the same reason as the single-file proposal; also bloats `engine.toml`. - **Drop `engine.toml` entirely; pass everything as CLI flags or env vars.** Rejected: per-chain RPC URLs and module lists are awkward as flags, and `RUST_LOG` already covers the only thing that env vars naturally express. ## Consequences - A deployment needs both files. A missing `engine.toml` falls back to "no chains, default state_dir" — the example logging module still runs; cow-api / chain backends report `unsupported`. -- A missing `nexum.toml` triggers the 0.1-compat deprecation warning in `manifest::fallback_manifest()` (defined in `crates/nexum-engine/src/manifest.rs`) and treats every linked capability as required. This fallback is scheduled for removal in 0.3 per `docs/migration/0.1-to-0.2.md`. -- Module-bundle redistribution carries `nexum.toml` with the artifact; engines do not need to ship templates. -- Future content-addressed module distribution (0.3) embeds `nexum.toml` in the bundle hash; `engine.toml` references modules by content address rather than filesystem path. The split survives that migration unchanged. +- A missing `module.toml` triggers the 0.1-compat deprecation warning in `manifest::fallback_manifest()` (defined in `crates/nexum-engine/src/manifest.rs`) and treats every linked capability as required. This fallback is scheduled for removal in 0.3 per `docs/migration/0.1-to-0.2.md`. +- Module-bundle redistribution carries `module.toml` with the artifact; engines do not need to ship templates. +- Future content-addressed module distribution (0.3) embeds `module.toml` in the bundle hash; `engine.toml` references modules by content address rather than filesystem path. The split survives that migration unchanged. +- Implementation impact: `crates/nexum-engine/src/manifest.rs` and `engine_config.rs` need to update the filename lookup from `nexum.toml` to `module.toml`. The 0.1-compat fallback in `manifest::fallback_manifest()` should accept both names during the transition; after 0.3 only `module.toml` is recognised. diff --git a/docs/adr/0002-provider-pool-transport-by-scheme.md b/docs/adr/0002-provider-pool-transport-by-scheme.md index f027480..c8e8fab 100644 --- a/docs/adr/0002-provider-pool-transport-by-scheme.md +++ b/docs/adr/0002-provider-pool-transport-by-scheme.md @@ -13,11 +13,18 @@ implemented-in: nullislabs/shepherd#8, nullislabs/shepherd#9 The `ProviderPool::from_config` constructor reads each chain's `rpc_url` and switches by URL scheme prefix: -- `ws://` or `wss://` → `ProviderBuilder::new().connect_ws(WsConnect::new(url))`. Pubsub transport. Subscriptions and request/response both work. -- `http://` or `https://` → `ProviderBuilder::new().connect_http(parsed)`. HTTP transport. Request/response only; `subscribe-blocks` and `subscribe-logs` surface as a host error to the guest. +- `ws://` or `wss://` → `ProviderBuilder::new().connect_ws(WsConnect::new(url))`. Pubsub transport. Subscriptions and request/response both work. **This is the recommended configuration for any chain a module subscribes to.** +- `http://` or `https://` → `ProviderBuilder::new().connect_http(parsed)`. HTTP transport. Request/response only; `subscribe-blocks` and `subscribe-logs` surface as `host-error.unsupported` to the guest. Both transports erase to `DynProvider` so the rest of the engine is transport-agnostic. +Alloy is capable of emulating `eth_subscribe` on HTTP via polling, but this is intentionally **not** enabled. The engine takes an opinionated stance favouring WebSockets for subscriptions; operators who want push-based events configure WSS endpoints. HTTP-only chains are supported for `request` traffic but not for subscriptions. + +## Non-goals + +- **RPC failover, load balancing, and retry policies are explicitly out of scope for the engine.** This logic lives in upstream crates (alloy ships tower-style middleware for timeout / retry / rate-limit / fallback endpoint). The engine does not roll its own. Operators wanting failover configure it via alloy provider builders before passing them through, or rely on the provider's own fallback (Alchemy, Infura, etc. handle it server-side). +- Re-routing requests across chains, rebalancing across pools within a chain, and similar provider-management concerns are likewise alloy's responsibility. + ## Considered options - **Force WSS everywhere.** Rejected: many providers (Alchemy, Infura, self-hosted RPC) expose HTTP-only on free tiers, and modules that only need `request` (no subscriptions) shouldn't be blocked by a WSS requirement. diff --git a/docs/adr/0003-local-store-namespacing.md b/docs/adr/0003-local-store-namespacing.md index f265137..47d5378 100644 --- a/docs/adr/0003-local-store-namespacing.md +++ b/docs/adr/0003-local-store-namespacing.md @@ -3,31 +3,47 @@ status: proposed implemented-in: nullislabs/shepherd#8 --- -# Per-module namespacing in `local-store` via `[len:u8][module][key]` prefix +# Per-module namespacing in `local-store` via 32-byte deterministic hash prefix ## Context -`nexum:host/local-store` is a key-value store shared across all modules the engine runs. Two modules using the same key string (e.g. `"last-block"`) must see disjoint values; one module must never read or overwrite another's data. The engine knows each module's name at instantiation time, so namespacing is a host-side concern. +`nexum:host/local-store` is a key-value store shared across all modules the engine runs. Two modules using the same key string (e.g. `"last-block"`) must see disjoint values; one module must never read or overwrite another's data. The engine knows each module's identity at instantiation time, so namespacing is a host-side concern. + +Two properties matter for the namespace prefix: + +1. **Deterministic and unspoofable.** An arbitrary `module_name` string read out of `module.toml` lets a malicious or careless operator give two modules the same name and have one read the other's state. A fixed-size hash derived from the module's canonical identity is harder to collide and removes the operator-supplied-text attack surface. +2. **Composes with ENS-based module discovery** (per `docs/03-module-discovery.md`): when a module is identified by an ENS name (e.g. `twap-monitor.shepherd.eth`), the ENS namehash is a natural prefix. ENS TXT records pinning the `.wasm` content hash provide a separate verification path against the loaded bundle. ## Decision Single redb database file at `EngineConfig.engine.state_dir`, single shared table `nexum:local-store`. Every key handed to redb is composed host-side as: ``` -[len: u8] [module_name: len bytes] [raw key: rest of the bytes] +[32-byte namespace prefix][raw key bytes] ``` -Module names longer than 255 bytes are rejected at `LocalStore` construction (matches the one-byte length prefix). Modules see plain key strings on both the read and write paths; the prefix is invisible to the WIT-facing API. +The 32-byte prefix is computed deterministically from the module's canonical identity: + +- **ENS-identified modules** (M3+, per `docs/03`): prefix is `ens_namehash(name)` (EIP-137), e.g. `namehash("twap-monitor.shepherd.eth")`. +- **Locally-loaded modules** (current 0.2 scope, no ENS): prefix is `keccak256(module_name)` where `module_name` comes from `module.toml`'s `[module].name` field. + +Both produce a 32-byte digest with the same domain, so a module loaded locally during development and later published under an ENS name can keep its existing state by registering an alias (`alias = keccak256(name)`) the engine recognises during the migration window. The exact alias mechanism is out of scope for this ADR. + +Modules see plain key strings on both the read and write paths; the prefix is invisible to the WIT-facing API. ## Considered options -- **Separator string** (`{module}:{key}`). Rejected: any module name containing `:` collides with another module's `:`-bearing key. Length prefix is unambiguous regardless of payload bytes. +- **Separator string** (`{module}:{key}`). Rejected: any module name containing `:` collides with another module's `:`-bearing key. A fixed-size hash is unambiguous regardless of payload bytes. +- **`[len:u8][module_name][key]` length-prefixed string.** Rejected: spoofable (the name is operator-supplied text), and does not align with the ENS-based discovery path that 0.3 will introduce. The 32-byte hash is deterministic and namespace-uniform. - **One redb database file per module.** Rejected: multiplies open file handles linearly in modules, blocks any future cross-module atomic operations (not currently planned but cheap to keep on the table), and complicates backup tooling (N files vs 1). - **One redb *table* per module within a single file.** Rejected: redb `TableDefinition` lifetimes are `'static`, so table names must be known at compile time. Dynamic table opening per module would force string-leak workarounds and exposes the same name-collision question as separator-based keys. +- **Engine-allocated incrementing module id.** Rejected: stable across reboots only if the engine persists the allocation table, which adds a chicken-and-egg dependency on the local-store itself. Determinism from the name avoids the dependency entirely. ## Consequences -- Module data is physically interleaved in the redb tree (range scans for one module's keys are O(log n + module-key-count) — fine for our workload). -- Migrations changing the namespacing layout break every existing module's persisted state. The format must stay stable through 0.x. -- A module's `list-keys` (when added) iterates over the namespace range; the host strips the prefix before returning to the guest. -- 255-byte module-name limit is enforced loudly at construction, so configuration errors surface at boot rather than silently corrupting data at first write. +- The prefix is fixed-size (32 bytes) and independent of module name length. Range scans over a single module's keys are O(log n + module-key-count) — fine for our workload. +- Migrations changing the prefix derivation (e.g., switching the local-mode hash function or the ENS resolver) would orphan every existing module's persisted state. The derivation must stay stable through 0.x; ENS-mode introduction in 0.3 happens additively via the alias mechanism, not by changing existing prefixes. +- A module's `list-keys` iterates over the namespace range (32-byte prefix scan); the host strips the prefix before returning to the guest. +- Module data versioning (schema migrations across module versions) is the module's responsibility. The local-store does not version values; modules MAY embed a `schema_version` byte in their stored payloads and migrate on `init` when the read value's version differs from the current code's expectation. +- ENS-based discovery (per docs/03) integrates without a prefix-format change: when a module is loaded by ENS name, the prefix is `namehash(name)`. The corresponding `.wasm` content hash is verified via ENS TXT records before loading, separately from the local-store prefix derivation. +- Spoofing protection: an operator cannot make module A read module B's state by renaming, because the prefix is the hash of the canonical name. Renaming a module to match another's name produces a name conflict the engine refuses at boot, rather than silent state takeover. diff --git a/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md b/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md index 8dfb931..c8ba48b 100644 --- a/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md +++ b/docs/adr/0004-patch-cowprotocol-to-bleu-cow-rs.md @@ -10,10 +10,10 @@ implemented-in: nullislabs/shepherd#10 `cowprotocol` v1.0.0-alpha.3 (the version on crates.io) was cut from an early snapshot of `cowdao-grants/cow-rs` PR #5 at commit `1742ffa`. That PR is still open and is the canonical upstream channel for landing additions to the Rust SDK. Its head branch is `bleu/cow-rs:main`, currently at commit `c012404`, carrying 18 follow-up commits the engine materially depends on: - `composable::Proof` byte-width fix (consumed by the TWAP poll path). -- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding from the PR #5 review). +- `OrderCreation` zero-`from` fast-fail (closes a MEDIUM severity finding in PR #5). - `order_book` / `composable` submodule splits (cleaner imports on the engine side). -ADR-0007 commits us to landing TWAP, EthFlow, and `OrderPostError` primitives into PR #5 directly, by pushing additional commits to its head branch. Each commit advances both PR #5 and the patch rev consumed here. +ADR-0007 commits us to landing three protocol-level primitives into PR #5 directly (`OrderPostError` rich variants + `retry_hint`, `OrderBookApi::with_base_url`, and `wasm32` feature-gating) by pushing additional commits to its head branch. Each commit advances both PR #5 and the patch rev consumed here. There is no published `alpha.4` and no scheduled date for one; the engine cannot wait. @@ -25,14 +25,14 @@ This is not a parallel fork. `bleu/cow-rs:main` IS the head branch of upstream P ## Considered options -- **Vendor the missing types locally.** Rejected: re-implementing `composable::Proof`, `OrderCreation`, etc. in the engine repo is the AI-duplication anti-pattern flagged by reviewers on cow-rs PR #5. Reuse over reimplement applies. +- **Vendor the missing types locally.** Rejected: re-implementing `composable::Proof`, `OrderCreation`, etc. in the engine repo is the AI-duplication anti-pattern that the cow-rs SDK already solves. Reuse over reimplement applies. - **Pin every dependent to `cow-rs` git directly.** Works but every new workspace member has to remember the git source. `[patch.crates-io]` centralises the override. -- **Open a separate PR per primitive against `cowdao-grants/cow-rs`.** Rejected: fragments review across multiple PRs when one already exists at the appropriate granularity. Stacking commits on PR #5 keeps the review thread coherent and lets reviewers track the cumulative change. +- **Open a separate PR per primitive against `cowdao-grants/cow-rs`.** Rejected: fragments the change across multiple PRs when one already exists at the appropriate granularity. Stacking commits on PR #5 keeps the change coherent and lets the cumulative diff be tracked in one place. - **Wait for `alpha.4` to publish.** No ETA; the TWAP/EthFlow milestone cannot land without `composable::Proof` correct. ## Consequences - `cargo update` will re-resolve to the same `rev`; the lock pins it. - Bumping the rev is a single-line workspace edit; reviewers see one diff per primitive added to PR #5. -- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-helper additions (`composable::poll_and_build_order`, `eth_flow::decode_placement`, `OrderPostError` rich variants). Until then, expect the patch rev to advance with every push to PR #5. +- Drop the patch entirely once a published `cowprotocol` release contains both the alpha.3 follow-ups and the ADR-0007 protocol-primitive additions (`OrderPostError` rich variants + `retry_hint`, `OrderBookApi::with_base_url`, `wasm32` feature-gate). Until then, expect the patch rev to advance with every push to PR #5. - Modules built against this workspace inherit the patch transitively; modules built standalone against crates.io will see `alpha.3` and may hit the very bugs the patch closes. Flag this in the SDK README when M3 lands. diff --git a/docs/adr/0005-cow-api-via-cached-orderbookapi.md b/docs/adr/0005-cow-api-via-cached-orderbookapi.md index 5b37019..0862d54 100644 --- a/docs/adr/0005-cow-api-via-cached-orderbookapi.md +++ b/docs/adr/0005-cow-api-via-cached-orderbookapi.md @@ -11,7 +11,7 @@ implemented-in: nullislabs/shepherd#8 ## Decision -At engine boot, construct one `cowprotocol::OrderBookApi` per `cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, ArbitrumOne, Base) into a `BTreeMap` keyed by EVM chain id. "Cached" here means built once during boot and reused for the engine's lifetime; clients are not lazy-constructed on each call nor LRU-evicted. The map is created in `OrderBookPool::with_default_chains()` and never mutated after. +At engine boot, construct one `cowprotocol::OrderBookApi` per `cowprotocol::Chain` variant (currently Mainnet, Gnosis, Sepolia, ArbitrumOne, Base) into a `BTreeMap` keyed by EVM chain id. "Cached" here means built once during boot and reused for the engine's lifetime; clients are not lazy-constructed on each call nor LRU-evicted. The pool implements `Default` so callers instantiate it as `OrderBookPool::default()`; the trait impl populates the map with one entry per `cowprotocol::Chain` variant. Both `cow-api` operations consult this pool: @@ -22,13 +22,13 @@ Chains not in `cowprotocol::Chain` return `HostError { kind: unsupported }` at t ## Considered options -- **Raw `reqwest` for both.** Rejected: forces us to maintain the chain → base-URL table (drifts whenever cowprotocol adds a chain) and reimplement `post_order`'s body codec and error mapping, the exact duplication flagged in cow-rs PR #5 review. +- **Raw `reqwest` for both.** Rejected: forces us to maintain the chain → base-URL table (drifts whenever cowprotocol adds a chain) and reimplement `post_order`'s body codec and error mapping, the exact duplication the cow-rs SDK already eliminates. - **`OrderBookApi` for `submit-order`, raw `reqwest` for `request`.** Tempting (request is opaque to the crate) but means two separate chain-resolution paths, two HTTP clients, and a second place to keep the chain set in sync. - **Build `OrderBookApi` lazily on first call per chain.** Rejected: hides config errors at runtime. Up-front boot construction surfaces unknown chains immediately and amortises away the per-call cost. ## Consequences -- Operator-supplied custom orderbook URLs (barn, staging, forked deployments) are out of scope for the default constructor and require a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` constructor in the cow-rs crate (ADR-0007 item 4 — not vendored locally). +- Operator-supplied custom orderbook URLs (barn, staging, forked deployments) are out of scope for the default constructor and require a follow-on `OrderBookApi::with_base_url(chain_id, base_url)` constructor in the cow-rs crate (ADR-0007 item 2, not vendored locally). - Adding a chain means a `cowprotocol::Chain` variant lands in cow-rs first; the engine inherits it on the next patched rev bump. - The shared `reqwest::Client` enables connection pooling across both `request` and `submit-order` paths. -- TWAP and EthFlow helpers (ADR-0006) reuse the same pool — no duplicated client construction in those host wrappers. +- Guest-side TWAP and EthFlow modules (ADR-0006) submit orders through this `cow-api` interface; no specialised host helpers wrap it. diff --git a/docs/adr/0006-cow-twap-ethflow-host-helpers.md b/docs/adr/0006-cow-twap-ethflow-host-helpers.md index 300709a..0e9e776 100644 --- a/docs/adr/0006-cow-twap-ethflow-host-helpers.md +++ b/docs/adr/0006-cow-twap-ethflow-host-helpers.md @@ -2,85 +2,48 @@ status: proposed --- -# TWAP and EthFlow as intent helpers in `shepherd:cow@0.2.0` +# TWAP and EthFlow run as guest modules using low-level host primitives (no specialised `shepherd:cow` interfaces) ## Context -The reference engine already exposes `shepherd:cow/cow-api` for raw orderbook access (REST passthrough + `submit-order`). Two further CoW workflows show up in every non-trivial module: ComposableCoW conditional orders (TWAP being the canonical example) and EthFlow native-ETH orders. Both follow the same external-indexer/relayer pattern that CoW maintainers have signalled intent to extract from the monolithic `cowprotocol/services` repository: +TWAP (over ComposableCoW) and EthFlow are the two CoW workflows the M2 grant ships modules for. The natural-seeming approach is to add `shepherd:cow/twap` and `shepherd:cow/ethflow` WIT interfaces that the host implements on top of `cowprotocol` crate primitives, so modules would call `twap.poll-and-submit(...)` and `ethflow.submit-from-log(...)` as host functions. This ADR rejects that direction. -- **TWAP / ComposableCoW** is already extracted as the standalone `cowprotocol/watch-tower` (TypeScript). Listens to `ConditionalOrderCreated`, polls `getTradeableOrderWithSignature` on each block, posts to the orderbook when an order becomes tradeable. -- **EthFlow indexer** still lives inside `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`. Listens to `OrderPlacement` / `OrderInvalidation` / `OrderRefund`, inserts into the `ethflow_orders` table. The intent is to extract it into a standalone service following the same path `watch-tower` and the `refunder` crate already took. The Shepherd `ethflow-watcher` module is positioned as that extraction. +The dividing line is protocol vs implementation. CoW Protocol primitives — order types, signing schemes, the orderbook REST surface — are protocol concerns and belong in shared layers (`cowprotocol` crate, `shepherd:cow/cow-api` interface). TWAP is one of many strategies built _on top of_ those primitives; ComposableCoW is the contract surface a TWAP module observes, but the act of polling, deciding when to submit, and reacting to orderbook errors is application logic. Putting that application logic in the host or in `cowprotocol` couples every consumer to one implementation and one error-handling policy. -Both flows share the same pattern: observe an on-chain event, derive a signed `OrderCreation`, submit it to the orderbook. The derivation has enough protocol detail (signing scheme, ComposableCoW eth_call, EthFlow EIP-1271 contract signature, log decoding) that a guest module would either ship that logic itself (large WASM, duplicates work in the `cowprotocol` Rust SDK) or make ten round-trips to the host through generic `chain`/`cow-api` calls. - -Per ADR-0007, the protocol logic itself lives in the `cowprotocol` crate, not in `nexum-engine`. This ADR consequently scopes the engine-side helpers to the WIT surface and the glue that wires the upstream primitives into the host call boundary. - -The newer ComposableCoW iteration in development simplifies polling versus the watch-tower TypeScript implementation: less of the SDK's discriminated `PollResultCode` mapping may need to be replicated in `cowprotocol::composable` for `twap.poll-and-submit` to work. The rich `PollOutcome` variants described below remain the target API surface; the upstream implementation may end up simpler than the watch-tower mirror suggests. +Embedding a concrete TWAP implementation in an SDK is an architectural smell the grant explicitly seeks to alleviate. The grant seeks to enable Shepherd as the runtime where many independent strategy implementations coexist, each compiled to its own WASM module. A specialised `twap` interface in the host would defeat that goal: every Shepherd deployment would have to use the same polling implementation, the same error-mapping, the same retry hints, with no room for different strategies to differ on those choices. ## Decision -Add two new interfaces to package `shepherd:cow@0.2.0`: - -```wit -interface twap { - use nexum:host/types@0.2.0.{chain-id, log, host-error}; - use cow-api.{order-uid}; - - /// Discriminated outcome of a single poll attempt against - /// ComposableCoW. Mirrors watchtower's PollResultCode so modules - /// avoid spamming RPC/orderbook when an order is known-not-ready. - variant poll-outcome { - submitted(order-uid), - try-at-epoch(u64), // unix seconds; module skips polls until then - try-on-block(u64), // specific block number - try-next-block, // default retry - dont-try-again, // terminal: TWAP completed or cancelled - } - - poll-and-submit: func( - chain-id: chain-id, - registration: log, - ) -> result; -} - -interface ethflow { - use nexum:host/types@0.2.0.{chain-id, log, host-error}; - use cow-api.{order-uid}; - - submit-from-log: func( - chain-id: chain-id, - placement: log, - ) -> result; -} -``` +The `shepherd:cow` WIT package contains only the existing `cow-api` interface (REST passthrough + `submit-order`), which is protocol-level. No `twap` interface, no `ethflow` interface, no host-side helpers specific to either workflow. -Both interfaces ship in the existing `shepherd` world alongside `cow-api`. `order-uid` is added to `cow-api` as `type order-uid = list` (56 bytes, validated host-side) and reused by all three interfaces; `cow-api/submit-order` keeps returning it instead of `string`. Capability names `"twap"` and `"ethflow"` are appended to `KNOWN_CAPABILITIES` so manifests can declare them under `[capabilities].required`. +TWAP and EthFlow modules implement their logic in Rust guest code using: -Host implementations are thin wrappers (~20–30 LOC each) over three upstream primitives that land in `cowprotocol` first (see ADR-0007): +- **`nexum:host/chain`** — `request` (for `eth_call`, `eth_getLogs`, etc.), `subscribe-blocks`, `subscribe-logs`. +- **`nexum:host/local-store`** — for watch lists, cursors, and backoff state. +- **`nexum:host/logging`** — for structured logs. +- **`shepherd:cow/cow-api`** — `submit-order` for orderbook submission. +- **`cowprotocol` crate** (consumed directly by the module, gated on the wasm32 feature work in ADR-0007) — for protocol types: `Order`, `OrderCreation`, `OrderUid`, signing schemes, `OrderPostError`, etc. +- **`alloy_sol_types`** (or equivalent) — for ABI-aware decoding of `ConditionalOrderCreated`, `OrderPlacement`, `getTradeableOrderWithSignature` return values, and similar Solidity-typed payloads. -- `cowprotocol::composable::poll_and_build_order(provider, owner, params, proof) -> PollOutcome` — returns the same discriminated outcome (`Submitted`, `TryAtEpoch`, `TryOnBlock`, `TryNextBlock`, `DontTryAgain`). Backs `twap.poll-and-submit`. -- `cowprotocol::eth_flow::decode_placement(log)` — decodes `OrderPlacement` into `(owner, OrderCreation, OrderUid)` with the EIP-1271 signing scheme pointing at the EthFlow contract. Backs `ethflow.submit-from-log`. -- `cowprotocol::OrderPostError` (rich variants + `retry_hint()`) — typed orderbook submission errors with backoff/drop classification. Modules consume the hints to react to transient vs permanent failures without spamming. +Concretely, a TWAP module's `on_event(block)` handler iterates the local-store watch set, makes an `eth_call` to `ComposableCoW.getTradeableOrderWithSignature(owner, params, "", [])` via `chain.request`, decodes the return (or revert reason) with `alloy_sol_types`, constructs an `OrderCreation` with `cowprotocol` types, and submits via `cow-api/submit-order`. Orderbook errors are interpreted via `OrderPostError::retry_hint()` (ADR-0007). Backoff state is persisted to `local-store`. All of this lives in module Rust source, not in the engine. -The engine wires these primitives into HostState and maps their errors to `host-error` kinds; no protocol logic lives in `nexum-engine`. Modules continue to declare their own log subscriptions via `[[subscription]]` in `nexum.toml`; the helpers only decode and submit, they do not auto-subscribe. +An EthFlow module's `on_event(log)` handler decodes the `OrderPlacement` event with `alloy_sol_types`, constructs the `OrderCreation` (with the EIP-1271 signing scheme pointing at the `CoWSwapEthFlow` contract), and submits the same way. Module-side, no host helper required. ## Considered options -- **Low-level primitives only** (`chain.eth-call`, `chain.keccak256`, `chain.sign-digest`, raw `cow-api/submit-order`). Maximally orthogonal, but every guest module re-derives the same EIP-712 / GPv2 / ComposableCoW / EthFlow glue. "Reuse over reimplement" applies: that derivation already lives in `cowprotocol::{Order, OrderBookApi, eth_flow, composable}` and should not be re-shipped in every WASM artifact. -- **Implement the protocol glue inside `nexum-engine` host code, port upstream later.** Rejected per ADR-0007: every line of TWAP polling or EthFlow decoding that lives in the engine is a line that future Rust consumers cannot reuse, and a line that diverges as cow-rs evolves. -- **EthFlow as pure passive observer (no `submit-from-log`).** Briefly considered after reading "watcher" / "monitor" in docs/00 and docs/04 as "no submission". Rejected after verifying that CoW's own autopilot DOES post equivalent (insert into `ethflow_orders` table); the Shepherd module is intended to externalize that role, not replace it with passive observation. The `pending_orders` state mentioned in docs/04 is a side-effect of the relay (local accounting of what's been observed), not the goal. -- **Simple `option` return on twap instead of `poll-outcome` variant.** A 1-hour-spaced TWAP polled every block would spam ~300 RPC calls per part with `None` returns. The richer outcome (`try-at-epoch`, etc.) matches watchtower's existing `PollResult` and lets modules skip polls until the contract says it's worth retrying. Production-critical. -- **Single combined interface** `shepherd:cow/orders` with both helpers. Cheaper world surface but harder to gate per-capability — a module that only watches EthFlow shouldn't have to import TWAP and vice versa. Splitting keeps `[capabilities].required` honest. -- **`log-json: list` payload** instead of the typed `nexum:host/types.log` record. The record already exists and the engine's event dispatch already projects `alloy_rpc_types_eth::Log` into it, so reuse wins on both ergonomics and "no duplicate decoders". -- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 helper only handles `ComposableCoW.create()` (empty proof, single conditional order). `setRoot` polling requires off-chain proof derivation that itself warrants a separate helper (`twap.poll-and-submit-with-proof`) once a module actually needs it. -- **Bumping the package to `shepherd:cow@0.3.0`.** Not needed: adding imports to an existing world is additive under WIT subsumption rules. Modules compiled against the current 0.2.0 surface continue to build. +- **Specialised `shepherd:cow/twap` and `shepherd:cow/ethflow` interfaces** with rich `PollOutcome` variants and per-event host helpers, backed by `composable::poll_and_build_order` and `eth_flow::decode_placement` primitives in the `cowprotocol` crate. Rejected: this puts a single concrete TWAP / EthFlow implementation behind a WIT boundary, forcing every Shepherd deployment to use the same polling policy, the same error-mapping, the same retry hints. It also blurs the protocol-vs-implementation boundary the grant is meant to clarify. Multiple TWAP implementations (different polling cadences, different error tolerances, different cancel-on-loss thresholds) must be able to coexist as separate modules without changing the host or the SDK. +- **Move TWAP / EthFlow primitives into `cowprotocol` crate but skip the WIT interfaces**, leaving modules to call `composable::poll_and_build_order` from guest code. Rejected for the same reason: `cowprotocol` is the protocol SDK, not the strategy SDK. Putting TWAP logic there embeds an implementation in the shared layer, which is the smell the grant seeks to fix. +- **Ship a thin `shepherd-sdk` helper crate** that wraps the low-level primitive calls (eth_call, decode, submit) into a convenient `Twap::poll(...)` interface for guest modules. **Acceptable for M3** because the helper would live in guest-callable code, not behind a WIT boundary — a module that wants different polling policy just doesn't use the SDK helper. The host stays neutral. +- **EthFlow as pure passive observer (no submission)**. Rejected on closer read of `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`: the canonical CoW flow expects the event to be relayed into the orderbook, which is what autopilot currently does internally. Shepherd's `ethflow-watcher` externalises that role, so the module does submit; just from guest code, not via a specialised host interface. +- **TWAP merkle-proof / `setRoot` support in v1.** Deferred. The 0.2 module only handles `ComposableCoW.create()` (empty proof, single conditional order). `setRoot` polling requires off-chain proof derivation; when a real module needs it, it will be implemented in guest code using the same low-level primitives, possibly with an SDK helper to encapsulate the proof bookkeeping. ## Consequences -- `cow-api/submit-order` return type changes from `string` to `order-uid`. No external consumers today (0.2 is unreleased), so this is internal. -- Host helpers require a chain to be configured in `[chains.]` — uncovered chains return `host-error.unsupported`. Same posture as `cow-api`. -- Orderbook idempotency (same UID on duplicate submit) is preserved but invisible to the module. Modules that need dedup must record UIDs in `local-store` themselves. -- TWAP modules must implement the `poll-outcome` state machine: persist `next_attempt` hints (epoch or block number) in local-store, skip polls until trigger, remove watches on `dont-try-again`. Without this, the poll loop becomes O(blocks × twaps) with most calls wasted. The M3 SDK is expected to ship a helper that encapsulates the state machine. -- Orderbook errors return as `host-error` with the original CoW error code in `code`. Modules use `OrderPostError::try_from(host_error)` plus `retry_hint()` (ADR-0007 item 3) to map to next-block / backoff / drop. Without this layered approach, modules spam the orderbook with permanently-broken orders. -- Implementation order: the three `cowprotocol` primitives (`composable::poll_and_build_order` with rich `PollOutcome`, `eth_flow::decode_placement`, `OrderPostError` rich + `retry_hint`) land in `bleu/cow-rs` first; `nullis-shepherd` adopts via the existing `[patch.crates-io]` rev bump (ADR-0004). Host-side issues stay blocked on upstream merges. -- Failure modes map onto existing `host-error-kind` variants (`invalid-input`, `denied`, `rate-limited`, `timeout`, `unavailable`, `unsupported`, `internal`). No new error taxonomy. +- `shepherd:cow@0.2.0` keeps `cow-api` as its only interface. No new WIT files in this ADR. +- `KNOWN_CAPABILITIES` in `crates/nexum-engine/src/manifest.rs` does **not** gain `"twap"` or `"ethflow"` entries. Modules declare the universal capabilities they actually use: `chain`, `local-store`, `logging`, `cow-api`. +- Modules ship larger (~150 LOC each estimated, up from the ~30 LOC the host-helper design implied), because event decoding, eth_call orchestration, OrderCreation construction, and error-hint interpretation now live in guest code. This is the explicit trade-off: more code per module, less coupling, more freedom for different strategies to coexist. +- Different TWAP polling strategies can coexist as different modules. Operators choose which to load via `engine.toml`'s `[[modules]]` array. +- The watch-tower TypeScript implementation remains the closest reference for what a TWAP module's logic looks like, but it is reference material, not a template the Rust module mirrors verbatim. A newer ComposableCoW iteration in development may simplify the polling surface significantly; the relevant decisions live in the module, not the host. +- `OrderPostError` rich variants + `retry_hint()` (ADR-0007 item 1, formerly item 3) become the primary protocol-level contract between the orderbook and any module submitting orders. Modules `match` on the typed error and apply the `RetryHint` (try-next-block / backoff-seconds / drop). This logic is generic across TWAP, EthFlow, stop-loss, and any future strategy. +- The M3 SDK (`shepherd-sdk` crate) is the natural home for ergonomic guest-side helpers: `WatchSet`, `PollLoop`, `BackoffLedger`, decode-and-submit utilities. The SDK is opt-in for module authors and lives entirely on the guest side; the host remains protocol-neutral. +- The architecture and sequence diagrams in `docs/diagrams/` that depict `twap.poll-and-submit` and `ethflow.submit-from-log` host calls reflect the rejected design and must be updated to show modules calling low-level primitives directly. diff --git a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md index b6aeca6..e8e949c 100644 --- a/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md +++ b/docs/adr/0007-upstream-protocol-logic-to-cow-rs.md @@ -2,46 +2,45 @@ status: proposed --- -# Push CoW Protocol logic to `cow-rs` first, adopt in `nexum-engine` second +# Push CoW Protocol primitives to `cow-rs` first, adopt in `nexum-engine` second ## Context -Implementing ADR-0006 (twap + ethflow host helpers) and ADR-0005 (cow-api backend) surfaces a recurring question: when the engine needs a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (TWAP polling glue, EthFlow log decoding, rich orderbook error variants, custom orderbook URLs), do we write that logic locally in `nexum-engine` and tidy it up upstream later, or do we add it to the open upstream PR first and only land the engine wiring afterwards? +Implementing ADR-0005 (cow-api backend) and supporting guest-side TWAP / EthFlow modules per ADR-0006 surfaces a recurring question: when the engine or its modules need a piece of CoW Protocol logic that the `cowprotocol` Rust SDK does not yet expose (rich orderbook error variants, custom orderbook URLs, wasm32 compatibility), do we write that logic locally and tidy it up upstream later, or do we add it to the open upstream PR first and only land the engine wiring afterwards? -Review feedback on cow-rs PR #5 named the failure mode explicitly: duplicating work that an existing crate could do is the AI-coding anti-pattern most likely to land in a contribution. The same risk applies to any engine-side reimplementation of protocol logic. +The failure mode is well-known: duplicating work that an existing crate could do is the AI-coding anti-pattern most likely to land in a contribution. The same risk applies to any engine-side reimplementation of protocol logic. -CoW maintainers have signalled intent to keep extracting services from the `cowprotocol/services` monolith: `watch-tower` is already extracted, the `refunder` crate likewise, and the `ethflow_events` indexer (`crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`) is the next extraction target. The Rust SDK that Bleu is delivering through PR #5 is the natural home for the protocol primitives those extractions need. +The line between **protocol primitives** (which belong in `cowprotocol`) and **strategy implementations** (which belong in guest modules, per ADR-0006) is the operating principle. This ADR covers only the protocol-primitive additions; TWAP polling and EthFlow event decoding stay in guest modules and are explicitly **not** primitives we push to `cowprotocol`. ## Decision -Protocol-level CoW logic, meaning anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need, lands as additional commits on `cowdao-grants/cow-rs` PR #5 first (head branch `bleu/cow-rs:main`), and is consumed by `nexum-engine` via the `[patch.crates-io]` rev bump (ADR-0004). The engine never writes throwaway local copies of the same logic with the intent to "port later". +Protocol-level CoW logic — anything that an indexer, a bot, or a non-`nexum` Rust consumer of CoW Protocol would also need to interact with the protocol — lands as additional commits on `cowdao-grants/cow-rs` PR #5 first (head branch `bleu/cow-rs:main`), and is consumed by `nexum-engine` and by guest modules via the `[patch.crates-io]` rev bump (ADR-0004). The engine and the modules never write throwaway local copies of the same logic with the intent to "port later". -The concrete set of primitives we know we need is, in priority order: +The concrete set of primitives this ADR commits to upstream, in priority order: -1. **`cowprotocol::composable::poll_and_build_order(provider, owner, params, proof) -> Result`** — eth_call against `ComposableCoW.getTradeableOrderWithSignature`, decode return, rebuild `OrderCreation`. `PollOutcome` mirrors watchtower's `PollResultCode` (TS): `Submitted(OrderCreation, Vec)`, `TryAtEpoch(u64)`, `TryOnBlock(u64)`, `TryNextBlock`, `DontTryAgain`. Backs `twap.poll-and-submit` (ADR-0006). +1. **`cowprotocol::OrderPostError` rich variants + `retry_hint(&self) -> RetryHint`** — typed orderbook submission errors (`QuoteNotFound`, `InvalidQuote`, `InsufficientAllowance`, `InsufficientBalance`, `TooManyLimitOrders`, `InvalidAppData`, `AppDataFromMismatch`, `SellAmountOverflow`, `ZeroAmount`, `TransferSimulationFailed`, `ExcessiveValidTo`, …) with a `retry_hint()` helper classifying each into `TryNextBlock`, `BackoffSeconds(u64)`, or `Drop`. Mirrors watch-tower's `API_ERRORS_TRY_NEXT_BLOCK` / `API_ERRORS_BACKOFF` / `API_ERRORS_DROP` tables. Without this, every Rust consumer of CoW reinvents the same mapping, and modules spam the orderbook with permanently-broken orders. **Critical-path, not optional.** -2. **`cowprotocol::eth_flow::decode_placement(log) -> Result`** — decode `OrderPlacement` event log, reconstruct `OrderCreation` with the EIP-1271 signing scheme pointing at the `CoWSwapEthFlow` contract, compute `OrderUid`. Replicates the indexing logic currently inside `cowprotocol/services/crates/autopilot/src/database/onchain_order_events/ethflow_events.rs`. Backs `ethflow.submit-from-log` (ADR-0006). +2. **`cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)`** — custom-URL constructor for barn / staging / forked deployments. Unblocks per-chain orderbook URL overrides in `engine.toml` (ADR-0005). -3. **`cowprotocol::OrderPostError` rich variants + `retry_hint(&self) -> RetryHint`** — typed orderbook submission errors (`QuoteNotFound`, `InvalidQuote`, `InsufficientAllowance`, `InsufficientBalance`, `TooManyLimitOrders`, `InvalidAppData`, `AppDataFromMismatch`, `SellAmountOverflow`, `ZeroAmount`, `TransferSimulationFailed`, `ExcessiveValidTo`, …) with a `retry_hint()` helper classifying each into `TryNextBlock`, `BackoffSeconds(u64)`, or `Drop`. Mirrors watchtower's `API_ERRORS_TRY_NEXT_BLOCK` / `API_ERRORS_BACKOFF` / `API_ERRORS_DROP` tables. Without this, every Rust consumer of CoW reinvents the same mapping, and modules spam the orderbook with permanently-broken orders. **Critical-path, not optional.** +3. **`cowprotocol` `wasm32` compatibility** — feature-gate the `reqwest` dependency so guest modules can use the pure types (`Order`, `OrderCreation`, `OrderUid`, signing schemes, error variants) without dragging in an HTTP client. **Critical for ADR-0006**: modules implement TWAP and EthFlow logic in guest code and need `cowprotocol` types compiled to wasm32. Without this, guest modules fall back to duplicating type definitions. -4. **`cowprotocol::OrderBookApi::with_base_url(chain_id, base_url)`** — custom-URL constructor for barn / staging / forked deployments. Unblocks per-chain orderbook URL overrides in `engine.toml` (ADR-0005). - -5. **`cowprotocol` `wasm32` compatibility** — feature-gate the `reqwest` dependency so guest modules can use the pure types (`Order`, `OrderCreation`, `OrderUid`, `composable::*`, `eth_flow::decode_*`) without dragging in an HTTP client. Unblocks M3 SDK guest modules consuming `cowprotocol` directly. - -Lower-priority follow-ons (`OrderUid::from_slice`, retry middleware on `OrderBookApi`, `OrderCreation::from_gpv2`) are good-to-have but are not blocking for the M2 host scope. +Lower-priority follow-ons (`OrderUid::from_slice`, retry middleware on `OrderBookApi`, `OrderCreation::from_gpv2`) are good-to-have but are not blocking for the M2 host or module scope. ## Considered options - **Implement locally, refactor upstream later.** Faster short term but predictably leaves an indeterminate amount of duplicated logic in the engine, contradicts the conventions established on cow-rs PR #5, and grows technical debt every time cow-rs evolves the underlying types. Rejected. +- **Push TWAP / ComposableCoW primitives** (`composable::poll_and_build_order`) into `cowprotocol`. Rejected: TWAP is a concrete strategy on top of the protocol, not part of the protocol. Putting it in the SDK forces every consumer to use one polling implementation and one error-mapping policy. Per ADR-0006, TWAP polling lives in guest module code, not in shared layers. +- **Push EthFlow log-decoding primitives** (`eth_flow::decode_placement`) into `cowprotocol`. **Rejected for the same reason**: EthFlow event decoding is an implementation detail of how a particular module relays orders into the orderbook. The protocol layer defines the order types and the orderbook submission endpoint; the act of decoding an on-chain event into an `OrderCreation` is module-side logic. Modules decode `OrderPlacement` directly with `alloy_sol_types` and construct the `OrderCreation` with the EIP-1271 signing scheme. - **Wait for cow-rs upstream maintainers to add these on their own.** No evidence anyone else is doing this work; the grant timeline does not permit waiting. - **Vendor a fork of cow-rs inside `nullislabs/shepherd`.** Worst of all worlds: blocks neither the engine nor cow-rs from drifting, and forces every other CoW consumer to re-derive the same primitives. -- **Simple `Ready/NotReady` PollOutcome on item 1.** Rejected: doesn't capture watchtower's `TRY_AT_EPOCH(t)` hint, which is what prevents the polling loop from RPC-spamming during the 1-hour gap between TWAP parts. +- **Host-side `AppDataResolver` (LRU cache + GET against `/api/v1/app_data/{hash}`).** Rejected after verifying watch-tower's behavior: it never fetches app-data. The trader uploads the JSON to the orderbook via `PUT /api/v1/app_data/{hash}` separately; the relayer module just submits and reacts to `INVALID_APP_DATA` (backoff 1 min) / `APPDATA_FROM_MISMATCH` (drop) via the error map in item 1 above. ## Consequences -- Every M2 engine issue that consumes one of the five primitives above is blocked on the corresponding commit landing in PR #5's head branch. Items 1, 2, 3 can be authored as independent commits and pushed in parallel rather than serially. +- Every M2 engine or module issue that consumes one of the three primitives above is blocked on the corresponding commit landing in PR #5's head branch. Items 1, 2, 3 can be authored as independent commits and pushed in parallel rather than serially. - `[patch.crates-io]` rev in the workspace `Cargo.toml` (ADR-0004) is bumped after each push to PR #5; the bump is the engine's signal that a new primitive is consumable. -- Commits added to PR #5 follow the conventions established by its review thread: severity-tagged review notes, alloy reuse over local reimplementation, GPL-3.0, edition 2024, terse rustdoc. +- Commits added to PR #5 follow its established conventions: alloy reuse over local reimplementation, GPL-3.0, edition 2024, terse rustdoc. - The engine repo stays small: `nexum-engine` contains WIT, host wiring, supervisor, redb store, alloy provider pool, and `engine.toml` schema, with nothing about CoW Protocol semantics. -- The rich `PollOutcome` (item 1) plus `OrderPostError` and `retry_hint` (item 3) design naturally leads to tighter M3 SDK helpers: `WatchSet`, `PollLoop`, `BackoffLedger` patterns that any module re-using `shepherd-sdk` gets for free. -- A follow-on Bleu module, the Rust-side equivalent of `cowprotocol/refunder` (permissionless `invalidateOrder` triggering for expired EthFlow orders), becomes natural to ship once `ethflow.submit-from-log` lands. Out of scope for M2 but explicitly enabled by the same primitives. +- Guest modules consume `cowprotocol` types directly (gated on the wasm32 feature in item 3). The `shepherd-sdk` crate in M3 may add ergonomic wrappers on top, but those live on the guest side, not behind a WIT boundary. +- A follow-on Bleu module — the Rust-side equivalent of `cowprotocol/refunder` (permissionless `invalidateOrder` triggering for expired EthFlow orders) — becomes natural to ship once an ethflow-watcher module lands. Out of scope for M2 but explicitly enabled by the same primitives. +- TWAP polling logic (decode `ConditionalOrderCreated`, eth_call `getTradeableOrderWithSignature`, decode return, build `OrderCreation`) and EthFlow event decoding stay entirely in guest module code. The `cowprotocol` crate provides only the types and the orderbook client; the strategy is the module's. diff --git a/docs/adr/0008-factory-subscriptions-in-manifest.md b/docs/adr/0008-factory-subscriptions-in-manifest.md index fead3a9..5a23356 100644 --- a/docs/adr/0008-factory-subscriptions-in-manifest.md +++ b/docs/adr/0008-factory-subscriptions-in-manifest.md @@ -1,127 +1,56 @@ --- -status: proposed +status: deferred +deferred-to: 0.3 --- -# Dynamic address registration for log subscriptions +# Dynamic address registration for log subscriptions (deferred to 0.3) + +## Status + +**Deferred to 0.3.** Neither TWAP nor EthFlow (the M2 grant deliverables) needs this capability, and the design's complexity is not justified by current need. + +This ADR is preserved as a reference for the design space; the final shape will be revisited when the first module actually requiring dynamic address registration emerges. ## Context -Some module archetypes need to track contracts deployed dynamically by a factory, for example Uniswap V3 pools (deployed by `UniswapV3Factory`). Static `[[subscription]]` declarations in `nexum.toml` cannot express this: the child addresses are not known when the module's manifest is authored. - -Neither TWAP nor EthFlow (the M2 grant deliverables) needs this. Both subscribe to a single well-known contract per chain. This ADR is forward-looking, motivated by `docs/migration/0.1-to-0.2.md` §522 declaring 0.2 the breaking-change window; adding factory support after 0.2 would require another major version bump. - -Envio HyperIndex uses a hybrid pattern that fits Shepherd's design: topics are declared statically in the manifest, and the watched address set is mutated at runtime via a `register()` host call. The engine maintains a single aggregated log subscription per template; the address set grows as the module learns of new contracts. - -Whether the engine should also handle historical backfill on register (the module passes `from-block`, engine paginates `eth_getLogs` from there to head before going live) is a separate decision flagged for upstream review. This ADR keeps the engine surface minimal and defers historical replay to the existing module-driven catch-up pattern documented in `docs/02:260`. - -## Decision - -Two pieces: - -**1. Manifest schema gains `[[subscription.template]]`**, a topic-only log subscription whose address set is populated at runtime: - -```toml -[[subscription.template]] -chain_id = 1 -name = "uniswap_v3_pool" -event_topics = [ - "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67", # Swap - "0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde", # Mint -] -``` - -Existing `[[subscription]]` blocks with concrete `address` are unchanged. A module typically declares one static `[[subscription]]` for the factory event itself, plus one `[[subscription.template]]` per child contract type. - -**2. `nexum:host/chain` gains two host functions** for the module to manage the address set: - -```wit -interface chain { - // existing request, request-batch, subscribe-blocks, subscribe-logs ... - - /// Add an address to the watch set for `template-name` on `chain-id`. - /// Idempotent: calling twice with the same arguments is a no-op. - register-address: func( - chain-id: chain-id, - template-name: string, - address: list, // 20 bytes - ) -> result<_, host-error>; - - /// Remove an address from the watch set. Subsequent events on that - /// address are dropped. Idempotent. - unregister-address: func( - chain-id: chain-id, - template-name: string, - address: list, - ) -> result<_, host-error>; -} -``` - -**3. `log-source` variant in `nexum:host/event-module`** gains a `template` case so modules can route events: - -```wit -variant log-source { - static(u32), // existing: index into [[subscription]] - template(string), // new: name of the [[subscription.template]] -} -``` - -Module code (Rust, Uniswap V3 indexer): - -```rust -fn init(config: Vec<(String, String)>) -> Result<(), HostError> { - // Resume: re-register every pool we already discovered. - for key in local_store::list_keys("pool:")? { - let pool = parse_addr(&key); - chain::register_address(1, "uniswap_v3_pool", &pool)?; - } - Ok(()) -} - -fn on_event(event: Event) -> Result<(), HostError> { - match event { - // Factory event, declared as a static [[subscription]] for the factory. - Event::Log(LogEvent { log, source: LogSource::Static(0) }) => { - let pool = decode_pool_created(&log)?; - chain::register_address(1, "uniswap_v3_pool", &pool)?; - local_store::set(&format!("pool:{}", hex(&pool)), b"")?; - } - // Pool event, dispatched through the template. - Event::Log(LogEvent { log, source: LogSource::Template(name) }) - if name == "uniswap_v3_pool" => - { - process_pool_event(log)?; - } - _ => {} - } - Ok(()) -} -``` - -Engine internals: - -- **Boot**: read all `[[subscription.template]]` blocks. Initialise per-template address sets from the reserved key `__nexum:template:{name}:addresses` in the module's `local-store` namespace if present from a prior run. -- **Live**: one `eth_subscribe logs` per chain per template with filter `(topic in event_topics) AND (address in current_set)`. When `register-address` mutates the set, the engine re-subscribes. -- **Persistence**: engine writes `__nexum:template:{name}:addresses` whenever the set changes. Resume after restart is automatic if the module re-calls `register-address` in `init`. - -Historical backfill is the module's responsibility, consistent with the catch-up pattern documented in `docs/02:260`. The module calls `chain.request("eth_getLogs", ...)` during `init` to replay history before going live. The engine does not backfill on `register-address`. - -## Considered options - -- **Ponder full-declarative** (factory address, event, parameter declared in manifest; engine extracts child). Rejected: schema must express ABI-aware extraction (`child = { source = "topic", index = 1 }` or similar), and grows with every exotic factory shape (nested factories, multi-child events, address computed from multiple fields). The Envio model pushes that complexity into module Rust code, where it belongs given the module already decodes events with `alloy_sol_types`. -- **Pure imperative `chain.open-log-stream(filter) -> stream-handle`.** Rejected: each call opens a new subscription, so N pools means N WSS connections. Doesn't scale to indexers with 10k+ tracked addresses. The Envio model keeps one subscription per template and mutates its address set; engine batches naturally. -- **Engine-driven historical backfill on register** (with `from-block` parameter). Rejected after PR review flagged the added complexity. Module-driven catch-up via `init` + `eth_getLogs` already exists in mfw's design (`docs/02:260`) and covers the same use case without adding engine state (per-address cursor, paginated `eth_getLogs` orchestration). M3 SDK can ship a helper that wraps the pattern. -- **Wildcard manifest** (`address = "*"` with topic-only filter, module client-side filters). Rejected: mainnet has ~10k contracts emitting `Transfer` or `Swap` per day. The engine would deliver every matching event to every wildcard subscriber; modules pay fuel and bandwidth to discard 99% of them. -- **Defer factory pattern entirely to 0.3.** Rejected: 0.2 is the breaking-change window per migration:522. Adding either `[[subscription.template]]` or `register-address` after 0.2 requires another major bump. -- **Templates declared inside `[[subscription]]` with optional `address` (one block, two modes).** Rejected: conflates two semantically distinct cases. Modules looking at `[[subscription]]` would have to inspect for the presence of `address` to know whether they need to call `register-address`. Separate block name is clearer. - -## Consequences - -- `nexum.toml` schema gains `[[subscription.template]]` with `chain_id`, `name`, `event_topics`. Schema extension needs upstream approval. -- `nexum:host/chain` gains `register-address` and `unregister-address`; `nexum:host/event-module`'s `log-source` variant gains `template(string)`. WIT change needs upstream approval. -- Reserved key namespace `__nexum:template:*` in each module's `local-store` namespace. Modules MUST NOT write to keys with this prefix. -- Module boilerplate per factory is roughly 5 lines (decode the factory event, call `register-address`, persist for resume). The M3 SDK can ship a helper that wraps it. -- Register sources are not limited to factory events. A module can register addresses from any signal: HTTP API responses, governance votes, operator-supplied lists. Composability is a deliberate feature. -- Nested factories (a child contract that is itself a factory) work without schema changes. The child's event handler calls `register-address` on the grandchild template. -- Conditional registration ("only register pools with fee = 3000") works without schema changes. The module's factory-event handler decides. -- The address set per template is bounded only by `local-store` quota. The engine enforces a soft cap (default 50k addresses per template) configurable in `engine.toml`; exceeding the cap returns `host-error.denied` from `register-address`. -- Open follow-up: whether to support `from-block` historical backfill on register is left for upstream discussion. The minimal surface here can be extended additively if needed. +Some module archetypes need to track contracts deployed dynamically by a factory, for example Uniswap V3 pools (deployed by `UniswapV3Factory`). Static `[[subscription]]` declarations in `module.toml` cannot express this: the child addresses are not known when the module's manifest is authored. + +Neither TWAP nor EthFlow needs this; both subscribe to a single well-known contract per chain. This ADR was originally framed as forward-looking work to land in 0.2's breaking-change window. + +## Why deferred + +Two considerations motivate the deferral: + +1. **`eth_getLogs` already supports topic-only filtering.** The JSON-RPC method accepts a filter without an `address` field, so a module subscribing to a topic across all addresses can be served by the existing primitives if the operator's RPC endpoint cooperates. If topic-only filters at the JSON-RPC layer are good enough for the common case, the engine does not need a manifest-and-host-function mechanism on top. +2. **The schema and host-function surface add engine complexity that no M2 deliverable consumes.** The historical-backfill story is the largest contributor to that complexity and was already trimmed once; deferring the rest in the same spirit avoids paying for a mechanism nothing exercises yet. + +Combined: the dynamic-subscription design is not load-bearing for M2 deliverables, and the simplest path (topic-only `eth_subscribe` filters with module-side address filtering) may suffice for a wide range of indexer use cases. The dynamic-registration mechanism originally proposed (Envio-style `register-address`) addresses scaling concerns at high address counts but should land when a real consumer is on the table to validate the trade-off. + +## Reference design (not adopted in 0.2) + +The original proposal — kept here so future discussions have a starting point — was a hybrid of static topics and dynamic addresses: + +- `[[subscription.template]]` block in `module.toml` declaring `chain_id`, `name`, `event_topics` (no address). +- `chain.register-address(chain_id, template_name, address)` host function for the module to add addresses at runtime. +- `chain.unregister-address(chain_id, template_name, address)` mirror function. +- `log-source.template(string)` variant on the event dispatch so modules route by template name. +- Engine maintains a single aggregated `eth_subscribe logs` per chain per template, with filter `(topic ∈ event_topics) ∧ (address ∈ current_set)`. The address set is mutated as the module discovers new contracts. +- Historical backfill (`from-block` argument on register, paginated `eth_getLogs` orchestration) was contentious and was already trimmed before deferral. + +Envio HyperIndex's `context..register()` API is the closest existing pattern, validated in production for indexers tracking thousands of dynamically-discovered contracts. + +## Alternatives left open for 0.3 + +- **Topic-only `[[subscription]]`** (no address field; engine forwards `eth_subscribe logs` with topic-only filter; module client-side filters logs by address it cares about). Simplest, no new host functions. Trade-off: firehose volume for common topics like `Transfer`. +- **Dynamic register-address** (the original reference design above). +- **Engine-extracted factory child addresses** (Ponder-style declarative schema with ABI-aware extraction rules). Schema complexity grows with exotic factory shapes. +- **No factory pattern; modules wanting dynamic discovery use raw `chain.subscribe-logs` with topic-only filter and persist the discovered address set themselves**. + +The choice depends on what the first consumer actually needs. + +## Consequences of deferring + +- The `shepherd:cow` and `nexum:host` WIT surfaces remain unchanged in 0.2. +- `module.toml` schema does not gain `[[subscription.template]]` in 0.2. +- 0.2 is the breaking-change window; adding any of the above options in 0.3 may require a major version bump if the chosen shape extends `module.toml` or `nexum:host/chain` non-additively. This risk is accepted on the basis that the M2 grant deliverables do not require this surface. +- TWAP and EthFlow modules ship in 0.2 against the existing static `[[subscription]]` declarations (one address per subscription, known at manifest authorship time). This is consistent with how the autopilot ethflow indexer and watch-tower configure their subscriptions today. diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md new file mode 100644 index 0000000..c9c509e --- /dev/null +++ b/docs/diagrams/README.md @@ -0,0 +1,31 @@ +# Diagrams + +Mermaid sources and rendered PNGs covering the engine architecture, the CoW workflows that the M2 modules implement (TWAP and EthFlow, both as guest modules using low-level host primitives), and the engine internals that new contributors most often need to reason about. + +## Architecture and CoW flows + +| File | Type | Shows | +|---|---|---| +| `architecture.png` / `.mmd` | Component | Static view: external infra, nexum-engine internals, WASM modules (twap-monitor, ethflow-watcher) consuming low-level host primitives, and the `cowprotocol` crate (consumed via `[patch.crates-io]` and the wasm32 feature). The `shepherd:cow` package contains only `cow-api`; no specialised TWAP or EthFlow interfaces. | +| `sequence-ethflow.png` / `.mmd` | Sequence | `OrderPlacement` on-chain event handled entirely in the `ethflow-watcher` guest module: `alloy_sol_types` decodes the event, the module builds an `OrderCreation` with the EIP-1271 signing scheme using `cowprotocol` types, and submits via `cow-api/submit-order`. The orderbook error path runs through `OrderPostError::try_from(host-error).retry_hint()`. | +| `sequence-twap.png` / `.mmd` | Sequence | `ConditionalOrderCreated` registration plus the per-block polling loop driven by the `twap-monitor` guest module: `alloy_sol_types` decodes registrations and `eth_call` returns, the module makes the `getTradeableOrderWithSignature` call via `chain.request`, builds `OrderCreation` via `cowprotocol` types, and submits via `cow-api/submit-order`. Orderbook errors flow through `OrderPostError::retry_hint`. | + +## Engine internals (for contributors) + +| File | Type | Shows | +|---|---|---| +| `module-lifecycle.png` / `.mmd` | State machine | Resolve → Load → Init → Run → Restart → Dead transitions and what triggers each. Documents the exponential-backoff restart policy and the implicit write transaction around `init`. | +| `engine-boot.png` / `.mmd` | Sequence | Boot order: engine.toml → tracing → ProviderPool → LocalStore → OrderBookPool → Supervisor (load each module) → open subscriptions → run event loop. | +| `wit-call-path.png` / `.mmd` | Sequence | One host call traced end-to-end: module Rust source → wit-bindgen stubs → WASM Component → wasmtime Linker → HostState trait impl → ProviderPool → alloy → Chain RPC, and back. Demystifies the WASM/Rust boundary. | +| `subscription-dispatch.png` / `.mmd` | Flow chart | How the supervisor aggregates `[[subscription]]` declarations across modules, opens shared block subscriptions (broadcast) and per-filter log subscriptions (routed), and dispatches events to the right `on_event` handlers. | + +## Regenerate + +```sh +cd docs/diagrams +for f in *.mmd; do + npx -y @mermaid-js/mermaid-cli@latest -i "$f" -o "${f%.mmd}.png" -b white --width 1800 +done +``` + +Mermaid sources are the source of truth; PNGs are committed for offline viewing and PR previews. diff --git a/docs/diagrams/architecture.mmd b/docs/diagrams/architecture.mmd new file mode 100644 index 0000000..f5b97f2 --- /dev/null +++ b/docs/diagrams/architecture.mmd @@ -0,0 +1,69 @@ +graph TB + subgraph external["External Infrastructure"] + OB["CoW Orderbook
api.cow.fi"] + RPC["Chain RPC
ws:// or https://"] + CC["ComposableCoW
(Solidity contract)"] + EF["CoWSwapEthFlow
(Solidity contract)"] + end + + subgraph engine["nexum-engine binary"] + ETOML["engine.toml
(operator config)"] + SUP["Supervisor
(boot + dispatch)"] + EL["Event Loop
futures::select_all"] + + subgraph host["HostState (per module)"] + HW["Host Backends
cow-api, chain,
local-store, logging, ..."] + CP["OrderBookPool
(BTreeMap of OrderBookApi)"] + PP["ProviderPool
(alloy DynProvider per chain)"] + LS["LocalStore
(redb, 32-byte hash prefix per module)"] + end + end + + subgraph modules["WASM Modules (Component Model)"] + MTOML["module.toml
(module manifest)"] + TM["twap-monitor
(decodes events, polls,
builds OrderCreation,
submits via cow-api)"] + EM["ethflow-watcher
(decodes OrderPlacement,
builds OrderCreation,
submits via cow-api)"] + end + + subgraph cowrs["cowprotocol crate (via [patch.crates-io] to PR #5)"] + OBA["OrderBookApi
(orderbook submission)"] + TYPES["Protocol types
(Order, OrderCreation,
OrderUid, signing schemes)"] + OPE["OrderPostError
+ retry_hint -> RetryHint"] + end + + ETOML --> SUP + MTOML -.declares.-> SUP + SUP --> EL + SUP -.loads via wasmtime.-> TM + SUP -.loads via wasmtime.-> EM + + EL -- "on_event(block | log)" --> TM + EL -- "on_event(block | log)" --> EM + + TM -- "WIT host call" --> HW + EM -- "WIT host call" --> HW + + TM -- "consumes types
(wasm32 feature)" --> TYPES + EM -- "consumes types
(wasm32 feature)" --> TYPES + TM -- "matches on" --> OPE + EM -- "matches on" --> OPE + + HW --> CP + HW --> PP + HW --> LS + + CP --> OBA + PP --> RPC + OBA --> OB + CC -. "ConditionalOrderCreated" .-> RPC + EF -. "OrderPlacement / OrderInvalidation" .-> RPC + + classDef external fill:#f0e6ff,stroke:#7e3ff2,color:#000 + classDef engineNode fill:#e6f3ff,stroke:#1e88e5,color:#000 + classDef moduleNode fill:#fff4e6,stroke:#ff9800,color:#000 + classDef cowrsNode fill:#e6ffe6,stroke:#2e7d32,color:#000 + + class OB,RPC,CC,EF external + class ETOML,SUP,EL,HW,CP,PP,LS engineNode + class MTOML,TM,EM moduleNode + class OBA,TYPES,OPE cowrsNode diff --git a/docs/diagrams/diagrams.md b/docs/diagrams/diagrams.md new file mode 100644 index 0000000..762e6db --- /dev/null +++ b/docs/diagrams/diagrams.md @@ -0,0 +1,472 @@ +# Shepherd — Architecture Diagrams + +Visual reference for the Shepherd engine, its interactions with Nexum, CoW Protocol, and the WASM module layer. Derived from ADRs 0001–0008 and the internal architecture document. + +> **Scope note** — diagrams 1–4 and 7–8 reflect the **M1 implemented state** plus the **M2 target design** as described by the ADRs. Diagrams 5–6 (TWAP, EthFlow) describe **guest-module-driven flows**: the modules do all the protocol work themselves using low-level host primitives, with no specialised `twap` or `ethflow` host interfaces. Where the current code differs from the target design, a note is included in the relevant block reference. + +--- + +## 1. System Architecture Overview + +High-level component map: what lives where, how repositories depend on each other. + +```mermaid +graph TD + ET["engine.toml · module.toml\n(operator config + module manifest)"] + SUP["Supervisor::boot"] + POOLS["ProviderPool · OrderBookPool · LocalStore"] + HS["HostState (per module)\nnexum:host@0.2.0 + shepherd:cow@0.2.0"] + EL["EventLoop — futures::stream::select_all\nfan-out block/log streams to subscribers"] + MODS["WASM Modules\ntwap.wasm · eth-flow.wasm\n(self-contained protocol logic in guest)"] + BC["Blockchain (Sepolia / Mainnet / …)\nComposableCoW · CowEthFlow · RPC Node"] + CR["bleu/cow-rs ← [patch.crates-io]\nOrder · OrderCreation · OrderUid · signing schemes\nOrderBookApi · OrderPostError + retry_hint\nOrderBookApi::with_base_url · wasm32 feature"] + OB["api.cow.fi (Orderbook REST)"] + + ET --> SUP + SUP --> POOLS + POOLS --> HS + HS --> EL + BC -->|"block/log events (eth_subscribe)"| EL + EL -->|"on_event(block/log)"| MODS + MODS -->|"WIT calls (chain · local-store · cow-api · …)"| HS + MODS -.->|"consumes types (wasm32 feature)"| CR + HS -->|"eth_call / subscribe"| BC + HS -->|"OrderBookApi.post_order"| OB + HS -->|"cow-api passthrough"| OB +``` + +### Block reference + +| Block | What it is | +|---|---| +| **engine.toml** | Written by the operator. Declares which chains to connect to (RPC URLs), where to store state on disk, and which WASM modules to load at boot. | +| **module.toml** | Written by the module developer and shipped inside the module bundle. Declares which capabilities the module needs (`required`), which on-chain events to subscribe to, and any module-specific config keys. Renamed from `nexum.toml` per ADR-0001 so the operator/module split is directly apparent. | +| **Supervisor::boot** | The boot orchestrator. Reads both config files, creates the shared resource pools, loads each `.wasm` component via wasmtime, and wires their subscriptions into the event streams. | +| **ProviderPool · OrderBookPool · LocalStore** | The three shared backends. `ProviderPool` holds one alloy RPC client per chain. `OrderBookPool` holds one CoW orderbook HTTP client per chain. `LocalStore` is a single redb key-value database shared by all modules (with per-module 32-byte hash namespacing — ADR-0003). | +| **HostState (per module)** | The per-module bridge between WASM guest code and Rust host code. When a module calls a WIT function (`local-store/set`, `cow-api/submit-order`, etc.), wasmtime routes that call to the corresponding method on that module's `HostState`. Checks capability permissions before dispatching. | +| **EventLoop** | The main async loop. Runs all block-header and log-event streams concurrently via `futures::stream::select_all`. When a stream fires, it routes the event to every module that subscribed to it in their `module.toml`. | +| **WASM Modules** | The guest programs. Each module exports `init(config)` (called once at boot) and `on_event(event)` (called on every relevant block or log). They contain the protocol logic themselves: TWAP polling, EthFlow event decoding, OrderCreation construction. They call back into the host through universal WIT interfaces only — no CoW-specific helper interfaces (ADR-0006). | +| **Blockchain** | The EVM chain being watched. Delivers new block headers and contract log events over a persistent WebSocket (`eth_subscribe`). Also handles `eth_call` for on-chain reads (e.g. checking whether a TWAP order is ready). | +| **bleu/cow-rs [patch.crates-io]** | The Rust crate containing CoW Protocol **primitives**: order types, signing schemes, the orderbook HTTP client, and the typed orderbook error model (`OrderPostError` + `retry_hint`). Pulled via `[patch.crates-io]` pointing at the head of upstream PR #5. Modules consume the types directly via the `wasm32` feature; the engine consumes the orderbook client via its `cow-api` host backend. No TWAP or EthFlow strategy logic lives here — that stays in module code (ADR-0007). | +| **api.cow.fi (Orderbook REST)** | The CoW Protocol orderbook service. Accepts `POST /orders` to register new orders. Trader-uploaded app-data documents are PUT to `/app_data/{hash}` separately by whoever signed the order (not by the relayer module). | + +--- + +## 2. Domain / Class Diagram + +Key types, their fields, and relationships across the engine codebase. + +```mermaid +classDiagram + class EngineConfig { + +state_dir: PathBuf + +chains: BTreeMap~u64, ChainConfig~ + +modules: Vec~ModuleEntry~ + } + class Manifest { + +name: String + +capabilities_required: Vec~String~ + +subscriptions: Vec~Subscription~ + } + class Subscription { + +kind: Block | Log + +chain_id: u64 + +address: Option~Address~ + +topics: Vec~B256~ + } + class Supervisor { + +dispatch_block(chain_id, block) + +dispatch_log(owner, log) + } + class ProviderPool { + +providers: BTreeMap~u64, DynProvider~ + } + class OrderBookPool { + +clients: BTreeMap~u64, OrderBookApi~ + } + class LocalStore { + +db: redb~Database~ + +get(key: String) Option~Vec~u8~~ + +set(key: String, value: Vec~u8~) + +delete(key: String) + +list_keys(prefix: String) Vec~String~ + } + class HostState { + +wasi: WasiCtx + +table: ResourceTable + +http_allowlist: Vec~String~ + +monotonic_baseline: Instant + %% M2 additions (ADR-0005): + %% +module_namespace: [u8; 32] (ENS namehash or keccak256) + %% +provider_pool: Arc~ProviderPool~ + %% +ob_pool: Arc~OrderBookPool~ + %% +local_store: Arc~LocalStore~ + } + class EventLoop { + +streams: SelectAll~Pin~Box~dyn Stream~~~ + } + class TwapModule { + <> + +on_event(log) persist watch via local-store + +on_event(block) poll each watch via chain.request(eth_call) + +decode return via alloy_sol_types + +build OrderCreation via cowprotocol types + +submit via cow-api.submit-order + +interpret errors via OrderPostError.retry_hint + } + class EthFlowModule { + <> + +on_event(log) decode OrderPlacement via alloy_sol_types + +build OrderCreation (EIP-1271 sig) via cowprotocol types + +submit via cow-api.submit-order + +interpret errors via OrderPostError.retry_hint + } + + EngineConfig --> Supervisor + Manifest --> Supervisor + Manifest "1" --> "*" Subscription + Supervisor --> ProviderPool + Supervisor --> OrderBookPool + Supervisor --> LocalStore + Supervisor "1" --> "*" HostState : per module + HostState --> ProviderPool + HostState --> OrderBookPool + HostState --> LocalStore + EventLoop --> Supervisor + TwapModule ..> HostState : WIT calls (universal) + EthFlowModule ..> HostState : WIT calls (universal) +``` + +### Class reference + +| Class | What it is | +|---|---| +| **EngineConfig** | Deserialized from `engine.toml`. Holds the database path (`state_dir`), one `ChainConfig` per chain (just an RPC URL), and the list of module paths to load. | +| **Manifest** | Deserialized from `module.toml`, which ships inside the module bundle. Declares what capabilities the module needs, which on-chain events to watch, and any module-level config values. | +| **Subscription** | One event declaration inside `module.toml`. `kind=Block` fires on every new block for a given chain. `kind=Log` fires when a specific contract emits an event matching the given address and topics. Factory-style dynamic subscriptions (`[[subscription.template]]` + `register-address`) are deferred to 0.3 — see ADR-0008. | +| **Supervisor** | Orchestrates boot and event dispatch. Creates one `HostState` per module. On each incoming block or log, calls `dispatch_block` / `dispatch_log` to fan the event out to subscribed modules. | +| **ProviderPool** | Holds one alloy `DynProvider` per chain. `wss://` chains get a pubsub provider that supports both subscriptions and requests. `https://` chains get HTTP-only (subscriptions unavailable, by design — ADR-0002). | +| **OrderBookPool** | Holds one `OrderBookApi` client per known CoW chain (Mainnet, Gnosis, Sepolia, ArbitrumOne, Base). Instantiated via `OrderBookPool::default()` at boot (ADR-0005). | +| **LocalStore** | A single redb embedded database at `state_dir`. All modules write into the same file. Keys are prefixed host-side as `[32-byte module namespace][raw_key]` so two modules never collide, and the namespace is unspoofable (ADR-0003). The namespace is `keccak256(module_name)` for locally-loaded modules and `ens_namehash(name)` for ENS-discovered modules. | +| **HostState** | The per-module runtime context. `wasmtime::component::bindgen!` generates one trait per WIT interface (e.g. `shepherd::cow::cow_api::Host`); `HostState` implements each trait. `Shepherd::add_to_linker` registers all trait implementations with the `Linker` once at boot. **Current fields** (M1): `wasi: WasiCtx`, `table: ResourceTable`, `http_allowlist: Vec`, `monotonic_baseline: Instant`. **M2 additions** will add `module_namespace: [u8; 32]`, `provider_pool: Arc`, `ob_pool: Arc`, `local_store: Arc`. | +| **EventLoop** | Runs `futures::stream::select_all` over a `Vec + Send>>>`. The loop never exits until SIGINT/SIGTERM. Each fired event is forwarded to `Supervisor` for fan-out. | +| **TwapModule** | The TWAP watcher WASM component. On a `Log` event (ConditionalOrderCreated): persists the registration in `local-store`. On a `Block` event: iterates all watches and, for each, makes an `eth_call` via `chain.request`, decodes the result via `alloy_sol_types` (in-module), builds an `OrderCreation` via `cowprotocol` types (consumed via wasm32 feature), and submits via `cow-api.submit-order`. Orderbook errors flow through `OrderPostError::retry_hint`. All polling logic lives in the module, not the host (ADR-0006). | +| **EthFlowModule** | The EthFlow watcher WASM component. On a `Log` event (OrderPlacement): decodes the event via `alloy_sol_types` in-module, builds the `OrderCreation` with the EIP-1271 signing scheme via `cowprotocol` types, and submits via `cow-api.submit-order`. No polling loop — one log equals one submission attempt. | + +--- + +## 3. WIT Interface Hierarchy + +Two WIT packages: the universal `nexum:host` and the CoW-specific `shepherd:cow`. + +```mermaid +graph TD + NH["nexum:host@0.2.0\n(universal — no CoW knowledge)"] + SC["shepherd:cow@0.2.0\n(CoW Protocol extensions)"] + + NH --> n1["chain ✅ implemented\nrequest(chain-id, method, params)\nrequest-batch(chain-id, requests)\n—\nsubscribe-blocks · subscribe-logs →\n engine-managed via module.toml subscriptions\nregister-address · unregister-address →\n 🕓 deferred to 0.3 (ADR-0008)"] + NH --> n2["local-store ✅ implemented\nget(key) · set(key, value)\ndelete(key) · list-keys(prefix)\nnamespacing: 32-byte hash prefix (ADR-0003)"] + NH --> n3["identity · messaging · http · remote-store\n✅ stubs (Unsupported) — full impl in 0.3"] + NH --> n4["logging · clock · random ✅ implemented"] + + SC -->|"use nexum:host/types"| NH + SC --> s1["cow-api ✅ implemented\nrequest(chain-id, method, path, body)\nsubmit-order(chain-id, order-data)\n→ result\n(only protocol-level interface in shepherd:cow)"] + + note1["No twap / ethflow host interfaces.\nTWAP and EthFlow logic lives in guest\nmodule code, using universal primitives\n(chain · local-store · cow-api).\nSee ADR-0006."] + SC -.-> note1 + + style note1 fill:#fff4e6,stroke:#ff9800,color:#000 +``` + +### Interface reference + +| Interface | What it does | +|---|---| +| **nexum:host@0.2.0** | The base WIT package. Any module running in the engine — CoW-aware or not — imports from here. Defines shared types (`chain-id`, `log`, `host-error`) used by both packages. | +| **chain** | Reads from the blockchain via JSON-RPC. `request` sends a single call; `request-batch` sends several in one round-trip. **Subscriptions are not callable WIT functions** — they are declared in `module.toml` and opened by the engine at boot. Dynamic `register-address` for factory patterns is deferred to 0.3 (ADR-0008). | +| **local-store** | Persistent key-value storage that survives restarts. Operations: `get(key)`, `set(key, value)`, `delete(key)`, `list-keys(prefix)`. The host prefixes every key with a 32-byte deterministic namespace (`keccak256(module_name)` locally, or `ens_namehash(name)` when ENS-loaded) so modules are fully isolated and the namespace cannot be spoofed (ADR-0003). | +| **identity · messaging · http · remote-store** | Capabilities stubbed at 0.2 — they return `Unsupported`. `identity` will provide keystore-backed signing. `messaging` will send Waku messages. `http` will allow direct outbound HTTP calls (subject to the manifest's allowlist). `remote-store` will read/write Swarm/IPFS. | +| **logging · clock · random** | Lightweight utilities. `logging` emits to the engine's `tracing` subscriber (inherits `RUST_LOG` filters). `clock` returns wall-clock time. `random` returns cryptographically-secure random bytes. | +| **shepherd:cow@0.2.0** | The CoW Protocol extension package. Imports `nexum:host/types` for shared types so modules don't re-define `chain-id` or `log`. Only CoW-aware modules need to import this package. Contains exactly **one** interface in 0.2: `cow-api`. | +| **cow-api** | Generic orderbook access. `request` is a raw REST passthrough (returns JSON string). `submit-order` takes raw order bytes and returns a `result` where the string is the order UID. Routes through the engine's `OrderBookPool`. This is the only protocol-level CoW interface in 0.2 — the boundary between "what CoW Protocol *is*" (orderbook submission, order types) and "what's implemented *on top* of CoW" (TWAP polling, EthFlow event handling). | +| **(no twap interface)** | Per ADR-0006, no specialised TWAP host interface exists. The TWAP module implements polling, decoding, and submission entirely in guest code, using `chain.request` for `eth_call`, `local-store` for state, `alloy_sol_types` (in-module) for ABI decoding, `cowprotocol` types for `OrderCreation`, and `cow-api.submit-order` for orderbook submission. Multiple TWAP strategies can coexist as separate modules with different polling policies and error tolerances. | +| **(no ethflow interface)** | Per ADR-0006, no specialised EthFlow host interface exists. The EthFlow module decodes `OrderPlacement` directly in guest code via `alloy_sol_types`, constructs the `OrderCreation` with the EIP-1271 signing scheme via `cowprotocol` types, and submits via `cow-api`. | + +--- + +## 4. Engine Boot Sequence + +```mermaid +flowchart TD + Start([nexum-engine starts]) --> ReadConfig + ReadConfig["1. Read engine.toml\n(EngineConfig::load)"] + ReadConfig --> InitTracing + InitTracing["2. Init tracing\n(RUST_LOG / log_level)"] + InitTracing --> ProvPool + ProvPool["3. ProviderPool::from_config\nFor each chain:\n wss:// → pubsub DynProvider\n https:// → http DynProvider\n(fatal on connection error — ADR-0002)"] + ProvPool --> OpenStore + OpenStore["4. LocalStore::open(state_dir)\nOpen/create redb DB\nnexum:local-store table\n(ADR-0003)"] + OpenStore --> OBPoolInit + OBPoolInit["5. OrderBookPool::default()\nBuild OrderBookApi for:\nMainnet, Gnosis, Sepolia, ArbitrumOne, Base\n(ADR-0005)"] + OBPoolInit --> SupervisorBoot + SupervisorBoot["6. Supervisor::boot\nFor each [[modules]] in engine.toml:"] + SupervisorBoot --> LoadManifest[" a. Load module.toml (Manifest)"] + LoadManifest --> LoadWasm[" b. Load .wasm Component (wasmtime)"] + LoadWasm --> Instantiate[" c. Instantiate with dedicated HostState\n (links nexum:host + shepherd:cow impls)"] + Instantiate --> CallInit[" d. Call module.init(config)"] + CallInit --> AnnotateSubs[" e. Annotate subscriptions from manifest"] + AnnotateSubs --> MoreModules{More modules?} + MoreModules -->|yes| LoadManifest + MoreModules -->|no| OpenStreams + OpenStreams["7. open_block_streams + open_log_streams\neth_subscribe newHeads per chain\neth_subscribe logs per (chain, address, topics)"] + OpenStreams --> RunLoop + RunLoop["8. run_event_loop\nfutures::stream::select_all over all streams\nfan-out: block → all block subscribers\nlog → owner module only"] + RunLoop --> Wait([Await SIGINT/SIGTERM]) +``` + +### Step reference + +| Step | What happens | +|---|---| +| **1. Read engine.toml** | Deserializes the operator config. If the file is missing, the engine falls back to defaults (no chains, default `state_dir`). Modules that need chains will receive `Unsupported` errors at runtime. | +| **2. Init tracing** | Sets up the `tracing` subscriber using `RUST_LOG` or the `log_level` field from `engine.toml`. All host log output flows through here, including per-capability trace events. | +| **3. ProviderPool** | Opens one alloy connection per chain declared in `[chains]`. WebSocket URLs get a full pubsub provider (the recommended setup for any chain a module subscribes to). HTTP URLs get a request-only provider. Any connection failure at this step is fatal — the engine refuses to start with a broken chain rather than silently degrading. Failover and retry are out of scope; they live in alloy middleware (ADR-0002). | +| **4. LocalStore** | Opens (or creates) the redb database at `state_dir`. Creates the `nexum:local-store` table if it doesn't exist. Per-module namespacing uses a 32-byte deterministic hash prefix. Module state from previous runs is immediately available. | +| **5. OrderBookPool** | Constructs one `OrderBookApi` HTTP client for each supported CoW chain via the `Default` implementation. Built upfront so config errors (unknown chain IDs) surface at boot, not on the first order submission. | +| **6. Supervisor::boot (per module)** | For each module listed in `engine.toml`: reads its `module.toml`, loads the `.wasm` component into wasmtime, creates a dedicated `HostState`, calls the module's `init(config)` export, and records which subscriptions the module declared. | +| **7. Open streams** | Aggregates all subscriptions declared across all modules. Opens one `eth_subscribe newHeads` per chain and one `eth_subscribe logs` per (chain, contract-address, topics) filter. | +| **8. Event loop** | The engine enters its steady-state loop. `futures::stream::select_all` waits for the next event on any stream. Block events are broadcast to all modules subscribed to that chain. Log events are delivered only to the module that owns that subscription. | + +--- + +## 5. TWAP Complete Flow (Registration → Submit) + +The TWAP module runs the entire flow in guest Rust code, using only universal host primitives. + +```mermaid +sequenceDiagram + actor User + participant CC as ComposableCoW
Contract + participant RPC as RPC Node
(wss://) + participant EL as EventLoop + participant TM as twap module
(WASM guest) + participant SD as alloy_sol_types
(in module) + participant CR as cowprotocol types
(in module, wasm32) + participant HS as HostState
(Rust) + participant OB as api.cow.fi
(Orderbook) + participant LS as LocalStore
(redb) + + Note over User,CC: Step 0 — On-chain registration (off-engine) + User->>CC: ComposableCoW.create(twapParams) + CC-->>RPC: emit ConditionalOrderCreated(owner, params, proof) + + Note over RPC,LS: Step 1 — Indexing (once per TWAP) + RPC->>EL: log batch (eth_subscribe logs) + EL->>TM: on_event(Event::Logs([registration_log])) + TM->>SD: decode ConditionalOrderCreated + SD-->>TM: (owner, params, salt) + TM->>HS: local-store.set("watch:{owner}:{hash}", params) + HS->>LS: write [module_namespace][watch:...] = params_bytes + + Note over RPC,LS: Step 2 — Poll loop (every block) + loop Every block on chain_id + RPC->>EL: block header (eth_subscribe newHeads) + EL->>TM: on_event(Event::Block(Block { number: N, ... })) + TM->>HS: local-store.list-keys("watch:") + HS-->>TM: registration entries + + loop For each watch where next_attempt <= N + TM->>HS: chain.request(chain_id, "eth_call", [...]) + HS->>RPC: eth_call ComposableCoW.getTradeableOrderWithSignature(owner, params, "", []) + RPC-->>HS: return value or revert + HS-->>TM: result (JSON string) + TM->>SD: decode return value (or interpret revert reason) + SD-->>TM: ready(GPv2OrderData, signature) OR not-ready(hint) + + alt Order ready + TM->>CR: build OrderCreation + CR-->>TM: OrderCreation + TM->>HS: cow-api.submit-order(chain_id, order_json) + HS->>OB: POST /api/v1/orders + alt Submit OK + OB-->>HS: 200 OK, OrderUid + HS-->>TM: Ok(OrderUid) + TM->>LS: set("submitted:{uid}", order_uid) + else Orderbook error + OB-->>HS: 4xx with error code + HS-->>TM: Err(host-error) + TM->>CR: OrderPostError::try_from(host-error).retry_hint() + CR-->>TM: TryNextBlock / BackoffSeconds(s) / Drop + TM->>LS: update next_attempt or remove watch + end + else Not yet ready (TryAtEpoch / TryOnBlock / TryNextBlock / Terminal) + TM->>LS: persist hint (next_attempt) or delete watch + end + end + end +``` + +### Participant reference + +| Participant | Role in this flow | +|---|---| +| **User** | The trader. Interacts with the blockchain directly — the engine never touches private keys. | +| **ComposableCoW Contract** | The on-chain conditional order registry. Accepts TWAP parameters via `create()` and emits `ConditionalOrderCreated`. Also exposes `getTradeableOrderWithSignature()`, which the engine polls to check whether the current TWAP part is ready to trade. | +| **RPC Node** | The WebSocket connection to the chain. Delivers log events (subscriptions) and handles `eth_call` (synchronous reads). Must be `wss://` for this flow since it uses subscriptions. | +| **EventLoop** | Receives raw events from the RPC node and routes them to the module that subscribed to them. Opaque to the flow — it just calls `on_event`. | +| **twap module (WASM guest)** | Contains the entire TWAP strategy: decoding registrations, deciding when to poll (using stored hints), reacting to revert reasons, building orders, interpreting orderbook errors. Calls into the host only through universal WIT primitives. | +| **alloy_sol_types (in module)** | The ABI-aware decoder. Compiled into the module's WASM. Decodes `ConditionalOrderCreated` from raw log bytes; decodes the `getTradeableOrderWithSignature` return; interprets revert reasons. No host involvement for decoding. | +| **cowprotocol types (in module)** | The protocol-level types from `bleu/cow-rs`, consumed by the module via the wasm32 feature (ADR-0007 item 3). Used to build `OrderCreation`, manipulate `OrderUid`, and pattern-match `OrderPostError`. The crate's HTTP client (`OrderBookApi`) is **not** used directly by the module — orderbook submission goes through the host's `cow-api`. | +| **HostState (Rust)** | Provides only the universal primitives (`chain.request`, `local-store.*`, `cow-api.submit-order`). Knows nothing about TWAP semantics. | +| **api.cow.fi (Orderbook)** | Receives the signed `OrderCreation`, validates it, and returns a 56-byte `OrderUid`. The order is now visible to CoW solvers. | +| **LocalStore (redb)** | Persistent state for the TWAP module. `watch:{owner}:{hash}` entries hold registrations. `submitted:{uid}` entries record completed submissions. `next_attempt` hints (epoch or block) let the module skip polling during the gap between TWAP parts. All entries survive engine restarts. | + +--- + +## 6. EthFlow Complete Flow (Event-Driven) + +```mermaid +sequenceDiagram + actor User + participant EFC as CoWSwapEthFlow
Contract + participant RPC as RPC Node
(wss://) + participant EL as EventLoop + participant EM as eth-flow module
(WASM guest) + participant SD as alloy_sol_types
(in module) + participant CR as cowprotocol types
(in module, wasm32) + participant HS as HostState
(Rust) + participant OB as api.cow.fi
(Orderbook) + participant LS as LocalStore
(redb) + + Note over User,EFC: Step 0 — User creates ETH order on-chain + User->>EFC: createOrder(order, msg.value=ETH) + EFC->>EFC: store orders[hash] = onchainData,
emit OrderPlacement(sender, order, EIP1271-sig, data) + EFC-->>RPC: log emitted on block N + + Note over RPC,LS: Step 1 — Log arrives via subscription + RPC->>EL: log batch matching CoWSwapEthFlow address + OrderPlacement topic + EL->>EM: on_event(Event::Logs([placement_log])) + + Note over EM,LS: Step 2 — Decode and submit (1 log = 1 submission) + EM->>SD: decode OrderPlacement(sender, order, sig, data) + SD-->>EM: (sender, GPv2OrderData, EIP-1271 sig, data) + + EM->>CR: build OrderCreation with EIP-1271 scheme
pointing at CoWSwapEthFlow contract + CR-->>EM: OrderCreation + OrderUid + + EM->>HS: cow-api.submit-order(chain_id, order_json) + HS->>OB: POST /api/v1/orders + OB-->>HS: result + + alt 200 OK with OrderUid + HS-->>EM: Ok(OrderUid) + EM->>LS: set("submitted:{uid}", order_uid) + else 4xx with error code + HS-->>EM: Err(host-error with code) + EM->>CR: OrderPostError::try_from(host-error).retry_hint() + CR-->>EM: TryNextBlock / BackoffSeconds(s) / Drop + + alt TryNextBlock + Note over EM: log and skip; next block retries + else BackoffSeconds(s) + EM->>LS: set("backoff:{uid}", now + s) + else Drop + EM->>LS: set("dropped:{uid}", reason) + end + end +``` + +### Participant reference + +| Participant | Role in this flow | +|---|---| +| **User** | The trader. Deposits native ETH into the `CoWSwapEthFlow` contract and specifies swap parameters. The contract is the EIP-1271 signer on behalf of the user. | +| **CoWSwapEthFlow Contract** | Custodies the ETH, stores the order metadata on-chain, and emits `OrderPlacement` so off-chain relayers (this module, plus CoW's own internal autopilot indexer) can pick up the order. | +| **RPC Node** | Delivers the `OrderPlacement` log via the persistent WebSocket subscription. No `eth_call` is needed in this flow — the log contains everything required to reconstruct the order. | +| **EventLoop** | Routes the log to the eth-flow module based on the `[[subscription]]` entry in its `module.toml` (matching the `CoWSwapEthFlow` contract address and the `OrderPlacement` topic). | +| **eth-flow module (WASM guest)** | Contains the entire EthFlow relay logic: decoding, OrderCreation construction, submission, error handling. No polling loop; one log equals one submission attempt. | +| **alloy_sol_types (in module)** | Decodes the `OrderPlacement` event in module-side Rust. The event payload carries the typed `GPv2OrderData`, the EIP-1271 signature blob, and the extra data field. | +| **cowprotocol types (in module)** | Used to construct the `OrderCreation` with the EIP-1271 signing scheme (the signature points at the `CoWSwapEthFlow` contract address, not at the user's key) and to compute the 56-byte `OrderUid`. `OrderPostError` from the same crate is used to interpret orderbook errors. | +| **HostState (Rust)** | Provides only the `cow-api.submit-order` primitive for this flow. Maps orderbook errors to `host-error` with the original error code preserved so the module can recover the typed `OrderPostError`. | +| **api.cow.fi (Orderbook)** | Receives the order. Returns `OrderUid` on success. Returns a typed error code on failure, which the module recovers and passes through `OrderPostError::retry_hint()` to decide what to do next. App-data documents are **not** fetched here; the trader uploads them via `PUT /api/v1/app_data/{hash}` separately. | +| **LocalStore (redb)** | `submitted:{uid}` records successful submissions. `backoff:{uid}` records pending retries with a deadline. `dropped:{uid}` records permanently-failed orders. All entries survive restarts so the module does not re-submit known orders. | + +--- + +## 7. Capability Dispatch (Generic Host Call Path) + +How any WIT function call from a WASM module reaches the host backend it targets. + +```mermaid +flowchart TD + Module["WASM Module\n(twap or eth-flow)"] + Module -->|"e.g. cow-api.submit-order(chain_id, order_json)"| Linker + Linker["wasmtime Linker\n(resolves import → host function)"] + Linker --> CapCheck["HostState\nmanifest.required check\n(returns denied if not declared)"] + CapCheck --> Trace["tracing::info!\n[capability] op chain=…"] + Trace --> HostFn["host backend Rust function"] + HostFn -->|"cow-api"| OBPool["OrderBookPool\n.get(chain_id)?.post_order(order)"] + HostFn -->|"chain (request)"| ProviderP["ProviderPool\n.get(chain_id).request(method, params)"] + HostFn -->|"chain (subscribe-*)"| EngineSubs["engine-managed streams\nopened at boot from module.toml"] + HostFn -->|"local-store"| StoreP["LocalStore\n.get/set/delete/list_keys\n(namespace-prefixed host-side)"] + HostFn -->|"logging · clock · random"| Misc["direct host impl"] + OBPool --> CowAPI["api.cow.fi\nPOST /api/v1/orders"] + ProviderP --> RPCNode["RPC Node\nJSON-RPC"] + CowAPI -->|"OrderUid or error"| Module + RPCNode -->|"result"| Module +``` + +### Node reference + +| Node | What it does | +|---|---| +| **WASM Module** | The guest program. It calls imported WIT functions exactly like regular function calls — it has no visibility into the host machinery behind them. | +| **wasmtime Linker** | `Linker` built once at startup. `wasmtime::component::bindgen!` generates a `Shepherd` world struct and one trait per WIT interface (e.g. `shepherd::cow::cow_api::Host`, `nexum::host::local_store::Host`). `Shepherd::add_to_linker(&mut linker, \|state\| state)` registers every trait method as a host function. After that, calls from WASM resolve with zero dynamic dispatch overhead — the vtable is built at link time, not per-call. | +| **HostState — manifest.required check** | Before dispatching, `HostState` checks that the called capability is listed under `[capabilities].required` in the module's `module.toml`. If not, it returns `host-error { kind: denied }` immediately. The 0.2 engine validates known capability names at boot via `KNOWN_CAPABILITIES`; per-call gating is the M2 target. | +| **tracing::info!** | Every host call emits a structured trace event (capability name, chain id, etc.). Operators use `RUST_LOG=shepherd=debug` to see every call a module makes. | +| **host backend Rust function** | `HostState` implements one generated trait per WIT interface. Each `async fn` in the trait receives `&mut self` (giving access to all host resources) and returns the WIT-mapped Rust type. There are no CoW-strategy-specific backends — only the universal ones plus `cow-api` (ADR-0006). | +| **OrderBookPool** | Looks up the `OrderBookApi` client for the requested chain and calls `post_order`. Returns a 56-byte `OrderUid` on success or an `OrderPostError`-bearing host error on failure. | +| **ProviderPool (chain.request)** | Looks up the alloy provider for the requested chain and dispatches the JSON-RPC call (`eth_call`, `eth_getLogs`, etc.). | +| **engine-managed streams (chain.subscribe-*)** | Subscriptions are not exposed as runtime-callable host functions in 0.2. They are opened by the engine at boot from each module's declared `[[subscription]]` entries; events flow into the module via `on_event`. Dynamic `register-address` for factory patterns is deferred (ADR-0008). | +| **LocalStore** | Reads or writes a key in the module's namespace. The module sees plain keys; the host silently prepends a 32-byte namespace prefix. | +| **logging · clock · random** | Lightweight stateless helpers; implemented directly on `HostState` without a separate pool. | + +--- + +## 8. Repository Dependency Map + +```mermaid +graph TD + upstream["cowdao-grants/cow-rs\n(alpha.3 on crates.io — PR #5 base)"] + bleu_cr["bleu/cow-rs\n(PR #5 head branch)"] + prims["Protocol primitives added to PR #5:\n• OrderPostError + retry_hint\n• OrderBookApi::with_base_url\n• wasm32 feature-gate"] + existing["Already in PR #5:\nOrder · OrderCreation · OrderUid\nsigning schemes · OrderBookApi"] + patch["[patch.crates-io]\ncowprotocol → bleu/cow-rs @ rev"] + engine["nexum-engine\n(WIT host, supervisor, event loop)"] + witCowApi["shepherd:cow/cow-api WIT"] + modules["WASM modules\n(twap · eth-flow)"] + + upstream -->|"is PR base"| bleu_cr + bleu_cr --> prims + bleu_cr --> existing + patch -->|"redirects cowprotocol to rev of"| bleu_cr + engine -->|"workspace Cargo.toml declares"| patch + engine --> witCowApi + witCowApi -->|"backed by OrderBookApi"| existing + modules -->|"imports WIT"| witCowApi + modules -.->|"consumes types (wasm32 feature)"| existing + modules -.->|"matches on errors"| prims +``` + +### Node reference + +| Node | What it is | +|---|---| +| **cowdao-grants/cow-rs** | The upstream CoW Protocol Rust SDK, maintained by the DAO. Version `alpha.3` is published to crates.io but predates 18 follow-up commits Bleu has been pushing through PR #5. This is the PR base — changes land here eventually. | +| **bleu/cow-rs** | Bleu's repository, which is simultaneously the head branch of the DAO's open PR #5. Every commit Bleu pushes here also advances PR #5 for upstream review. This is not a long-lived parallel fork — it is the active PR branch (ADR-0004). | +| **Protocol primitives added to PR #5** | The three additions Bleu is pushing into PR #5: `OrderPostError` rich variants + `retry_hint()` (critical for module error handling), `OrderBookApi::with_base_url` (barn / staging / forked deployments), and `wasm32` feature-gating (critical so guest modules can consume `cowprotocol` types). All three are protocol primitives — they describe what CoW Protocol *is*, not how a particular strategy uses it. TWAP polling and EthFlow event decoding are explicitly *not* added here; they stay in module code (ADR-0007). | +| **Already in PR #5** | The types and orderbook client Bleu's modules consume but did not add: `Order`, `OrderCreation`, `OrderUid`, signing-scheme enums, and `OrderBookApi`. These existed in PR #5 before the M2 work. | +| **[patch.crates-io]** | A single line in the workspace `Cargo.toml` that tells Cargo to use `bleu/cow-rs` at a specific git rev instead of the `alpha.3` release on crates.io. Bumping the rev is the only change needed to pick up a new primitive after it is pushed to `bleu/cow-rs` (ADR-0004). | +| **nexum-engine** | The engine binary. Contains the WIT host implementations, Supervisor, EventLoop, config loaders, and alloy/redb integration. Contains no CoW Protocol logic — protocol primitives live in `bleu/cow-rs`; strategy logic lives in guest modules. | +| **shepherd:cow/cow-api WIT** | The only CoW-specific WIT interface in 0.2. The engine implements it (host side); WASM modules import it (guest side). Backed by `OrderBookPool` (and through that, `OrderBookApi` from `cow-rs`). | +| **WASM modules (twap · eth-flow)** | The grant deliverables. Compiled to `.wasm` Component Model binaries. Import only universal WIT interfaces (`chain`, `local-store`, `logging`) plus `shepherd:cow/cow-api`. Consume `cowprotocol` types directly through the wasm32 feature for building `OrderCreation` and pattern-matching on `OrderPostError`. Contain all TWAP and EthFlow strategy logic themselves (ADR-0006). | diff --git a/docs/diagrams/engine-boot.mmd b/docs/diagrams/engine-boot.mmd new file mode 100644 index 0000000..85ff094 --- /dev/null +++ b/docs/diagrams/engine-boot.mmd @@ -0,0 +1,59 @@ +sequenceDiagram + autonumber + actor Op as Operator + participant Bin as nexum-engine binary + participant FS as Filesystem + participant Trc as tracing subscriber + participant PP as ProviderPool + participant LS as LocalStore (redb) + participant OBP as OrderBookPool + participant Sup as Supervisor + participant Mod as Module instance(s) + participant EL as Event Loop + + Op->>Bin: nexum-engine --engine-config engine.toml + Bin->>FS: read engine.toml + FS-->>Bin: EngineConfig { engine, chains, modules } + + Bin->>Trc: init_tracing(log_level) + Note over Trc: RUST_LOG overrides
engine.toml.log_level + + Bin->>PP: ProviderPool::from_config(&cfg) + loop For each chain in cfg.chains + alt URL is ws:// or wss:// + PP->>PP: ProviderBuilder.connect_ws(WsConnect) + else URL is http:// or https:// + PP->>PP: ProviderBuilder.connect_http(Url) + end + Note over PP: Connection failure = fatal,
engine refuses to start + end + PP-->>Bin: pool ready + + Bin->>LS: LocalStore::open(state_dir) + Note over LS: Creates redb file if missing,
materialises shared table. + LS-->>Bin: store ready + + Bin->>OBP: OrderBookPool::default() + Note over OBP: One OrderBookApi per
cowprotocol::Chain variant. + OBP-->>Bin: pool ready + + Bin->>Sup: Supervisor::boot(cfg, host_resources) + loop For each [[modules]] entry in cfg.modules + Sup->>FS: read module.toml + FS-->>Sup: Manifest + Sup->>FS: load .wasm + FS-->>Sup: wasm bytes + Sup->>Mod: wasmtime Component compile + instantiate
with dedicated HostState + Sup->>Mod: call init(config) inside write txn + Mod-->>Sup: Ok or Err + Sup->>Sup: record subscriptions declared in manifest + end + Sup-->>Bin: modules loaded + + Bin->>EL: open block + log subscriptions + EL->>PP: subscribe_blocks(chain_id) per unique chain + EL->>PP: subscribe_logs(chain_id, filter) per unique filter + PP-->>EL: streams ready + + Bin->>EL: run_event_loop(futures::select_all) + Note over EL: Loop until SIGINT or SIGTERM diff --git a/docs/diagrams/module-lifecycle.mmd b/docs/diagrams/module-lifecycle.mmd new file mode 100644 index 0000000..7a73bd1 --- /dev/null +++ b/docs/diagrams/module-lifecycle.mmd @@ -0,0 +1,39 @@ +stateDiagram-v2 + direction LR + + [*] --> Resolve + + Resolve --> Load: bundle fetched OK + Resolve --> Dead: resolution failed permanently + + Load --> Init: wasmtime compile and instantiate OK + Load --> Restart: compile or instantiate failed + + Init --> Run: init(config) returned Ok + Init --> Restart: init returned Err or trapped + + Run --> Run: on_event handled OK + Run --> Restart: on_event trapped or returned Err or fuel exhausted + + Restart --> Init: backoff expired (1s, 2s, 4s, up to 5min cap) + Restart --> Dead: max_consecutive_failures reached (default 10) + + Dead --> Init: operator action (nexum module restart or reload) + Dead --> [*]: operator removes module + + note right of Restart + Exponential backoff with jitter. + Memory zeroed, local-store survives. + InstancePre reused, no recompile. + end note + + note right of Dead + Module excluded from dispatch. + Engine continues running other modules. + Manual intervention required to resume. + end note + + note right of Init + Runs inside an implicit write txn. + Ok commits, Err rolls back. + end note diff --git a/docs/diagrams/sequence-ethflow.mmd b/docs/diagrams/sequence-ethflow.mmd new file mode 100644 index 0000000..7427779 --- /dev/null +++ b/docs/diagrams/sequence-ethflow.mmd @@ -0,0 +1,39 @@ +sequenceDiagram + autonumber + actor User as User
(wallet) + participant Contract as CoWSwapEthFlow
(on-chain) + participant RPC as Chain RPC
(WSS) + participant EL as nexum-engine
Event Loop + participant Mod as ethflow-watcher
(WASM module) + participant SolDec as alloy_sol_types
(in module) + participant Cow as cowprotocol crate
(in module, wasm32) + participant Host as HostState + participant OB as CoW Orderbook + + User->>Contract: createOrder(order, msg.value=ETH) + Contract->>Contract: store orders[hash] = onchainData,
emit OrderPlacement + Contract-->>RPC: log emitted on block N + + RPC-->>EL: eth_subscribe logs delivery + EL->>Mod: on_event(LogEvent) + + Mod->>SolDec: decode OrderPlacement(sender, order, sig, data) + SolDec-->>Mod: (sender, GPv2OrderData, EIP1271-sig, data) + + Mod->>Cow: build OrderCreation with EIP-1271 scheme
pointing at CoWSwapEthFlow contract + Cow-->>Mod: OrderCreation + OrderUid + + Mod->>Host: cow-api.submit-order(chain_id, order_json) + Host->>OB: POST /api/v1/orders + OB-->>Host: result + + alt 200 OK with OrderUid + Host-->>Mod: Ok(OrderUid) + Mod->>Host: local-store.set("submitted:{uid}", ...) + else 4xx with error code + Host-->>Mod: Err(host-error with code) + Mod->>Cow: OrderPostError::try_from(host-error).retry_hint() + Cow-->>Mod: RetryHint variant + end + + Note over Mod: Module applies RetryHint:
TryNextBlock - log + skip
BackoffSeconds(s) - persist next_attempt = now + s
Drop - permanent failure, do not retry diff --git a/docs/diagrams/sequence-twap.mmd b/docs/diagrams/sequence-twap.mmd new file mode 100644 index 0000000..5b5e75e --- /dev/null +++ b/docs/diagrams/sequence-twap.mmd @@ -0,0 +1,56 @@ +sequenceDiagram + autonumber + actor User as User
(wallet) + participant Contract as ComposableCoW
(on-chain) + participant RPC as Chain RPC
(WSS) + participant EL as nexum-engine
Event Loop + participant Mod as twap-monitor
(WASM module) + participant SolDec as alloy_sol_types
(in module) + participant Cow as cowprotocol crate
(in module, wasm32) + participant Host as HostState
(host backends) + participant OB as CoW Orderbook + + Note over User,Contract: Registration (one-time per TWAP) + User->>Contract: create(twapParams) + Contract-->>RPC: emit ConditionalOrderCreated(owner, params) + RPC-->>EL: log delivered + EL->>Mod: on_event(LogEvent) + Mod->>SolDec: decode ConditionalOrderCreated + SolDec-->>Mod: (owner, params, salt) + Mod->>Host: local-store.set("watch:{owner}:{hash}", params) + + Note over EL,OB: Per-block polling loop + loop For each block N + RPC-->>EL: eth_subscribe newHeads + EL->>Mod: on_event(BlockEvent { N }) + Mod->>Host: local-store.list_keys("watch:") + + loop For each watch where next_attempt <= N + Mod->>Host: chain.request(chain_id, "eth_call", [...]) + Host->>RPC: eth_call ComposableCoW.
getTradeableOrderWithSignature(owner, params) + RPC-->>Host: return value or revert + Host-->>Mod: result (JSON encoded) + Mod->>SolDec: decode return or interpret revert reason + SolDec-->>Mod: PollOutcome (module-defined enum) + + alt Ready (order, signature) + Mod->>Cow: build OrderCreation + Cow-->>Mod: OrderCreation + Mod->>Host: cow-api.submit-order(chain_id, order_json) + Host->>OB: POST /api/v1/orders + OB-->>Host: Ok(OrderUid) or Err(error_code) + Host-->>Mod: Result + alt Ok(uid) + Mod->>Host: local-store.set("submitted:{uid}", ...) + else Err + Mod->>Cow: OrderPostError::try_from(host-error)
.retry_hint() + Cow-->>Mod: TryNextBlock / BackoffSeconds(s) / Drop + Mod->>Host: local-store update
(next_attempt or remove watch) + end + else NotReady (try at epoch t) + Mod->>Host: local-store.set("watch:...next_attempt", t) + else Terminal (TWAP completed or cancelled) + Mod->>Host: local-store.delete("watch:...") + end + end + end diff --git a/docs/diagrams/subscription-dispatch.mmd b/docs/diagrams/subscription-dispatch.mmd new file mode 100644 index 0000000..6f11359 --- /dev/null +++ b/docs/diagrams/subscription-dispatch.mmd @@ -0,0 +1,61 @@ +graph TB + subgraph manifests["Module Manifests (module.toml per module)"] + M1S["module twap-monitor:
[[subscription]] kind=log
chain=1, address=ComposableCoW,
topics=[ConditionalOrderCreated]
[[subscription]] kind=block, chain=1"] + M2S["module ethflow-watcher:
[[subscription]] kind=log
chain=1, address=CowEthFlow,
topics=[OrderPlacement]"] + M3S["module other-module:
[[subscription]] kind=block, chain=1"] + end + + subgraph supervisor["Supervisor: aggregate at boot"] + BlockAgg["Block subscriptions
grouped by chain_id"] + LogAgg["Log subscriptions
grouped by (chain_id, filter_key)
where filter_key = address+topics hash"] + end + + subgraph subs["Aggregated WSS subscriptions"] + BSub["eth_subscribe newHeads
chain=1 (shared by all subscribers)"] + LSub1["eth_subscribe logs
chain=1, filter=ComposableCoW+ConditionalOrderCreated
(owner: twap-monitor)"] + LSub2["eth_subscribe logs
chain=1, filter=CowEthFlow+OrderPlacement
(owner: ethflow-watcher)"] + end + + subgraph dispatch["Event Loop dispatch"] + Decide{Event kind?} + Broadcast["Fan-out: dispatch to ALL
subscribers of that chain"] + Route["Route: dispatch to OWNER
of matching filter_key only"] + end + + subgraph delivery["Module on_event invocations"] + D1["twap-monitor.on_event(BlockEvent)"] + D2["ethflow-watcher.on_event(LogEvent)
or twap-monitor (LogEvent)"] + D3["other-module.on_event(BlockEvent)"] + end + + M1S --> BlockAgg + M1S --> LogAgg + M2S --> LogAgg + M3S --> BlockAgg + + BlockAgg --> BSub + LogAgg --> LSub1 + LogAgg --> LSub2 + + BSub --> Decide + LSub1 --> Decide + LSub2 --> Decide + + Decide -- "block" --> Broadcast + Decide -- "log" --> Route + + Broadcast --> D1 + Broadcast --> D3 + Route --> D2 + + classDef manifest fill:#fff4e6,stroke:#ff9800,color:#000 + classDef agg fill:#e6f3ff,stroke:#1e88e5,color:#000 + classDef wss fill:#f0e6ff,stroke:#7e3ff2,color:#000 + classDef decide fill:#fffde7,stroke:#fbc02d,color:#000 + classDef dispatch fill:#e6ffe6,stroke:#2e7d32,color:#000 + + class M1S,M2S,M3S manifest + class BlockAgg,LogAgg agg + class BSub,LSub1,LSub2 wss + class Decide,Broadcast,Route decide + class D1,D2,D3 dispatch diff --git a/docs/diagrams/wit-call-path.mmd b/docs/diagrams/wit-call-path.mmd new file mode 100644 index 0000000..5fe4159 --- /dev/null +++ b/docs/diagrams/wit-call-path.mmd @@ -0,0 +1,34 @@ +sequenceDiagram + autonumber + participant Src as Module Rust source
(twap-monitor/src/lib.rs) + participant Bindgen as wit-bindgen
(generated stubs) + participant Wasm as WASM Component
(twap.wasm) + participant Wt as wasmtime instance + participant Lk as wasmtime Linker + participant HS as HostState (Rust) + participant PP as ProviderPool + participant Alloy as alloy provider + participant RPC as Chain RPC + + Note over Src,Wasm: Build phase (cargo build) + Src->>Bindgen: chain::request(chain_id, method, params)
(plain Rust call) + Bindgen->>Wasm: compile into WASM Component
with "nexum:host/chain" import + + Note over Wasm,RPC: Runtime: module calls chain::request + Wasm->>Wt: WASM import call:
"nexum:host/chain"."request" + Wt->>Lk: lookup binding for import + Lk->>HS: invoke trait impl:
HostState::Chain::request(...) + Note over HS: tracing::info!("[chain] request
chain=... method=...") + HS->>PP: pool.get(chain_id) + PP-->>HS: &DynProvider + HS->>Alloy: provider.raw_request(method, params) + Alloy->>RPC: JSON-RPC over WSS/HTTP + RPC-->>Alloy: response bytes + Alloy-->>HS: serde_json::Value + HS-->>Lk: Ok(stringified JSON) + Lk-->>Wt: return value (marshaled) + Wt-->>Wasm: import call returns + Wasm-->>Bindgen: unmarshal canonical ABI
back to Rust types + Bindgen-->>Src: returns Result + + Note over Src,RPC: All host calls follow this path.
Only the trait impl on the right side
(HS → PP/OBP/LS/...) changes per capability. From 78092532bdad7e07bfef7848ba64a64da0ca85a0 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Mon, 15 Jun 2026 10:24:54 -0300 Subject: [PATCH 23/29] chore(deps): bump cowprotocol patch to bleu/cow-rs main (BLEU-822 + BLEU-823 in) --- Cargo.toml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index df12a9a..fe8bde4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,15 +100,17 @@ todo = "deny" # `cowprotocol` v1.0.0-alpha.3 (the crates.io release the engine # depends on) was cut from `cowdao-grants/cow-rs` PR #5 at commit -# `1742ffa`. `bleu/cow-rs` main has 18 commits since, including the +# `1742ffa`. `bleu/cow-rs` main has diverged since with: the # `composable::Proof` width fix (relevant to the TWAP poll path), -# `OrderCreation` zero-from-address fast-fail (closes a MEDIUM -# review finding from PR #5), and the `order_book` / `composable` -# submodule splits. Patching to that commit picks them up without -# waiting for an alpha.4 publish. Drop once `cowprotocol >= 1.0.0-alpha.4` -# ships. +# `OrderCreation` zero-from-address fast-fail, the `order_book` / +# `composable` submodule splits, `OrderPostErrorKind` + `retry_hint()` +# (BLEU-822, the protocol-level retry contract M2 modules dispatch +# on), and `OrderBookApi::with_base_url(chain, base_url)` for barn / +# staging routing (BLEU-823). Patching to that commit picks the lot +# up without waiting for an alpha.4 publish. Drop once +# `cowprotocol >= 1.0.0-alpha.4` ships. [patch.crates-io] -cowprotocol = { git = "https://github.com/bleu/cow-rs", rev = "c012404ffefc411bff543d2290e19ba7fbef2516" } +cowprotocol = { git = "https://github.com/bleu/cow-rs", rev = "57f5f553ab28c9fff54089daf2d39b4282f3e4dd" } [profile.dev] panic = "abort" From 41fe0cbdc80e2b5ab041a5cf65acecef3bd046cb Mon Sep 17 00:00:00 2001 From: brunota20 Date: Tue, 23 Jun 2026 17:58:16 -0300 Subject: [PATCH 24/29] chore(rust-idiomatic): M2 compliance pass (filtered from M4/M5 compliance) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filtered subset of the compliance applied in PRs #66/#67 of bleu/nullis-shepherd, restricted to files that exist on the M2 epic head. M3+ files (shepherd-sdk, examples, backtest, deploy artifacts) and M4-coupled hunks (ProviderError typed-source variants, JoinSet reconnect tasks, supervisor restart helper) are skipped — they land via their own upstream PRs. Brings M2 epic in line with the repo-wide rust rubric (typed errors, no anyhow in libs, em-dash sweep, #[non_exhaustive] on public error enums). --- crates/nexum-engine/src/engine_config.rs | 23 +- crates/nexum-engine/src/host/cow_orderbook.rs | 1 - crates/nexum-engine/src/host/error.rs | 16 +- crates/nexum-engine/src/host/impls/cow_api.rs | 4 +- crates/nexum-engine/src/host/mod.rs | 13 +- crates/nexum-engine/src/main.rs | 3 +- crates/nexum-engine/src/manifest/error.rs | 56 +-- crates/nexum-engine/src/manifest/load.rs | 37 +- crates/nexum-engine/src/manifest/mod.rs | 6 +- crates/nexum-engine/src/supervisor.rs | 1 - crates/nexum-engine/src/supervisor/tests.rs | 386 +++++++++--------- 11 files changed, 271 insertions(+), 275 deletions(-) diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs index 9637981..e1aa9fb 100644 --- a/crates/nexum-engine/src/engine_config.rs +++ b/crates/nexum-engine/src/engine_config.rs @@ -20,8 +20,27 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde::Deserialize; +use thiserror::Error; use tracing::{info, warn}; +/// Errors surfaced by [`load_or_default`]. +/// +/// Library-side modules must not propagate `anyhow::Error`; the rust +/// idiomatic rubric reserves `anyhow` for `main.rs` and +/// `supervisor.rs` top-level dispatch. The variants carry the +/// upstream error via `#[from]` so the caller in `main.rs` (which +/// uses `anyhow`) gets a free conversion through `?`. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum EngineConfigError { + /// Failed to read the config file from disk. + #[error("read engine config: {0}")] + Io(#[from] std::io::Error), + /// Config file was unparseable as TOML. + #[error("parse engine config: {0}")] + Toml(#[from] toml::de::Error), +} + /// Engine-side configuration loaded from `engine.toml`. #[derive(Debug, Default, Deserialize)] pub struct EngineConfig { @@ -91,8 +110,8 @@ fn default_log_level() -> String { } /// Read an engine config from disk, returning defaults if the file is -/// missing. Parse errors propagate. -pub fn load_or_default(path: Option<&Path>) -> anyhow::Result { +/// missing. Parse errors propagate via [`EngineConfigError`]. +pub fn load_or_default(path: Option<&Path>) -> Result { let path = match path { Some(p) => p.to_path_buf(), None => PathBuf::from("engine.toml"), diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index 49c1945..865ab88 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -136,6 +136,5 @@ pub enum CowApiError { Orderbook(#[from] cowprotocol::Error), } - #[cfg(test)] mod tests; diff --git a/crates/nexum-engine/src/host/error.rs b/crates/nexum-engine/src/host/error.rs index 15b7255..65ee724 100644 --- a/crates/nexum-engine/src/host/error.rs +++ b/crates/nexum-engine/src/host/error.rs @@ -1,6 +1,5 @@ //! Small constructors that wrap the WIT `HostError` shape, used by -//! every `Host` trait impl, plus the lowercase hex encoder shared by -//! the `cow-api` submission path. +//! every `Host` trait impl. use crate::bindings::HostError; use crate::bindings::nexum::host::types::HostErrorKind; @@ -27,16 +26,3 @@ pub(crate) fn internal_error(domain: &str, detail: impl Into) -> HostErr data: None, } } - -/// Lowercase hex encoder. Kept in the engine binary rather than -/// pulling a `hex` crate just for one call site. Writes into the -/// pre-allocated buffer to avoid the per-byte `String` allocation -/// `format!("{b:02x}")` would do. -pub(crate) fn hex_encode(bytes: &[u8]) -> String { - use std::fmt::Write as _; - let mut s = String::with_capacity(bytes.len() * 2); - for b in bytes { - write!(s, "{b:02x}").expect("writing to String never fails"); - } - s -} diff --git a/crates/nexum-engine/src/host/impls/cow_api.rs b/crates/nexum-engine/src/host/impls/cow_api.rs index b3b51d5..5971e03 100644 --- a/crates/nexum-engine/src/host/impls/cow_api.rs +++ b/crates/nexum-engine/src/host/impls/cow_api.rs @@ -7,7 +7,7 @@ use std::time::Instant; use crate::bindings::nexum::host::types::HostErrorKind; use crate::bindings::{HostError, shepherd}; use crate::host::cow_orderbook::CowApiError; -use crate::host::error::{hex_encode, internal_error, unimplemented}; +use crate::host::error::{internal_error, unimplemented}; use crate::host::state::HostState; impl shepherd::cow::cow_api::Host for HostState { @@ -58,7 +58,7 @@ impl shepherd::cow::cow_api::Host for HostState { let start = Instant::now(); tracing::debug!(chain_id, bytes = order_data.len(), "cow-api::submit-order"); let result = match self.cow.submit_order_json(chain_id, &order_data).await { - Ok(uid) => Ok(format!("0x{}", hex_encode(uid.as_slice()))), + Ok(uid) => Ok(alloy_primitives::hex::encode_prefixed(uid.as_slice())), Err(CowApiError::UnknownChain(id)) => Err(unimplemented( "cow-api", format!("chain {id} not in cowprotocol"), diff --git a/crates/nexum-engine/src/host/mod.rs b/crates/nexum-engine/src/host/mod.rs index 20f2ec2..a6662ee 100644 --- a/crates/nexum-engine/src/host/mod.rs +++ b/crates/nexum-engine/src/host/mod.rs @@ -5,17 +5,16 @@ //! - [`state`]: `HostState` struct + `WasiView` impl, the receiver //! every WIT `Host` trait is implemented for. //! - [`error`]: small constructors that build the WIT `HostError` -//! shape (`unimplemented`, `internal_error`) plus the lowercase -//! `hex_encode` shared by the `cow-api` submission path. +//! shape (`unimplemented`, `internal_error`). //! - [`cow_orderbook`], [`provider_pool`], [`local_store_redb`]: //! capability backends. Pure code with no bindgen types, so each //! can be unit-tested without spinning up a wasmtime store. //! - [`impls`] (private): the bindgen-side trait impls, one file per //! WIT interface, that dispatch to the backends above. -pub mod cow_orderbook; -pub mod error; +pub(crate) mod cow_orderbook; +pub(crate) mod error; mod impls; -pub mod local_store_redb; -pub mod provider_pool; -pub mod state; +pub(crate) mod local_store_redb; +pub(crate) mod provider_pool; +pub(crate) mod state; diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 8a1598f..04bc076 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -116,7 +116,8 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } - let block_streams = runtime::event_loop::open_block_streams(&provider_pool, &block_chains).await; + let block_streams = + runtime::event_loop::open_block_streams(&provider_pool, &block_chains).await; let log_streams = runtime::event_loop::open_log_streams(&provider_pool, log_subs).await; let shutdown = async { diff --git a/crates/nexum-engine/src/manifest/error.rs b/crates/nexum-engine/src/manifest/error.rs index 98bf7d7..c69cc49 100644 --- a/crates/nexum-engine/src/manifest/error.rs +++ b/crates/nexum-engine/src/manifest/error.rs @@ -1,34 +1,35 @@ //! Error types for manifest parsing and capability enforcement. +use thiserror::Error; + use super::types::KNOWN_CAPABILITIES; /// Errors returned while loading or validating a manifest. -#[derive(Debug)] +#[derive(Debug, Error)] +#[non_exhaustive] pub enum ParseError { - Io(std::io::Error), - Toml(toml::de::Error), + /// Failed to read the manifest file from disk. + #[error("manifest: i/o: {0}")] + Io(#[from] std::io::Error), + /// Manifest file was not valid TOML. + #[error("manifest: parse: {0}")] + Toml(#[from] toml::de::Error), + /// `[capabilities].required` or `.optional` listed a capability + /// the engine does not recognise. + #[error( + "manifest: unknown capability {name:?} in [capabilities].required (known: {})", + KNOWN_CAPABILITIES.join(", "), + name = .0, + )] UnknownCapability(String), } -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Io(e) => write!(f, "manifest: i/o: {e}"), - Self::Toml(e) => write!(f, "manifest: parse: {e}"), - Self::UnknownCapability(name) => write!( - f, - "manifest: unknown capability {:?} in [capabilities].required (known: {})", - name, - KNOWN_CAPABILITIES.join(", ") - ), - } - } -} - -impl std::error::Error for ParseError {} - /// Error returned when a component's WIT imports exceed its declared capabilities. -#[derive(Debug)] +#[derive(Debug, Error)] +#[error( + "component imports `{capability}` ({wit_import}) but it is not listed in \ + [capabilities].required or [capabilities].optional" +)] pub struct CapabilityViolation { /// Capability name (e.g. `"remote-store"`). pub capability: String, @@ -36,16 +37,3 @@ pub struct CapabilityViolation { /// `"nexum:host/remote-store@0.2.0"`). pub wit_import: String, } - -impl std::fmt::Display for CapabilityViolation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "component imports `{}` ({}) but it is not listed in \ - [capabilities].required or [capabilities].optional", - self.capability, self.wit_import - ) - } -} - -impl std::error::Error for CapabilityViolation {} diff --git a/crates/nexum-engine/src/manifest/load.rs b/crates/nexum-engine/src/manifest/load.rs index b857a76..9f8975e 100644 --- a/crates/nexum-engine/src/manifest/load.rs +++ b/crates/nexum-engine/src/manifest/load.rs @@ -8,21 +8,24 @@ use std::collections::HashSet; use std::path::Path; +use tracing::{info, warn}; + use super::error::ParseError; use super::types::{KNOWN_CAPABILITIES, LoadedManifest, Manifest}; /// Read `module.toml` from `path`, parse, validate, and emit a deprecation /// warning if `[capabilities]` is absent (0.1-compat fallback). pub fn load(path: &Path) -> Result { - let raw = std::fs::read_to_string(path).map_err(ParseError::Io)?; - let manifest: Manifest = toml::from_str(&raw).map_err(ParseError::Toml)?; + let raw = std::fs::read_to_string(path)?; + let manifest: Manifest = toml::from_str(&raw)?; let caps = manifest.capabilities.as_ref(); if caps.is_none() { - eprintln!( - "[deprecation] no [capabilities] section in module.toml - \ - defaulting to all-required (0.1 behaviour). This default \ - will be removed in 0.3; add an explicit [capabilities] block." + warn!( + target: "manifest", + "no [capabilities] section in module.toml - defaulting to \ + all-required (0.1 behaviour). This default will be removed \ + in 0.3; add an explicit [capabilities] block." ); } @@ -34,16 +37,13 @@ pub fn load(path: &Path) -> Result { } } if !c.required.is_empty() { - eprintln!( - "[manifest] required capabilities: {}", - c.required.join(", ") - ); + info!(target: "manifest", required = %c.required.join(", "), "required capabilities"); } if !c.optional.is_empty() { - eprintln!( - "[manifest] optional capabilities (advisory in 0.2; trap-stub fallback \ - ships in 0.3): {}", - c.optional.join(", ") + info!( + target: "manifest", + optional = %c.optional.join(", "), + "optional capabilities (advisory in 0.2; trap-stub fallback ships in 0.3)", ); } } @@ -53,7 +53,7 @@ pub fn load(path: &Path) -> Result { .map(|h| h.allow.clone()) .unwrap_or_default(); if !http_allowlist.is_empty() { - eprintln!("[manifest] http allowlist: {}", http_allowlist.join(", ")); + info!(target: "manifest", allow = %http_allowlist.join(", "), "http allowlist"); } let config = manifest @@ -72,9 +72,10 @@ pub fn load(path: &Path) -> Result { /// Synthesise a "0.1 fallback" manifest for when no `module.toml` is found. /// Emits the same deprecation warning as a missing-section manifest. pub fn fallback_manifest() -> LoadedManifest { - eprintln!( - "[deprecation] no module.toml found - defaulting to all-required \ - (0.1 behaviour). This default will be removed in 0.3; ship a \ + warn!( + target: "manifest", + "no module.toml found - defaulting to all-required (0.1 \ + behaviour). This default will be removed in 0.3; ship a \ module.toml alongside your component." ); LoadedManifest { diff --git a/crates/nexum-engine/src/manifest/mod.rs b/crates/nexum-engine/src/manifest/mod.rs index 9cd00b6..4569157 100644 --- a/crates/nexum-engine/src/manifest/mod.rs +++ b/crates/nexum-engine/src/manifest/mod.rs @@ -31,9 +31,9 @@ mod error; mod load; mod types; -pub use capabilities::enforce_capabilities; -pub use load::{extract_host, fallback_manifest, host_allowed, load}; -pub use types::{LoadedManifest, Subscription}; +pub(crate) use capabilities::enforce_capabilities; +pub(crate) use load::{extract_host, fallback_manifest, host_allowed, load}; +pub(crate) use types::{LoadedManifest, Subscription}; // CapabilityViolation, ParseError, and the *Section structs are // reachable through these functions' return / argument types; // consumers that need to name them directly do so via diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 7aaec9e..adbfebb 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -427,6 +427,5 @@ fn build_alloy_filter( Ok(filter) } - #[cfg(test)] mod tests; diff --git a/crates/nexum-engine/src/supervisor/tests.rs b/crates/nexum-engine/src/supervisor/tests.rs index b290ed4..331cc9c 100644 --- a/crates/nexum-engine/src/supervisor/tests.rs +++ b/crates/nexum-engine/src/supervisor/tests.rs @@ -1,125 +1,125 @@ - use std::path::{Path, PathBuf}; - - use super::*; - - #[test] - fn empty_supervisor_returns_no_subscriptions() { - let sup = Supervisor { - modules: Vec::new(), - }; - assert!(sup.block_chains().is_empty()); - assert!(sup.log_subscriptions().is_empty()); - assert_eq!(sup.module_count(), 0); - } - - // ── E2E helpers ─────────────────────────────────────────────────────── - - /// Path to the pre-built example WASM component. Tests that need it - /// call `example_wasm_or_skip()` which skips gracefully if absent. - fn example_wasm() -> PathBuf { - // CARGO_MANIFEST_DIR → crates/nexum-engine - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .join("target/wasm32-wasip2/release/example.wasm") - } - - fn example_module_toml() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .join("modules/example/module.toml") - } - - /// Returns `None` and prints a skip message if the fixture isn't built. - fn example_wasm_or_skip() -> Option { - let p = example_wasm(); - if p.exists() { - Some(p) - } else { - eprintln!( - "SKIP: {} not found - run `just build-module` to enable E2E tests", - p.display() - ); - None - } - } - - fn make_wasmtime_engine() -> wasmtime::Engine { - let mut config = wasmtime::Config::new(); - config.wasm_component_model(true); - config.consume_fuel(true); - wasmtime::Engine::new(&config).expect("wasmtime engine") - } - - fn make_linker(engine: &wasmtime::Engine) -> Linker { - let mut linker = Linker::::new(engine); - crate::Shepherd::add_to_linker::>( - &mut linker, - |s| s, - ) - .expect("add_to_linker"); - wasmtime_wasi::p2::add_to_linker_async(&mut linker).expect("add_wasi"); - linker - } - - /// Return `(dir, store)` so the test holds the `TempDir` for the - /// duration of the test scope and cleans it up on drop. Forgetting - /// the dir (the old `ManuallyDrop` approach) leaks it for the - /// entire process lifetime. - fn temp_local_store() -> (tempfile::TempDir, crate::host::local_store_redb::LocalStore) { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("ls.redb"); - let store = crate::host::local_store_redb::LocalStore::open(path).expect("local store"); - (dir, store) - } - - // ── E2E tests ───────────────────────────────────────────────────────── - - /// Boot supervisor with the example module; verify it starts alive. - #[tokio::test] - async fn e2e_supervisor_boots_example_module() { - let Some(wasm) = example_wasm_or_skip() else { - return; - }; - let engine = make_wasmtime_engine(); - let linker = make_linker(&engine); - let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); - let provider_pool = crate::host::provider_pool::ProviderPool::empty(); - let (_dir, local_store) = temp_local_store(); - - let supervisor = Supervisor::boot_single( - &engine, - &linker, - &wasm, - Some(example_module_toml()).as_deref(), - &cow_pool, - &provider_pool, - &local_store, - ) - .await - .expect("boot_single"); - - assert_eq!(supervisor.module_count(), 1); - assert_eq!(supervisor.alive_count(), 1); +use std::path::{Path, PathBuf}; + +use super::*; + +#[test] +fn empty_supervisor_returns_no_subscriptions() { + let sup = Supervisor { + modules: Vec::new(), + }; + assert!(sup.block_chains().is_empty()); + assert!(sup.log_subscriptions().is_empty()); + assert_eq!(sup.module_count(), 0); +} + +// ── E2E helpers ─────────────────────────────────────────────────────── + +/// Path to the pre-built example WASM component. Tests that need it +/// call `example_wasm_or_skip()` which skips gracefully if absent. +fn example_wasm() -> PathBuf { + // CARGO_MANIFEST_DIR → crates/nexum-engine + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("target/wasm32-wasip2/release/example.wasm") +} + +fn example_module_toml() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("modules/example/module.toml") +} + +/// Returns `None` and prints a skip message if the fixture isn't built. +fn example_wasm_or_skip() -> Option { + let p = example_wasm(); + if p.exists() { + Some(p) + } else { + eprintln!( + "SKIP: {} not found - run `just build-module` to enable E2E tests", + p.display() + ); + None } - - /// Boot with a manifest that subscribes to block events; dispatch one - /// block event and verify the module was invoked and stayed alive. - #[tokio::test] - async fn e2e_block_subscription_dispatched() { - let Some(wasm) = example_wasm_or_skip() else { - return; - }; - let dir = tempfile::tempdir().unwrap(); - let manifest = dir.path().join("module.toml"); - std::fs::write( - &manifest, - r#" +} + +fn make_wasmtime_engine() -> wasmtime::Engine { + let mut config = wasmtime::Config::new(); + config.wasm_component_model(true); + config.consume_fuel(true); + wasmtime::Engine::new(&config).expect("wasmtime engine") +} + +fn make_linker(engine: &wasmtime::Engine) -> Linker { + let mut linker = Linker::::new(engine); + crate::Shepherd::add_to_linker::< + crate::HostState, + wasmtime::component::HasSelf, + >(&mut linker, |s| s) + .expect("add_to_linker"); + wasmtime_wasi::p2::add_to_linker_async(&mut linker).expect("add_wasi"); + linker +} + +/// Return `(dir, store)` so the test holds the `TempDir` for the +/// duration of the test scope and cleans it up on drop. Forgetting +/// the dir (the old `ManuallyDrop` approach) leaks it for the +/// entire process lifetime. +fn temp_local_store() -> (tempfile::TempDir, crate::host::local_store_redb::LocalStore) { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("ls.redb"); + let store = crate::host::local_store_redb::LocalStore::open(path).expect("local store"); + (dir, store) +} + +// ── E2E tests ───────────────────────────────────────────────────────── + +/// Boot supervisor with the example module; verify it starts alive. +#[tokio::test] +async fn e2e_supervisor_boots_example_module() { + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let (_dir, local_store) = temp_local_store(); + + let supervisor = Supervisor::boot_single( + &engine, + &linker, + &wasm, + Some(example_module_toml()).as_deref(), + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot_single"); + + assert_eq!(supervisor.module_count(), 1); + assert_eq!(supervisor.alive_count(), 1); +} + +/// Boot with a manifest that subscribes to block events; dispatch one +/// block event and verify the module was invoked and stayed alive. +#[tokio::test] +async fn e2e_block_subscription_dispatched() { + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let manifest = dir.path().join("module.toml"); + std::fs::write( + &manifest, + r#" [module] name = "example" @@ -130,73 +130,77 @@ required = ["logging"] kind = "block" chain_id = 1 "#, - ) - .unwrap(); - - let engine = make_wasmtime_engine(); - let linker = make_linker(&engine); - let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); - let provider_pool = crate::host::provider_pool::ProviderPool::empty(); - let (_dir, local_store) = temp_local_store(); - - let mut supervisor = Supervisor::boot_single( - &engine, - &linker, - &wasm, - Some(&manifest), - &cow_pool, - &provider_pool, - &local_store, - ) - .await - .expect("boot_single"); - - let block = nexum::host::types::Block { - chain_id: 1, - number: 19_000_000, - hash: vec![0xab; 32], - timestamp: 1_700_000_000_000, - }; - let dispatched = supervisor.dispatch_block(block).await; - assert_eq!(dispatched, 1, "one module subscribed to chain 1 blocks"); - assert_eq!(supervisor.alive_count(), 1, "module must remain alive"); - } - - // ── build_alloy_filter ──────────────────────────────────────────────── - - #[test] - fn alloy_filter_with_address_and_topic() { - let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; - let topic = "0x237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c00"; - let filter = build_alloy_filter(Some(addr), Some(topic)).unwrap(); - // Check address is set (alloy Filter doesn't expose a simple getter, - // but we can verify the filter serialises the address field). - let serialised = serde_json::to_value(&filter).unwrap(); - let addr_field = serialised.get("address").unwrap().to_string().to_lowercase(); - assert!(addr_field.contains(&addr.to_lowercase()[2..])); // strip 0x - } - - #[test] - fn alloy_filter_no_address_no_topic() { - let filter = build_alloy_filter(None, None).unwrap(); - let serialised = serde_json::to_value(&filter).unwrap(); - // Address and topics should be absent or null. - assert!( - serialised.get("address").is_none() - || serialised["address"].is_null() - || serialised["address"] == serde_json::json!([]) - ); - } - - #[test] - fn alloy_filter_rejects_bad_address() { - let err = build_alloy_filter(Some("not-an-address"), None); - assert!(err.is_err()); - } - - #[test] - fn alloy_filter_rejects_bad_topic() { - let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; - let err = build_alloy_filter(Some(addr), Some("not-a-topic")); - assert!(err.is_err()); - } + ) + .unwrap(); + + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let (_dir, local_store) = temp_local_store(); + + let mut supervisor = Supervisor::boot_single( + &engine, + &linker, + &wasm, + Some(&manifest), + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot_single"); + + let block = nexum::host::types::Block { + chain_id: 1, + number: 19_000_000, + hash: vec![0xab; 32], + timestamp: 1_700_000_000_000, + }; + let dispatched = supervisor.dispatch_block(block).await; + assert_eq!(dispatched, 1, "one module subscribed to chain 1 blocks"); + assert_eq!(supervisor.alive_count(), 1, "module must remain alive"); +} + +// ── build_alloy_filter ──────────────────────────────────────────────── + +#[test] +fn alloy_filter_with_address_and_topic() { + let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; + let topic = "0x237e158222e3e6968b72b9db0d8043aacf074ad9f650f0d1606b4d82ee432c00"; + let filter = build_alloy_filter(Some(addr), Some(topic)).unwrap(); + // Check address is set (alloy Filter doesn't expose a simple getter, + // but we can verify the filter serialises the address field). + let serialised = serde_json::to_value(&filter).unwrap(); + let addr_field = serialised + .get("address") + .unwrap() + .to_string() + .to_lowercase(); + assert!(addr_field.contains(&addr.to_lowercase()[2..])); // strip 0x +} + +#[test] +fn alloy_filter_no_address_no_topic() { + let filter = build_alloy_filter(None, None).unwrap(); + let serialised = serde_json::to_value(&filter).unwrap(); + // Address and topics should be absent or null. + assert!( + serialised.get("address").is_none() + || serialised["address"].is_null() + || serialised["address"] == serde_json::json!([]) + ); +} + +#[test] +fn alloy_filter_rejects_bad_address() { + let err = build_alloy_filter(Some("not-an-address"), None); + assert!(err.is_err()); +} + +#[test] +fn alloy_filter_rejects_bad_topic() { + let addr = "0xC92E8bdf79f0507f65a392b0ab4667716BFE0110"; + let err = build_alloy_filter(Some(addr), Some("not-a-topic")); + assert!(err.is_err()); +} From 2c879c5196cbee2c926ce91f8a867cd60d36537e Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 16:42:10 -0300 Subject: [PATCH 25/29] docs(nexum-engine): fix rustdoc intra-doc links after pub(crate) sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The M2 compliance pass (2b11f91) narrowed several manifest/host re-exports from `pub` to `pub(crate)`. Three intra-doc links inherited from the wider M1 docs no longer resolve under `-D warnings`: - `crate::host::impls` — module is `mod impls;` (always private); the link doc-rendered as code at the M1-era visibility. Demote to a plain code span; the path is still grep-able and accurate. - `manifest::mod` link to `[load]` — ambiguous now that `pub(crate) use load::{... load};` makes `load` both a module and a function. Use `mod@load` to disambiguate to the module (matches the surrounding prose, which describes the file's responsibilities). - `manifest::types` link to `[super::load]` — same ambiguity, same fix: `mod@super::load`. Fixes the `RUSTDOCFLAGS="-D warnings" cargo doc` gate on dev/m2-base. --- crates/nexum-engine/src/bindings.rs | 2 +- crates/nexum-engine/src/manifest/mod.rs | 2 +- crates/nexum-engine/src/manifest/types.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/nexum-engine/src/bindings.rs b/crates/nexum-engine/src/bindings.rs index d0f57dd..9ddd00c 100644 --- a/crates/nexum-engine/src/bindings.rs +++ b/crates/nexum-engine/src/bindings.rs @@ -5,7 +5,7 @@ //! natively - no vendored `deps/` tree needed. The world name is fully //! qualified. //! -//! Every `Host` trait impl in [`crate::host::impls`] consumes types +//! Every `Host` trait impl in `crate::host::impls` consumes types //! generated here. wasmtime::component::bindgen!({ diff --git a/crates/nexum-engine/src/manifest/mod.rs b/crates/nexum-engine/src/manifest/mod.rs index 4569157..45c48da 100644 --- a/crates/nexum-engine/src/manifest/mod.rs +++ b/crates/nexum-engine/src/manifest/mod.rs @@ -21,7 +21,7 @@ //! //! - [`types`]: the serde `Manifest` shape + `LoadedManifest` the engine //! actually consumes, plus the `KNOWN_CAPABILITIES` registry. -//! - [`load`]: `module.toml` -> `LoadedManifest`, plus the host/URL +//! - [`mod@load`]: `module.toml` -> `LoadedManifest`, plus the host/URL //! helpers the `http` backend uses at request time. //! - [`capabilities`]: WIT-import vs declared-capabilities cross-check. //! - [`error`]: `ParseError`, `CapabilityViolation`. diff --git a/crates/nexum-engine/src/manifest/types.rs b/crates/nexum-engine/src/manifest/types.rs index 403a201..e91bff1 100644 --- a/crates/nexum-engine/src/manifest/types.rs +++ b/crates/nexum-engine/src/manifest/types.rs @@ -1,7 +1,7 @@ //! Data structures: `Manifest`, sections, and `LoadedManifest`. //! //! Plain serde shapes plus the `KNOWN_CAPABILITIES` registry. The parsing -//! and validation logic lives in [`super::load`]; capability enforcement +//! and validation logic lives in [`mod@super::load`]; capability enforcement //! in [`super::capabilities`]. use serde::Deserialize; From 1984a3e7d602311188ebf30f9554db87ff32f6ef Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 19:08:39 -0300 Subject: [PATCH 26/29] chore(nexum-engine): derive strum::IntoStaticStr on error enums Audit reference: milestone-rubric-grant-audit-2026-06-25.md, Major #1. Vendored rubric mandates `strum::IntoStaticStr` (with `#[strum(serialize_all = "snake_case")]`) on every error enum so `error_kind` labels on the `shepherd_chain_request_total` / `shepherd_cow_api_*` counters stay in lock-step with the Rust source of truth instead of growing a `match err { ... => "connect" ... }` ladder per call site. Enums covered on this milestone (the ones present on dev/m2-base): - `nexum_engine::host::cow_orderbook::CowApiError` - `nexum_engine::host::provider_pool::ProviderError` - `nexum_engine::manifest::error::ParseError` - `nexum_engine::engine_config::EngineConfigError` Also adds `#[non_exhaustive]` to `CowApiError` and `ProviderError` (audit Major #2). The other two already carried it. `strum = "0.26"` lands as a direct dep on nexum-engine. The workspace-deps hoist (audit P1, Major #5) is intentionally a separate judgment call left to Bruno; this commit ships the substantive rubric fix without coupling to the broader Cargo.toml restructure. --- crates/nexum-engine/Cargo.toml | 9 +++++++++ crates/nexum-engine/src/engine_config.rs | 7 ++++++- crates/nexum-engine/src/host/cow_orderbook.rs | 10 +++++++++- crates/nexum-engine/src/host/provider_pool.rs | 9 ++++++++- crates/nexum-engine/src/manifest/error.rs | 8 +++++++- 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/crates/nexum-engine/Cargo.toml b/crates/nexum-engine/Cargo.toml index b054ff0..223fd4e 100644 --- a/crates/nexum-engine/Cargo.toml +++ b/crates/nexum-engine/Cargo.toml @@ -16,6 +16,15 @@ wasmtime-wasi = "45" # Async + error plumbing. anyhow.workspace = true thiserror.workspace = true +# `strum::IntoStaticStr` on error enums gives metric labels (`error_kind`) +# free via a snake_case `&'static str` for every variant. Used at +# `tracing::warn!(error_kind = .into(), ...)` sites and +# any `metrics::counter!(... "error_kind" => kind)` recordings, so the +# Prometheus labels stay in lock-step with the Rust enum source of +# truth instead of needing a `match err { ... => "connect" ... }` +# ladder per call site. Pinned via the workspace so every consumer +# moves in lockstep. +strum.workspace = true tokio.workspace = true clap.workspace = true diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs index e1aa9fb..aeb0157 100644 --- a/crates/nexum-engine/src/engine_config.rs +++ b/crates/nexum-engine/src/engine_config.rs @@ -20,6 +20,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde::Deserialize; +use strum::IntoStaticStr; use thiserror::Error; use tracing::{info, warn}; @@ -30,7 +31,11 @@ use tracing::{info, warn}; /// `supervisor.rs` top-level dispatch. The variants carry the /// upstream error via `#[from]` so the caller in `main.rs` (which /// uses `anyhow`) gets a free conversion through `?`. -#[derive(Debug, Error)] +/// +/// `IntoStaticStr` exposes the snake_case variant name for metric +/// labels and structured-log `error_kind` fields. +#[derive(Debug, Error, IntoStaticStr)] +#[strum(serialize_all = "snake_case")] #[non_exhaustive] pub enum EngineConfigError { /// Failed to read the config file from disk. diff --git a/crates/nexum-engine/src/host/cow_orderbook.rs b/crates/nexum-engine/src/host/cow_orderbook.rs index 865ab88..364e8df 100644 --- a/crates/nexum-engine/src/host/cow_orderbook.rs +++ b/crates/nexum-engine/src/host/cow_orderbook.rs @@ -19,6 +19,7 @@ use std::collections::BTreeMap; use cowprotocol::{Chain, OrderBookApi, OrderCreation, OrderUid}; +use strum::IntoStaticStr; use thiserror::Error; /// Process-wide pool of `OrderBookApi` clients keyed by EVM chain id. @@ -120,7 +121,14 @@ impl OrderBookPool { } } -#[derive(Debug, Error)] +/// `IntoStaticStr` exposes the snake_case variant name as a +/// `&'static str` (`"unknown_chain"`, `"bad_method"`, ...) so the +/// `shepherd_cow_api_*` metric labels and structured-log fields stay +/// in sync with the Rust source of truth instead of growing a +/// `match err { ... => "decode" ... }` ladder per call site. +#[derive(Debug, Error, IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +#[non_exhaustive] pub enum CowApiError { #[error("unknown chain {0} (no cowprotocol::Chain variant)")] UnknownChain(u64), diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index d65e391..fa90889 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -20,6 +20,7 @@ use alloy_rpc_types_eth::{Filter, Header, Log}; use futures::stream::Stream; use futures::stream::StreamExt as _; use serde_json::value::RawValue; +use strum::IntoStaticStr; use thiserror::Error; use tracing::info; @@ -158,7 +159,13 @@ pub type BlockStream = Pin> pub type LogStream = Pin> + Send>>; /// Errors surfaced by [`ProviderPool`]. -#[derive(Debug, Error)] +/// +/// `IntoStaticStr` produces the snake_case variant name as +/// `&'static str` for metric labels and structured-log fields; the +/// per-variant Display still carries the detail via `thiserror`. +#[derive(Debug, Error, IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +#[non_exhaustive] pub enum ProviderError { /// Chain id absent from the engine config. #[error("unknown chain {0} (no engine.toml entry)")] diff --git a/crates/nexum-engine/src/manifest/error.rs b/crates/nexum-engine/src/manifest/error.rs index c69cc49..1a445d2 100644 --- a/crates/nexum-engine/src/manifest/error.rs +++ b/crates/nexum-engine/src/manifest/error.rs @@ -1,11 +1,17 @@ //! Error types for manifest parsing and capability enforcement. +use strum::IntoStaticStr; use thiserror::Error; use super::types::KNOWN_CAPABILITIES; /// Errors returned while loading or validating a manifest. -#[derive(Debug, Error)] +/// +/// `IntoStaticStr` exposes the snake_case variant name as a +/// `&'static str` for the manifest-loader's `tracing::warn!` / +/// `metrics::counter!` call sites. +#[derive(Debug, Error, IntoStaticStr)] +#[strum(serialize_all = "snake_case")] #[non_exhaustive] pub enum ParseError { /// Failed to read the manifest file from disk. From 2ea4b55289be46b626094d89bbfafdce1ed374b5 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 19:08:55 -0300 Subject: [PATCH 27/29] refactor(local-store): extract `local_store_err` map closure helper Audit reference: milestone-rubric-grant-audit-2026-06-25.md, duplication finding "internal_error('local-store', err.to_string()) map closures" (4 sites in one file). The four `local-store` host endpoints all `.map_err`-ed the same `StorageError -> HostError` conversion inline. Replace with a single `local_store_err(StorageError) -> HostError` free function so a future error-model change (richer kind, structured `data`) lands in one place instead of four call sites. Behaviour is identical; the helper is `fn`, not a closure, so codegen is one shared symbol. --- .../nexum-engine/src/host/impls/local_store.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/nexum-engine/src/host/impls/local_store.rs b/crates/nexum-engine/src/host/impls/local_store.rs index 66bcc52..fe4b9de 100644 --- a/crates/nexum-engine/src/host/impls/local_store.rs +++ b/crates/nexum-engine/src/host/impls/local_store.rs @@ -3,30 +3,40 @@ use crate::bindings::HostError; use crate::bindings::nexum; use crate::host::error::internal_error; +use crate::host::local_store_redb::StorageError; use crate::host::state::HostState; +/// Shared `StorageError` -> `HostError` conversion used by every +/// `local-store` host endpoint. Centralised so the `("local-store", +/// err.to_string())` shape stays consistent and a future error-model +/// change (richer kind, structured `data`) lands in one place +/// instead of four call sites. +fn local_store_err(err: StorageError) -> HostError { + internal_error("local-store", err.to_string()) +} + impl nexum::host::local_store::Host for HostState { async fn get(&mut self, key: String) -> Result>, HostError> { self.store .get(&self.module_namespace, &key) - .map_err(|err| internal_error("local-store", err.to_string())) + .map_err(local_store_err) } async fn set(&mut self, key: String, value: Vec) -> Result<(), HostError> { self.store .set(&self.module_namespace, &key, &value) - .map_err(|err| internal_error("local-store", err.to_string())) + .map_err(local_store_err) } async fn delete(&mut self, key: String) -> Result<(), HostError> { self.store .delete(&self.module_namespace, &key) - .map_err(|err| internal_error("local-store", err.to_string())) + .map_err(local_store_err) } async fn list_keys(&mut self, prefix: String) -> Result, HostError> { self.store .list_keys(&self.module_namespace, &prefix) - .map_err(|err| internal_error("local-store", err.to_string())) + .map_err(local_store_err) } } From 88ce8aaceead5eeeef52989328e74976931993e8 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 25 Jun 2026 19:09:07 -0300 Subject: [PATCH 28/29] fix(supervisor): emit nexum.toml deprecation via tracing::warn! Audit reference: milestone-rubric-grant-audit-2026-06-25.md, Major #10 (`eprintln!` in daemon-side manifest loader code). Every other deprecation site in the engine routes through `tracing` (the main binary installs a JSON `tracing_subscriber` and the manifest loader itself uses `warn!`). The supervisor's `nexum.toml` fallback was the lone `eprintln!` survivor, which bypasses the structured-log pipeline and breaks operators who set `RUST_LOG=warn,manifest=warn` for daemon log aggregation. Switches to `warn!(target: "manifest", path = %legacy.display(), ...)` so the deprecation surfaces through the same channel as the no-`[capabilities]` warning emitted a few lines later by `manifest::load`. --- crates/nexum-engine/src/supervisor.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index adbfebb..ad08081 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -116,8 +116,10 @@ impl Supervisor { } let legacy = dir.join("nexum.toml"); if legacy.exists() { - eprintln!( - "[deprecation] nexum.toml is deprecated; rename to module.toml \ + warn!( + target: "manifest", + path = %legacy.display(), + "nexum.toml is deprecated; rename to module.toml \ (ADR-0001). Support will be removed in 0.3." ); return Some(legacy); From e1af4f79edfc4acc25942cfc4ce4e1c13aec88ba Mon Sep 17 00:00:00 2001 From: Jean Neiverth Date: Mon, 29 Jun 2026 19:51:02 -0300 Subject: [PATCH 29/29] test(nexum-engine): cover untested error variants and concurrent access (issues 3, 5, 7) Add 12 tests across three M2 host modules: cow_orderbook (5 tests): Network error on dead server, 5xx passthrough, BadPath leniency, Decode on invalid/wrong-schema JSON for submit_order. provider_pool (3 tests): InvalidParams through full request path, Rpc error on unreachable node, Rpc error on malformed node response. Adds test_config helper for EngineConfig construction. local_store_redb (4 tests): Concurrent writes across namespaces, concurrent reads during writes, list_keys/delete race, stress test with 8 writers on one namespace. --- .../src/host/cow_orderbook/tests.rs | 83 +++++++++ .../src/host/local_store_redb/tests.rs | 162 ++++++++++++++++++ crates/nexum-engine/src/host/provider_pool.rs | 66 +++++++ 3 files changed, 311 insertions(+) diff --git a/crates/nexum-engine/src/host/cow_orderbook/tests.rs b/crates/nexum-engine/src/host/cow_orderbook/tests.rs index ef318c9..f66a253 100644 --- a/crates/nexum-engine/src/host/cow_orderbook/tests.rs +++ b/crates/nexum-engine/src/host/cow_orderbook/tests.rs @@ -206,3 +206,86 @@ fn sample_order_json() -> String { .expect("valid OrderCreation"); serde_json::to_string(&creation).expect("serialise OrderCreation") } + +#[tokio::test] +async fn request_rejects_malformed_path() { + // `Url::join` is very lenient for valid UTF-8 inputs. The + // `BadPath` variant fires only when `Url::join` returns a parse + // error, which is hard to provoke. Using a bare scheme-like + // string (`"://not-a-path"`) is NOT rejected because after + // stripping the leading `/` it is treated as a relative path + // component. Instead, feed a string that *will* reach the + // network but is handled by wiremock with a 404, confirming the + // passthrough returns Ok even for nonsensical paths. + let mock = MockServer::start().await; + let pool = pool_with_mainnet_at(&mock); + // wiremock returns 404 for any un-mocked route — the response + // body is still surfaced to the caller. + let result = pool + .request(Chain::Mainnet.id(), "GET", "://not-a-path", None) + .await; + assert!(result.is_ok(), "Url::join treats this as a relative path, so no BadPath error"); +} + +#[tokio::test] +async fn request_network_error_on_dead_server() { + // Build the pool against a port that no one is listening on. + // We use port 1 (TCP echo / privileged) which is never bound + // by user-space processes, guaranteeing a connection-refused. + let mut clients = std::collections::BTreeMap::new(); + clients.insert( + Chain::Mainnet.id(), + OrderBookApi::new_with_base_url( + "http://127.0.0.1:1/".parse().expect("valid url"), + ), + ); + let pool = OrderBookPool { + clients, + http: reqwest::Client::new(), + }; + let err = pool + .request(Chain::Mainnet.id(), "GET", "/api/v1/version", None) + .await + .unwrap_err(); + assert!(matches!(err, CowApiError::Network(_))); +} + +#[tokio::test] +async fn request_5xx_response_is_returned_verbatim() { + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v1/health")) + .respond_with( + ResponseTemplate::new(500).set_body_string(r#"{"error":"internal"}"#), + ) + .expect(1) + .mount(&mock) + .await; + + let pool = pool_with_mainnet_at(&mock); + let body = pool + .request(Chain::Mainnet.id(), "GET", "/api/v1/health", None) + .await + .expect("5xx body is returned, not an Err"); + assert_eq!(body, r#"{"error":"internal"}"#); +} + +#[tokio::test] +async fn submit_order_rejects_invalid_json() { + let pool = OrderBookPool::default(); + let err = pool + .submit_order_json(Chain::Mainnet.id(), b"not json") + .await + .unwrap_err(); + assert!(matches!(err, CowApiError::Decode(_))); +} + +#[tokio::test] +async fn submit_order_rejects_wrong_schema() { + let pool = OrderBookPool::default(); + let err = pool + .submit_order_json(Chain::Mainnet.id(), br#"{"valid":"json"}"#) + .await + .unwrap_err(); + assert!(matches!(err, CowApiError::Decode(_))); +} diff --git a/crates/nexum-engine/src/host/local_store_redb/tests.rs b/crates/nexum-engine/src/host/local_store_redb/tests.rs index 5c4feba..113e4ca 100644 --- a/crates/nexum-engine/src/host/local_store_redb/tests.rs +++ b/crates/nexum-engine/src/host/local_store_redb/tests.rs @@ -78,3 +78,165 @@ fn similar_names_differ() { let pb = namespace_prefix("module-b").unwrap(); assert_ne!(pa, pb); } + +// --------------------------------------------------------------------------- +// Concurrent access tests +// --------------------------------------------------------------------------- + +#[test] +fn concurrent_writes_from_different_namespaces() { + let (_dir, store) = fresh(); + + let handles: Vec<_> = (0..8) + .map(|i| { + let s = store.clone(); + std::thread::spawn(move || { + let ns = format!("ns-{i}"); + for j in 0..100 { + let key = format!("key-{j}"); + let val = format!("val-{i}-{j}").into_bytes(); + s.set(&ns, &key, &val).unwrap(); + } + }) + }) + .collect(); + + for h in handles { + h.join().expect("thread panicked"); + } + + for i in 0..8 { + let ns = format!("ns-{i}"); + for j in 0..100 { + let key = format!("key-{j}"); + let expected = format!("val-{i}-{j}").into_bytes(); + assert_eq!( + store.get(&ns, &key).unwrap().as_deref(), + Some(expected.as_slice()), + ); + } + } +} + +#[test] +fn concurrent_reads_during_writes() { + let (_dir, store) = fresh(); + + // Pre-populate namespace "rw" with 50 keys. + for j in 0..50 { + store + .set("rw", &format!("k-{j}"), b"old") + .unwrap(); + } + + let writer_store = store.clone(); + let writer = std::thread::spawn(move || { + for j in 0..50 { + writer_store + .set("rw", &format!("k-{j}"), b"new") + .unwrap(); + } + }); + + let readers: Vec<_> = (0..4) + .map(|_| { + let s = store.clone(); + std::thread::spawn(move || { + for _ in 0..100 { + for j in 0..50 { + let val = s.get("rw", &format!("k-{j}")).unwrap(); + let val = val.expect("key must exist"); + assert!( + val == b"old" || val == b"new", + "unexpected value: {:?}", + val, + ); + } + } + }) + }) + .collect(); + + writer.join().expect("writer panicked"); + for r in readers { + r.join().expect("reader panicked"); + } + + // Final state: all keys must be "new". + for j in 0..50 { + assert_eq!( + store.get("rw", &format!("k-{j}")).unwrap().as_deref(), + Some(&b"new"[..]), + ); + } +} + +#[test] +fn list_keys_races_with_delete() { + let (_dir, store) = fresh(); + + // Pre-populate namespace "race" with 100 keys. + for i in 0..100 { + store + .set("race", &format!("k:{i}"), b"x") + .unwrap(); + } + + let deleter_store = store.clone(); + let deleter = std::thread::spawn(move || { + for i in 0..100 { + deleter_store + .delete("race", &format!("k:{i}")) + .unwrap(); + } + }); + + let lister_store = store.clone(); + let lister = std::thread::spawn(move || { + for _ in 0..50 { + let keys = lister_store.list_keys("race", "k:").unwrap(); + assert!( + keys.len() <= 100, + "list_keys returned more keys than expected: {}", + keys.len(), + ); + } + }); + + deleter.join().expect("deleter panicked"); + lister.join().expect("lister panicked"); +} + +#[test] +fn stress_many_writers_one_namespace() { + let (_dir, store) = fresh(); + + let handles: Vec<_> = (0..8) + .map(|i| { + let s = store.clone(); + std::thread::spawn(move || { + for j in 0..100 { + let key = format!("t{i}-k{j}"); + let val = format!("v-{i}-{j}").into_bytes(); + s.set("shared", &key, &val).unwrap(); + } + }) + }) + .collect(); + + for h in handles { + h.join().expect("thread panicked"); + } + + // Verify all 800 keys are present with correct values. + for i in 0..8 { + for j in 0..100 { + let key = format!("t{i}-k{j}"); + let expected = format!("v-{i}-{j}").into_bytes(); + assert_eq!( + store.get("shared", &key).unwrap().as_deref(), + Some(expected.as_slice()), + ); + } + } +} diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index fa90889..1615ff3 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -238,4 +238,70 @@ mod tests { let result = RawValue::from_string(bad.to_owned()); assert!(result.is_err(), "invalid JSON should fail RawValue parse"); } + + /// Helper: build an `EngineConfig` with a single HTTP chain entry. + fn test_config(chain_id: u64, rpc_url: &str) -> EngineConfig { + use crate::engine_config::{ChainConfig, EngineConfig}; + let mut chains = BTreeMap::new(); + chains.insert( + chain_id, + ChainConfig { + rpc_url: rpc_url.to_owned(), + }, + ); + EngineConfig { + chains, + ..Default::default() + } + } + + #[tokio::test] + async fn invalid_params_through_request_produces_error() { + let cfg = test_config(1, "http://127.0.0.1:1"); + let pool = ProviderPool::from_config(&cfg).await.unwrap(); + let err = pool + .request(1, "eth_blockNumber".into(), "not json {{{".into()) + .await + .unwrap_err(); + assert!( + matches!(err, ProviderError::InvalidParams { .. }), + "expected InvalidParams, got: {err:?}" + ); + } + + #[tokio::test] + async fn rpc_error_on_unreachable_node() { + let cfg = test_config(1, "http://127.0.0.1:1"); + let pool = ProviderPool::from_config(&cfg).await.unwrap(); + let err = pool + .request(1, "eth_blockNumber".into(), "[]".into()) + .await + .unwrap_err(); + assert!( + matches!(err, ProviderError::Rpc { .. }), + "expected Rpc error, got: {err:?}" + ); + } + + #[tokio::test] + async fn rpc_error_on_malformed_node_response() { + use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; + + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_string("not json")) + .mount(&server) + .await; + + let cfg = test_config(1, &server.uri()); + let pool = ProviderPool::from_config(&cfg).await.unwrap(); + let err = pool + .request(1, "eth_blockNumber".into(), "[]".into()) + .await + .unwrap_err(); + assert!( + matches!(err, ProviderError::Rpc { .. }), + "expected Rpc error from malformed response, got: {err:?}" + ); + } }